Skip to content

Commit d7cd656

Browse files
alexvrualexvru
authored
Allow changing static PDisk configs through distconf (#25733)
1 parent baebf65 commit d7cd656

File tree

8 files changed

+86
-28
lines changed

8 files changed

+86
-28
lines changed

‎ydb/core/blobstorage/nodewarden/distconf_invoke_storage_config.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -476,6 +476,7 @@ namespace NKikimr::NStorage {
476476

477477
case EControllerOp::OTHER:
478478
record.SetOperation(NKikimrBlobStorage::TEvControllerDistconfRequest::ValidateConfig);
479+
record.MutableStorageConfig()->PackFrom(ProposedStorageConfig);
479480
break;
480481

481482
case EControllerOp::UNSET:

‎ydb/core/blobstorage/nodewarden/node_warden_impl.cpp‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1395,6 +1395,10 @@ bool NKikimr::NStorage::DeriveStorageConfig(const NKikimrConfig::TAppConfig& app
13951395
const auto& bsFrom = appConfig.GetBlobStorageConfig();
13961396
auto *bsTo = config->MutableBlobStorageConfig();
13971397

1398+
const auto hasStaticGroupInfo = [](const NKikimrBlobStorage::TNodeWardenServiceSet& ss) {
1399+
return ss.PDisksSize() && ss.VDisksSize() && ss.GroupsSize();
1400+
};
1401+
13981402
if (bsFrom.HasServiceSet()) {
13991403
const auto& ssFrom = bsFrom.GetServiceSet();
14001404
auto *ssTo = bsTo->MutableServiceSet();
@@ -1411,10 +1415,6 @@ bool NKikimr::NStorage::DeriveStorageConfig(const NKikimrConfig::TAppConfig& app
14111415
ssTo->ClearReplBrokerConfig();
14121416
}
14131417

1414-
const auto hasStaticGroupInfo = [](const NKikimrBlobStorage::TNodeWardenServiceSet& ss) {
1415-
return ss.PDisksSize() && ss.VDisksSize() && ss.GroupsSize();
1416-
};
1417-
14181418
// update static group information unless distconf is enabled
14191419
if (!hasStaticGroupInfo(ssFrom) && config->GetSelfManagementConfig().GetEnabled()) {
14201420
// distconf enabled, keep it as is
@@ -1548,6 +1548,25 @@ bool NKikimr::NStorage::DeriveStorageConfig(const NKikimrConfig::TAppConfig& app
15481548
bsTo->ClearBscSettings();
15491549
}
15501550

1551+
// copy PDiskConfig from DefineHostConfig/DefineBox if this section is managed automatically
1552+
if (!hasStaticGroupInfo(bsFrom.GetServiceSet()) && config->GetSelfManagementConfig().GetEnabled()) {
1553+
THashMap<std::tuple<ui32, TString>, NKikimrBlobStorage::TPDiskConfig> pdiskConfigs;
1554+
auto callback = [&](const auto& node, const auto& drive) {
1555+
if (drive.HasPDiskConfig()) {
1556+
pdiskConfigs.emplace(std::make_tuple(node.GetNodeId(), drive.GetPath()), drive.GetPDiskConfig());
1557+
}
1558+
};
1559+
EnumerateConfigDrives(*config, 0, callback, nullptr, true);
1560+
for (auto& pdisk : *bsTo->MutableServiceSet()->MutablePDisks()) {
1561+
const auto key = std::make_tuple(pdisk.GetNodeID(), pdisk.GetPath());
1562+
if (const auto it = pdiskConfigs.find(key); it != pdiskConfigs.end()) {
1563+
pdisk.MutablePDiskConfig()->CopyFrom(it->second);
1564+
} else {
1565+
pdisk.ClearPDiskConfig();
1566+
}
1567+
}
1568+
}
1569+
15511570
// copy nameservice-related things
15521571
if (!appConfig.HasNameserviceConfig()) {
15531572
*errorReason = "origin config missing mandatory NameserviceConfig section";

‎ydb/core/mind/bscontroller/bsc.cpp‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -593,6 +593,8 @@ std::unique_ptr<TEvBlobStorage::TEvControllerConfigRequest> TBlobStorageControll
593593
request->SetRollback(true);
594594
}
595595

596+
request->MutableStorageConfig()->PackFrom(storageConfig);
597+
596598
if (request->CommandSize()) {
597599
return ev;
598600
}
@@ -757,21 +759,24 @@ void TBlobStorageController::Handle(TEvBlobStorage::TEvControllerDistconfRequest
757759
break;
758760
}
759761

760-
const TString& effectiveConfig = storageYaml ? *storageYaml : *mainYaml;
761762
NKikimrBlobStorage::TStorageConfig storageConfig;
762-
763-
try {
764-
NKikimrConfig::TAppConfig appConfig = NYaml::Parse(effectiveConfig);
765-
TString errorReason;
766-
if (!NKikimr::NStorage::DeriveStorageConfig(appConfig, &storageConfig, &errorReason)) {
763+
if (record.HasStorageConfig()) {
764+
record.GetStorageConfig().UnpackTo(&storageConfig);
765+
} else {
766+
const TString& effectiveConfig = storageYaml ? *storageYaml : *mainYaml;
767+
try {
768+
NKikimrConfig::TAppConfig appConfig = NYaml::Parse(effectiveConfig);
769+
TString errorReason;
770+
if (!NKikimr::NStorage::DeriveStorageConfig(appConfig, &storageConfig, &errorReason)) {
771+
rr.SetStatus(NKikimrBlobStorage::TEvControllerDistconfResponse::Error);
772+
rr.SetErrorReason("failed to derive storage config: " + errorReason);
773+
break;
774+
}
775+
} catch (const std::exception& ex) {
767776
rr.SetStatus(NKikimrBlobStorage::TEvControllerDistconfResponse::Error);
768-
rr.SetErrorReason("failed to derive storage config: " + errorReason);
777+
rr.SetErrorReason(TStringBuilder() << "failed to parse YAML: " << ex.what());
769778
break;
770779
}
771-
} catch (const std::exception& ex) {
772-
rr.SetStatus(NKikimrBlobStorage::TEvControllerDistconfResponse::Error);
773-
rr.SetErrorReason(TStringBuilder() << "failed to parse YAML: " << ex.what());
774-
break;
775780
}
776781

777782
const ui64 cookie = NextValidationCookie++;

‎ydb/core/mind/bscontroller/config.h‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,9 @@ namespace NKikimr {
128128
const ui32 DefaultMaxSlots;
129129

130130
// static pdisk/vdisk states
131+
std::map<TVSlotId, TStaticVSlotInfo> NewStaticVSlots;
132+
std::map<TPDiskId, TStaticPDiskInfo> NewStaticPDisks;
133+
std::map<TGroupId, TStaticGroupInfo> NewStaticGroups;
131134
std::map<TVSlotId, TStaticVSlotInfo>& StaticVSlots;
132135
std::map<TPDiskId, TStaticPDiskInfo>& StaticPDisks;
133136
std::map<TGroupId, TStaticGroupInfo>& StaticGroups;
@@ -147,7 +150,7 @@ namespace NKikimr {
147150

148151
public:
149152
TConfigState(TBlobStorageController &controller, const THostRecordMap &hostRecords, TInstant timestamp,
150-
TMonotonic mono)
153+
TMonotonic mono, const NKikimrBlobStorage::TStorageConfig *storageConfig = nullptr)
151154
: Self(controller)
152155
, HostConfigs(&controller.HostConfigs)
153156
, Boxes(&controller.Boxes)
@@ -168,14 +171,33 @@ namespace NKikimr {
168171
, Mono(mono)
169172
, DonorMode(controller.DonorMode)
170173
, DefaultMaxSlots(controller.DefaultMaxSlots)
171-
, StaticVSlots(controller.StaticVSlots)
172-
, StaticPDisks(controller.StaticPDisks)
173-
, StaticGroups(controller.StaticGroups)
174+
, StaticVSlots(storageConfig && storageConfig->HasBlobStorageConfig() ? NewStaticVSlots : controller.StaticVSlots)
175+
, StaticPDisks(storageConfig && storageConfig->HasBlobStorageConfig() ? NewStaticPDisks : controller.StaticPDisks)
176+
, StaticGroups(storageConfig && storageConfig->HasBlobStorageConfig() ? NewStaticGroups : controller.StaticGroups)
174177
, SerialManagementStage(&controller.SerialManagementStage)
175178
, StoragePoolStat(*controller.StoragePoolStat)
176179
, BridgeInfo(controller.BridgeInfo)
177180
{
178181
Y_ABORT_UNLESS(HostRecords);
182+
if (storageConfig && storageConfig->HasBlobStorageConfig()) {
183+
const auto& bsConfig = storageConfig->GetBlobStorageConfig();
184+
const auto& ss = bsConfig.GetServiceSet();
185+
for (const auto& pdisk : ss.GetPDisks()) {
186+
const TPDiskId pdiskId(pdisk.GetNodeID(), pdisk.GetPDiskID());
187+
NewStaticPDisks.try_emplace(pdiskId, pdisk, controller.StaticPDisks);
188+
}
189+
for (const auto& vslot : ss.GetVDisks()) {
190+
const auto& location = vslot.GetVDiskLocation();
191+
const TPDiskId pdiskId(location.GetNodeID(), location.GetPDiskID());
192+
const TVSlotId vslotId(pdiskId, location.GetVDiskSlotID());
193+
NewStaticVSlots.try_emplace(vslotId, vslot, controller.StaticVSlots, Mono);
194+
++StaticPDisks.at(pdiskId).StaticSlotUsage;
195+
}
196+
for (const auto& group : ss.GetGroups()) {
197+
const auto groupId = TGroupId::FromProto(&group, &NKikimrBlobStorage::TGroupInfo::GetGroupID);
198+
NewStaticGroups.try_emplace(groupId, group, controller.StaticGroups);
199+
}
200+
}
179201
}
180202

181203
void Commit() {

‎ydb/core/mind/bscontroller/config_cmd.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,12 @@ namespace NKikimr::NBsController {
185185
}
186186

187187
const auto& hostRecords = EnforceHostRecords ? *EnforceHostRecords : Self->HostRecords;
188-
State.emplace(*Self, hostRecords, TActivationContext::Now(), TActivationContext::Monotonic());
188+
std::optional<NKikimrBlobStorage::TStorageConfig> storageConfig;
189+
if (Cmd.HasStorageConfig() && Self->SelfManagementEnabled) {
190+
Cmd.GetStorageConfig().UnpackTo(&storageConfig.emplace());
191+
}
192+
State.emplace(*Self, hostRecords, TActivationContext::Now(), TActivationContext::Monotonic(),
193+
storageConfig ? &storageConfig.value() : nullptr);
189194
State->CheckConsistency();
190195

191196
TString m;

‎ydb/core/mind/bscontroller/impl.h‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2420,15 +2420,15 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
24202420
TGroupInfo::TGroupStatus Status;
24212421
bool LayoutCorrect = true;
24222422

2423-
TStaticGroupInfo(const NKikimrBlobStorage::TGroupInfo& group, std::map<TGroupId, TStaticGroupInfo>& prev) {
2423+
TStaticGroupInfo(const NKikimrBlobStorage::TGroupInfo& group, const std::map<TGroupId, TStaticGroupInfo>& prev) {
24242424
TStringStream err;
24252425
Info = TBlobStorageGroupInfo::Parse(group, nullptr, &err);
24262426
Y_VERIFY_DEBUG_S(Info, "failed to parse static group, error# " << err.Str());
24272427
if (!Info) {
24282428
return;
24292429
}
24302430
if (const auto it = prev.find(Info->GroupID); it != prev.end()) {
2431-
TStaticGroupInfo& item = it->second;
2431+
const TStaticGroupInfo& item = it->second;
24322432
Status = item.Status;
24332433
LayoutCorrect = item.LayoutCorrect;
24342434
}
@@ -2456,15 +2456,15 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
24562456
bool MetricsDirty = false;
24572457

24582458
TStaticVSlotInfo(const NKikimrBlobStorage::TNodeWardenServiceSet::TVDisk& vdisk,
2459-
std::map<TVSlotId, TStaticVSlotInfo>& prev, TMonotonic mono)
2459+
const std::map<TVSlotId, TStaticVSlotInfo>& prev, TMonotonic mono)
24602460
: VDiskId(VDiskIDFromVDiskID(vdisk.GetVDiskID()))
24612461
, VDiskKind(vdisk.GetVDiskKind())
24622462
{
24632463
const auto& loc = vdisk.GetVDiskLocation();
24642464
const TVSlotId vslotId(loc.GetNodeID(), loc.GetPDiskID(), loc.GetVDiskSlotID());
24652465
if (const auto it = prev.find(vslotId); it != prev.end()) {
2466-
TStaticVSlotInfo& item = it->second;
2467-
VDiskMetrics = std::move(item.VDiskMetrics);
2466+
const TStaticVSlotInfo& item = it->second;
2467+
VDiskMetrics = item.VDiskMetrics;
24682468
VDiskStatus = item.VDiskStatus;
24692469
VDiskStatusTimestamp = item.VDiskStatusTimestamp;
24702470
ReadySince = item.ReadySince;
@@ -2493,7 +2493,7 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
24932493
std::optional<NKikimrBlobStorage::TPDiskMetrics> PDiskMetrics;
24942494

24952495
TStaticPDiskInfo(const NKikimrBlobStorage::TNodeWardenServiceSet::TPDisk& pdisk,
2496-
std::map<TPDiskId, TStaticPDiskInfo>& prev)
2496+
const std::map<TPDiskId, TStaticPDiskInfo>& prev)
24972497
: NodeId(pdisk.GetNodeID())
24982498
, PDiskId(pdisk.GetPDiskID())
24992499
, Path(pdisk.GetPath())
@@ -2511,8 +2511,8 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
25112511

25122512
const TPDiskId pdiskId(NodeId, PDiskId);
25132513
if (const auto it = prev.find(pdiskId); it != prev.end()) {
2514-
TStaticPDiskInfo& item = it->second;
2515-
PDiskMetrics = std::move(item.PDiskMetrics);
2514+
const TStaticPDiskInfo& item = it->second;
2515+
PDiskMetrics = item.PDiskMetrics;
25162516
}
25172517
}
25182518

‎ydb/core/protos/blobstorage.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import "ydb/core/protos/blobstorage_config.proto";
88
import "ydb/core/protos/blobstorage_disk.proto";
99
import "ydb/core/protos/blobstorage_disk_color.proto";
1010
import "ydb/core/protos/node_whiteboard.proto";
11+
import "google/protobuf/any.proto";
1112

1213
package NKikimrBlobStorage;
1314
option java_package = "ru.yandex.kikimr.proto";
@@ -1563,6 +1564,7 @@ message TEvControllerDistconfRequest {
15631564
optional uint64 ExpectedStorageConfigVersion = 5;
15641565
optional string PeerName = 6;
15651566
optional bytes UserToken = 7;
1567+
optional google.protobuf.Any StorageConfig = 8; // TStorageConfig with this request that is going to be applied after operation
15661568
}
15671569

15681570
message TEvControllerDistconfResponse {

‎ydb/core/protos/blobstorage_config.proto‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import "ydb/core/protos/blobstorage_disk_color.proto";
77
import "ydb/core/protos/blobstorage_pdisk_config.proto";
88
import "ydb/core/protos/blob_depot_config.proto";
99
import "ydb/library/actors/protos/interconnect.proto";
10+
import "google/protobuf/any.proto";
1011

1112
package NKikimrBlobStorage;
1213

@@ -686,6 +687,9 @@ message TConfigRequest {
686687

687688
// requesting user identifier
688689
string UserSID = 13;
690+
691+
// current TStorageConfig against which we are processing this request
692+
google.protobuf.Any StorageConfig = 14;
689693
}
690694

691695
enum ETriStateBool {

0 commit comments

Comments
 (0)