optimize cs loading (#29001)

swalrus1 · web-flow · commit 7d8b0c265bbf · 2025-11-19T12:58:31.000+03:00
diff --git a/ydb/core/tx/columnshard/counters/engine_logs.cpp b/ydb/core/tx/columnshard/counters/engine_logs.cpp
@@ -12,7 +12,7 @@ namespace NKikimr::NColumnShard {
 namespace {
 
 ui64 GetBadPortionSizeLimit() {
-    return HasAppData() ? AppDataVerified().ColumnShardConfig.GetBadPortionSizeLimit() : 512_KB;
+    return HasAppData() ? AppData()->ColumnShardConfig.GetBadPortionSizeLimit() : 512_KB;
 }
 
 }
diff --git a/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp b/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp
@@ -40,14 +40,14 @@ TPortionMetaConstructor::TPortionMetaConstructor(const TPortionMeta& meta) {
 
 TPortionMeta TPortionMetaConstructor::Build() {
     AFL_VERIFY(FirstAndLastPK);
-    TMemoryProfileGuard mGuard1("meta_construct/pk");
+    TMemoryProfileGuard mGuard1("meta_construct/pk", IS_DEBUG_LOG_ENABLED(NKikimrServices::TX_COLUMNSHARD_SCAN_MEMORY));
     static TAtomicCounter sumValues = 0;
     static TAtomicCounter sumValuesMeta = 0;
     static TAtomicCounter countValues = 0;
 //    FirstAndLastPK->Reallocate();
     AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("memory_size", FirstAndLastPK->GetMemorySize())("data_size", FirstAndLastPK->GetDataSize())(
         "sum", sumValues.Add(FirstAndLastPK->GetMemorySize()))("count", countValues.Inc());
-    TMemoryProfileGuard mGuard("meta_construct/others");
+    TMemoryProfileGuard mGuard("meta_construct/others", IS_DEBUG_LOG_ENABLED(NKikimrServices::TX_COLUMNSHARD_SCAN_MEMORY));
     AFL_VERIFY(RecordSnapshotMin);
     AFL_VERIFY(RecordSnapshotMax);
     TPortionMeta result(*FirstAndLastPK, *RecordSnapshotMin, *RecordSnapshotMax);
diff --git a/ydb/core/tx/columnshard/engines/portions/constructor_portion.cpp b/ydb/core/tx/columnshard/engines/portions/constructor_portion.cpp
@@ -15,14 +15,16 @@ std::shared_ptr<TPortionInfo> TPortionInfoConstructor::Build() {
     AFL_VERIFY(!Constructed);
     Constructed = true;
     std::shared_ptr<TPortionInfo> result;
+    const bool enableGranularMemoryProfile = IS_DEBUG_LOG_ENABLED(NKikimrServices::TX_COLUMNSHARD_SCAN_MEMORY);
+    TMemoryProfileGuard mGuard0("portion_construct/general", !enableGranularMemoryProfile);
     {
-        TMemoryProfileGuard mGuard0("portion_construct/meta::" + ::ToString(GetType()));
+        TMemoryProfileGuard mGuard0("portion_construct/meta::" + ::ToString(GetType()), enableGranularMemoryProfile);
         auto meta = MetaConstructor.Build();
-        TMemoryProfileGuard mGuard("portion_construct/main::" + ::ToString(GetType()));
+        TMemoryProfileGuard mGuard("portion_construct/main::" + ::ToString(GetType()), enableGranularMemoryProfile);
         result = BuildPortionImpl(std::move(meta));
     }
     {
-        TMemoryProfileGuard mGuard1("portion_construct/others::" + ::ToString(GetType()));
+        TMemoryProfileGuard mGuard1("portion_construct/others::" + ::ToString(GetType()), enableGranularMemoryProfile);
         AFL_VERIFY(PathId);
         result->PathId = PathId;
         result->PortionId = GetPortionIdVerified();
diff --git a/ydb/core/tx/columnshard/engines/portions/meta.h b/ydb/core/tx/columnshard/engines/portions/meta.h
@@ -117,7 +117,7 @@ class TPortionMeta {
         , LastPKRow(pk.GetLast().GetContent())
         , RecordSnapshotMin(min)
         , RecordSnapshotMax(max) {
-        AFL_VERIFY(IndexKeyStart() <= IndexKeyEnd())("start", IndexKeyStart().DebugString())("end", IndexKeyEnd().DebugString());
+        AFL_VERIFY_DEBUG(IndexKeyStart() <= IndexKeyEnd())("start", IndexKeyStart().DebugString())("end", IndexKeyEnd().DebugString());
     }
     TSnapshot RecordSnapshotMin;
     TSnapshot RecordSnapshotMax;
diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h
@@ -238,6 +238,13 @@ class TPortionInfo {
         RuntimeFeatures &= (Max<TRuntimeFeatures>() - (TRuntimeFeatures)feature);
     }
 
+    bool IsDefaultTier(const TString& defaultTierName) const {
+        if (GetMeta().GetTierName()) {
+            return GetMeta().GetTierName() == defaultTierName;
+        }
+        return true;
+    }
+
     TString GetTierNameDef(const TString& defaultTierName) const {
         if (GetMeta().GetTierName()) {
             return GetMeta().GetTierName();
@@ -327,7 +334,7 @@ class TPortionInfo {
         if (HasRemoveSnapshot()) {
             return NPortion::INACTIVE;
         }
-        if (GetTierNameDef(NBlobOperations::TGlobal::DefaultStorageId) != NBlobOperations::TGlobal::DefaultStorageId) {
+        if (!IsDefaultTier(NBlobOperations::TGlobal::DefaultStorageId)) {
             return NPortion::EVICTED;
         }
         return GetPortionType() == EPortionType::Compacted ? NPortion::EProduced::SPLIT_COMPACTED : NPortion::EProduced::INSERTED;
diff --git a/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp b/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp
@@ -188,14 +188,14 @@ void TGranuleMeta::ResetOptimizer(const std::shared_ptr<NStorageOptimizer::IOpti
     NStorageOptimizer::IOptimizerPlannerConstructor::TBuildContext context(PathId, storages, pkSchema);
     OptimizerPlanner = constructor->BuildPlanner(context).DetachResult();
     AFL_VERIFY(!!OptimizerPlanner);
-    THashMap<ui64, std::shared_ptr<TPortionInfo>> portions;
+    std::vector<std::shared_ptr<TPortionInfo>> portions;
     for (auto&& i : Portions) {
         if (i.second->HasRemoveSnapshot()) {
             continue;
         }
-        portions.emplace(i.first, i.second);
+        portions.emplace_back(i.second);
     }
-    OptimizerPlanner->ModifyPortions(portions, {});
+    OptimizerPlanner->ModifyPortions(std::move(portions), {});
 }
 /*
 
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.cpp
@@ -10,12 +10,12 @@ std::vector<std::shared_ptr<TColumnEngineChanges>> IOptimizerPlanner::GetOptimiz
 }
 
 IOptimizerPlanner::TModificationGuard& IOptimizerPlanner::TModificationGuard::AddPortion(const std::shared_ptr<TPortionInfo>& portion) {
-    AFL_VERIFY(AddPortions.emplace(portion->GetPortionId(), portion).second);
+    AddPortions.emplace_back(portion);
     return*this;
 }
 
 IOptimizerPlanner::TModificationGuard& IOptimizerPlanner::TModificationGuard::RemovePortion(const std::shared_ptr<TPortionInfo>& portion) {
-    AFL_VERIFY(RemovePortions.emplace(portion->GetPortionId(), portion).second);
+    RemovePortions.emplace_back(portion);
     return*this;
 }
 
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h
@@ -112,7 +112,7 @@ class IOptimizerPlanner {
 
 protected:
     virtual void DoModifyPortions(
-        const THashMap<ui64, std::shared_ptr<TPortionInfo>>& add, const THashMap<ui64, std::shared_ptr<TPortionInfo>>& remove) = 0;
+        const std::vector<std::shared_ptr<TPortionInfo>>& add, const std::vector<std::shared_ptr<TPortionInfo>>& remove) = 0;
     virtual std::vector<std::shared_ptr<TColumnEngineChanges>> DoGetOptimizationTasks(
         std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const = 0;
     virtual TOptimizationPriority DoGetUsefulMetric() const = 0;
@@ -208,8 +208,8 @@ class IOptimizerPlanner {
     class TModificationGuard: TNonCopyable {
     private:
         IOptimizerPlanner& Owner;
-        THashMap<ui64, std::shared_ptr<TPortionInfo>> AddPortions;
-        THashMap<ui64, std::shared_ptr<TPortionInfo>> RemovePortions;
+        std::vector<std::shared_ptr<TPortionInfo>> AddPortions;
+        std::vector<std::shared_ptr<TPortionInfo>> RemovePortions;
 
     public:
         TModificationGuard& AddPortion(const std::shared_ptr<TPortionInfo>& portion);
@@ -241,7 +241,7 @@ class IOptimizerPlanner {
         return DoSerializeToJsonVisual();
     }
 
-    void ModifyPortions(const THashMap<ui64, std::shared_ptr<TPortionInfo>>& add, const THashMap<ui64, std::shared_ptr<TPortionInfo>>& remove) {
+    void ModifyPortions(const std::vector<std::shared_ptr<TPortionInfo>>& add, const std::vector<std::shared_ptr<TPortionInfo>>& remove) {
         NActors::TLogContextGuard g(NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("path_id", PathId));
         LocalPortionsCount.Add(add.size());
         LocalPortionsCount.Sub(remove.size());
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h
@@ -1231,9 +1231,9 @@ class TOptimizerPlanner: public IOptimizerPlanner {
         return Buckets.IsLocked(dataLocksManager);
     }
 
-    virtual void DoModifyPortions(const THashMap<ui64, TPortionInfo::TPtr>& add, const THashMap<ui64, TPortionInfo::TPtr>& remove) override {
+    virtual void DoModifyPortions(const std::vector<TPortionInfo::TPtr>& add, const std::vector<TPortionInfo::TPtr>& remove) override {
         const TInstant now = TInstant::Now();
-        for (auto&& [_, i] : remove) {
+        for (auto&& i : remove) {
             if (i->GetMeta().GetTierName() != IStoragesManager::DefaultStorageId && i->GetMeta().GetTierName() != "") {
                 continue;
             }
@@ -1242,7 +1242,7 @@ class TOptimizerPlanner: public IOptimizerPlanner {
                 Counters->OptimizersCount->Sub(1);
             }
         }
-        for (auto&& [_, i] : add) {
+        for (auto&& i : add) {
             if (i->GetMeta().GetTierName() != IStoragesManager::DefaultStorageId && i->GetMeta().GetTierName() != "") {
                 continue;
             }
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h
@@ -67,12 +67,12 @@ class TOptimizerPlanner: public IOptimizerPlanner {
         return false;
     }
 
-    virtual void DoModifyPortions(const THashMap<ui64, TPortionInfo::TPtr>& add, const THashMap<ui64, TPortionInfo::TPtr>& remove) override {
+    virtual void DoModifyPortions(const std::vector<TPortionInfo::TPtr>& add, const std::vector<TPortionInfo::TPtr>& remove) override {
         std::vector<std::vector<TPortionInfo::TPtr>> removePortionsByLevel;
         removePortionsByLevel.resize(Levels.size());
         std::vector<std::vector<TPortionInfo::TPtr>> addPortionsByLevels;
         addPortionsByLevels.resize(Levels.size());
-        for (auto&& [_, i] : remove) {
+        for (auto&& i : remove) {
             if (i->GetProduced() == NPortion::EProduced::EVICTED) {
                 continue;
             }
@@ -81,7 +81,7 @@ class TOptimizerPlanner: public IOptimizerPlanner {
             removePortionsByLevel[i->GetCompactionLevel()].emplace_back(i);
         }
         std::vector<TPortionInfo::TPtr> problemPortions;
-        for (auto&& [_, i] : add) {
+        for (auto&& i : add) {
             if (i->GetProduced() == NPortion::EProduced::EVICTED) {
                 continue;
             }
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp
@@ -510,9 +510,9 @@ class TOptimizerPlanner : public IOptimizerPlanner, private TSettings {
         return false;
     }
 
-    void DoModifyPortions(const THashMap<ui64, TPortionInfo::TPtr>& add, const THashMap<ui64, TPortionInfo::TPtr>& remove) override {
+    void DoModifyPortions(const std::vector<TPortionInfo::TPtr>& add, const std::vector<TPortionInfo::TPtr>& remove) override {
         std::vector<TPortionInfo::TPtr> sortedRemove;
-        for (const auto& [_, p] : remove) {
+        for (const auto& p : remove) {
             sortedRemove.push_back(p);
         }
         std::sort(sortedRemove.begin(), sortedRemove.end(), [](const auto& a, const auto& b) {
@@ -530,7 +530,7 @@ class TOptimizerPlanner : public IOptimizerPlanner, private TSettings {
         }
 
         std::vector<TPortionInfo::TPtr> sortedAdd;
-        for (const auto& [_, p] : add) {
+        for (const auto& p : add) {
             sortedAdd.push_back(p);
         }
         std::sort(sortedAdd.begin(), sortedAdd.end(), [](const auto& a, const auto& b) {

Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,7 @@ namespace NKikimr::NColumnShard {`
`12`	`12`	`namespace {`
`13`	`13`
`14`	`14`	`ui64 GetBadPortionSizeLimit() {`
`15`		`- return HasAppData() ? AppDataVerified().ColumnShardConfig.GetBadPortionSizeLimit() : 512_KB;`
	`15`	`+ return HasAppData() ? AppData()->ColumnShardConfig.GetBadPortionSizeLimit() : 512_KB;`
`16`	`16`	`}`
`17`	`17`
`18`	`18`	`}`
Original file line number	Diff line number	Diff line change
`@@ -117,7 +117,7 @@ class TPortionMeta {`
`117`	`117`	`, LastPKRow(pk.GetLast().GetContent())`
`118`	`118`	`, RecordSnapshotMin(min)`
`119`	`119`	`, RecordSnapshotMax(max) {`
`120`		`- AFL_VERIFY(IndexKeyStart() <= IndexKeyEnd())("start", IndexKeyStart().DebugString())("end", IndexKeyEnd().DebugString());`
	`120`	`+ AFL_VERIFY_DEBUG(IndexKeyStart() <= IndexKeyEnd())("start", IndexKeyStart().DebugString())("end", IndexKeyEnd().DebugString());`
`121`	`121`	`}`
`122`	`122`	`TSnapshot RecordSnapshotMin;`
`123`	`123`	`TSnapshot RecordSnapshotMax;`
Original file line number	Diff line number	Diff line change
`@@ -188,14 +188,14 @@ void TGranuleMeta::ResetOptimizer(const std::shared_ptr<NStorageOptimizer::IOpti`
`188`	`188`	`NStorageOptimizer::IOptimizerPlannerConstructor::TBuildContext context(PathId, storages, pkSchema);`
`189`	`189`	`OptimizerPlanner = constructor->BuildPlanner(context).DetachResult();`
`190`	`190`	`AFL_VERIFY(!!OptimizerPlanner);`
`191`		`- THashMap<ui64, std::shared_ptr<TPortionInfo>> portions;`
	`191`	`+ std::vector<std::shared_ptr<TPortionInfo>> portions;`
`192`	`192`	`for (auto&& i : Portions) {`
`193`	`193`	`if (i.second->HasRemoveSnapshot()) {`
`194`	`194`	`continue;`
`195`	`195`	`}`
`196`		`- portions.emplace(i.first, i.second);`
	`196`	`+ portions.emplace_back(i.second);`
`197`	`197`	`}`
`198`		`- OptimizerPlanner->ModifyPortions(portions, {});`
	`198`	`+ OptimizerPlanner->ModifyPortions(std::move(portions), {});`
`199`	`199`	`}`
`200`	`200`	`/*`
`201`	`201`
Original file line number	Diff line number	Diff line change
`@@ -10,12 +10,12 @@ std::vector<std::shared_ptr<TColumnEngineChanges>> IOptimizerPlanner::GetOptimiz`
`10`	`10`	`}`
`11`	`11`
`12`	`12`	`IOptimizerPlanner::TModificationGuard& IOptimizerPlanner::TModificationGuard::AddPortion(const std::shared_ptr<TPortionInfo>& portion) {`
`13`		`- AFL_VERIFY(AddPortions.emplace(portion->GetPortionId(), portion).second);`
	`13`	`+ AddPortions.emplace_back(portion);`
`14`	`14`	`return*this;`
`15`	`15`	`}`
`16`	`16`
`17`	`17`	`IOptimizerPlanner::TModificationGuard& IOptimizerPlanner::TModificationGuard::RemovePortion(const std::shared_ptr<TPortionInfo>& portion) {`
`18`		`- AFL_VERIFY(RemovePortions.emplace(portion->GetPortionId(), portion).second);`
	`18`	`+ RemovePortions.emplace_back(portion);`
`19`	`19`	`return*this;`
`20`	`20`	`}`
`21`	`21`
Original file line number	Diff line number	Diff line change
`@@ -1231,9 +1231,9 @@ class TOptimizerPlanner: public IOptimizerPlanner {`
`1231`	`1231`	`return Buckets.IsLocked(dataLocksManager);`
`1232`	`1232`	`}`
`1233`	`1233`
`1234`		`- virtual void DoModifyPortions(const THashMap<ui64, TPortionInfo::TPtr>& add, const THashMap<ui64, TPortionInfo::TPtr>& remove) override {`
	`1234`	`+ virtual void DoModifyPortions(const std::vector<TPortionInfo::TPtr>& add, const std::vector<TPortionInfo::TPtr>& remove) override {`
`1235`	`1235`	`const TInstant now = TInstant::Now();`
`1236`		`- for (auto&& [_, i] : remove) {`
	`1236`	`+ for (auto&& i : remove) {`
`1237`	`1237`	`if (i->GetMeta().GetTierName() != IStoragesManager::DefaultStorageId && i->GetMeta().GetTierName() != "") {`
`1238`	`1238`	`continue;`
`1239`	`1239`	`}`
`@@ -1242,7 +1242,7 @@ class TOptimizerPlanner: public IOptimizerPlanner {`
`1242`	`1242`	`Counters->OptimizersCount->Sub(1);`
`1243`	`1243`	`}`
`1244`	`1244`	`}`
`1245`		`- for (auto&& [_, i] : add) {`
	`1245`	`+ for (auto&& i : add) {`
`1246`	`1246`	`if (i->GetMeta().GetTierName() != IStoragesManager::DefaultStorageId && i->GetMeta().GetTierName() != "") {`
`1247`	`1247`	`continue;`
`1248`	`1248`	`}`
Original file line number	Diff line number	Diff line change
`@@ -67,12 +67,12 @@ class TOptimizerPlanner: public IOptimizerPlanner {`
`67`	`67`	`return false;`
`68`	`68`	`}`
`69`	`69`
`70`		`- virtual void DoModifyPortions(const THashMap<ui64, TPortionInfo::TPtr>& add, const THashMap<ui64, TPortionInfo::TPtr>& remove) override {`
	`70`	`+ virtual void DoModifyPortions(const std::vector<TPortionInfo::TPtr>& add, const std::vector<TPortionInfo::TPtr>& remove) override {`
`71`	`71`	`std::vector<std::vector<TPortionInfo::TPtr>> removePortionsByLevel;`
`72`	`72`	`removePortionsByLevel.resize(Levels.size());`
`73`	`73`	`std::vector<std::vector<TPortionInfo::TPtr>> addPortionsByLevels;`
`74`	`74`	`addPortionsByLevels.resize(Levels.size());`
`75`		`- for (auto&& [_, i] : remove) {`
	`75`	`+ for (auto&& i : remove) {`
`76`	`76`	`if (i->GetProduced() == NPortion::EProduced::EVICTED) {`
`77`	`77`	`continue;`
`78`	`78`	`}`
`@@ -81,7 +81,7 @@ class TOptimizerPlanner: public IOptimizerPlanner {`
`81`	`81`	`removePortionsByLevel[i->GetCompactionLevel()].emplace_back(i);`
`82`	`82`	`}`
`83`	`83`	`std::vector<TPortionInfo::TPtr> problemPortions;`
`84`		`- for (auto&& [_, i] : add) {`
	`84`	`+ for (auto&& i : add) {`
`85`	`85`	`if (i->GetProduced() == NPortion::EProduced::EVICTED) {`
`86`	`86`	`continue;`
`87`	`87`	`}`