diff --git a/ydb/core/tx/schemeshard/olap/table/table.h b/ydb/core/tx/schemeshard/olap/table/table.h index 6948361cb692..e9fca9431c4e 100644 --- a/ydb/core/tx/schemeshard/olap/table/table.h +++ b/ydb/core/tx/schemeshard/olap/table/table.h @@ -109,9 +109,9 @@ struct TColumnTableInfo { Stats.UpdateShardStats(diskSpaceUsageDelta, shardIdx, newStats, now); } - void UpdateTableStats(TDiskSpaceUsageDelta* diskSpaceUsageDelta, const TShardIdx shardIdx, const TPathId& pathId, const TPartitionStats& newStats, TInstant now) { + void UpdateTableStats(const TShardIdx shardIdx, const TPathId& pathId, const TPartitionStats& newStats, TInstant now) { Stats.TableStats[pathId].Aggregated.PartCount = GetColumnShards().size(); - Stats.UpdateTableStats(diskSpaceUsageDelta, shardIdx, pathId, newStats, now); + Stats.UpdateTableStats(shardIdx, pathId, newStats, now); } TConclusion> BuildEntity(const TPathId& pathId, const NOlap::NAlter::TEntityInitializationContext& iContext) const; diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp index e5cc3637a214..f37d3eba60ac 100644 --- a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp @@ -459,7 +459,7 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId, LOG_TRACE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "add stats for exists table with pathId=" << tablePathId); - Self->ColumnTables.GetVerifiedPtr(tablePathId)->UpdateTableStats(&diskSpaceUsageDelta, shardIdx, tablePathId, newTableStats, now); + Self->ColumnTables.GetVerifiedPtr(tablePathId)->UpdateTableStats(shardIdx, tablePathId, newTableStats, now); } else { LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "failed add stats for table with pathId=" << tablePathId); diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index a6a18e40f5f0..26d2a6c9d753 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -279,27 +279,33 @@ void TSubDomainInfo::AggrDiskSpaceUsage(IQuotaCounters* counters, const TPartiti } void TSubDomainInfo::AggrDiskSpaceUsage(IQuotaCounters* counters, const TDiskSpaceUsageDelta& diskSpaceUsageDelta) { + // just (in)sanity check, diskSpaceUsageDelta should have at least 1 element + if (diskSpaceUsageDelta.empty()) { + return; + } // see filling of diskSpaceUsageDelta in UpdateShardStats() - for (const auto& [poolKind, delta] : diskSpaceUsageDelta) { - if (poolKind.empty()) { - // total space - DiskSpaceUsage.Tables.DataSize += delta.DataSize; - counters->ChangeDiskSpaceTablesDataBytes(delta.DataSize); - - DiskSpaceUsage.Tables.IndexSize += delta.IndexSize; - counters->ChangeDiskSpaceTablesIndexBytes(delta.IndexSize); - - i64 oldTotalBytes = DiskSpaceUsage.Tables.TotalSize; - DiskSpaceUsage.Tables.TotalSize = DiskSpaceUsage.Tables.DataSize + DiskSpaceUsage.Tables.IndexSize; - i64 newTotalBytes = DiskSpaceUsage.Tables.TotalSize; - counters->ChangeDiskSpaceTablesTotalBytes(newTotalBytes - oldTotalBytes); - } else { - // space separated by storage pool kinds - auto& r = DiskSpaceUsage.StoragePoolsUsage[poolKind]; - r.DataSize += delta.DataSize; - r.IndexSize += delta.IndexSize; - counters->AddDiskSpaceTables(GetUserFacingStorageType(poolKind), delta.DataSize, delta.IndexSize); - } + // total space usage, index 0 + { + const auto& [poolKind, delta] = diskSpaceUsageDelta[0]; + + DiskSpaceUsage.Tables.DataSize += delta.DataSize; + counters->ChangeDiskSpaceTablesDataBytes(delta.DataSize); + + DiskSpaceUsage.Tables.IndexSize += delta.IndexSize; + counters->ChangeDiskSpaceTablesIndexBytes(delta.IndexSize); + + i64 oldTotalBytes = DiskSpaceUsage.Tables.TotalSize; + DiskSpaceUsage.Tables.TotalSize = DiskSpaceUsage.Tables.DataSize + DiskSpaceUsage.Tables.IndexSize; + i64 newTotalBytes = DiskSpaceUsage.Tables.TotalSize; + counters->ChangeDiskSpaceTablesTotalBytes(newTotalBytes - oldTotalBytes); + } + // space usage by storage pool kinds, from index 1 onwards + for (size_t i = 1; i < diskSpaceUsageDelta.size(); ++i) { + const auto& [poolKind, delta] = diskSpaceUsageDelta[i]; + auto& r = DiskSpaceUsage.StoragePoolsUsage[poolKind]; + r.DataSize += delta.DataSize; + r.IndexSize += delta.IndexSize; + counters->AddDiskSpaceTables(GetUserFacingStorageType(poolKind), delta.DataSize, delta.IndexSize); } } @@ -1853,10 +1859,9 @@ void TTableAggregatedStats::UpdateShardStats(TDiskSpaceUsageDelta* diskSpaceUsag diskSpaceUsageDelta->emplace_back(poolKind, delta); } } - for (const auto& [poolKind, delta] : *diskSpaceUsageDelta) { - if (poolKind.empty()) { - continue; - } + // from index 1 onwards (as index 0 holds total space delta) + for (size_t i = 1; i < diskSpaceUsageDelta->size(); ++i) { + const auto& [poolKind, delta] = (*diskSpaceUsageDelta)[i]; auto& r = Aggregated.StoragePoolsStats[poolKind]; r.DataSize += delta.DataSize; r.IndexSize += delta.IndexSize; @@ -1961,10 +1966,11 @@ void TTableAggregatedStats::UpdateShardStatsForFollower( oldStats.TopCpuUsage.Update(newStats.TopCpuUsage); // The left is new stats now! } -void TAggregatedStats::UpdateTableStats(TDiskSpaceUsageDelta* diskSpaceUsageDelta, TShardIdx shardIdx, const TPathId& pathId, const TPartitionStats& newStats, TInstant now) { +void TAggregatedStats::UpdateTableStats(TShardIdx shardIdx, const TPathId& pathId, const TPartitionStats& newStats, TInstant now) { auto& tableStats = TableStats[pathId]; tableStats.PartitionStats[shardIdx]; // insert if none - tableStats.UpdateShardStats(diskSpaceUsageDelta, shardIdx, newStats, now); + TDiskSpaceUsageDelta unused; + tableStats.UpdateShardStats(&unused, shardIdx, newStats, now); } void TTableInfo::RegisterSplitMergeOp(TOperationId opId, const TTxState& txState) { diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index e5faf7214347..cbcba92f7103 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -483,7 +483,7 @@ struct TTableAggregatedStats { struct TAggregatedStats : public TTableAggregatedStats { THashMap TableStats; - void UpdateTableStats(TDiskSpaceUsageDelta* diskSpaceUsageDelta, TShardIdx datashardIdx, const TPathId& pathId, const TPartitionStats& newStats, TInstant now); + void UpdateTableStats(TShardIdx datashardIdx, const TPathId& pathId, const TPartitionStats& newStats, TInstant now); }; struct TSubDomainInfo; diff --git a/ydb/core/tx/schemeshard/ut_subdomain/ut_subdomain.cpp b/ydb/core/tx/schemeshard/ut_subdomain/ut_subdomain.cpp index d69eaec1542c..a283586c7eef 100644 --- a/ydb/core/tx/schemeshard/ut_subdomain/ut_subdomain.cpp +++ b/ydb/core/tx/schemeshard/ut_subdomain/ut_subdomain.cpp @@ -3,6 +3,10 @@ #include #include #include +#include // for TSchemeShard +#include // for MakeTestBlob +#include // for NTypeIds and TTypeInfo + using namespace NKikimr; using namespace NSchemeShard; @@ -3129,17 +3133,42 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { )", {NKikimrScheme::StatusAccepted}); } - Y_UNIT_TEST(DiskSpaceUsage) { + Y_UNIT_TEST_FLAGS(DiskSpaceUsage, DisableStatsBatching, EnablePersistentPartitionStats) { TTestBasicRuntime runtime; + TTestEnvOptions opts; - opts.DisableStatsBatching(true); - opts.EnablePersistentPartitionStats(true); - TTestEnv env(runtime, opts); + opts.DisableStatsBatching(DisableStatsBatching); + opts.EnablePersistentPartitionStats(EnablePersistentPartitionStats); + opts.EnableBackgroundCompaction(false); // make sure background compaction will not interfere + opts.DataShardStatsReportIntervalSeconds(0); // make sure stats will be reported swiftly + + TSchemeShard* schemeshard = nullptr; + TTestEnv env(runtime, opts, + /*TSchemeShardFactory ssFactory*/ + [&schemeshard](const TActorId& tablet, TTabletStorageInfo* info) { + schemeshard = new TSchemeShard(tablet, info); + Cerr << "TEST create schemeshard, " << (void*)schemeshard << Endl; + return schemeshard; + } + ); + + NDataShard::gDbStatsDataSizeResolution = 1; + NDataShard::gDbStatsRowCountResolution = 1; + + if (DisableStatsBatching == false) { + runtime.GetAppData().SchemeShardConfig.SetStatsMaxBatchSize(2); + } + const auto sender = runtime.AllocateEdgeActor(); - auto waitForTableStats = [&](ui32 shards) { + auto waitForFullStatsUpdate = [&](const ui32 count) { + ui64 statsCountBaseline = schemeshard->TabletCounters->Cumulative()[COUNTER_STATS_WRITTEN].Get(); TDispatchOptions options; - options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvDataShard::EvPeriodicTableStats, shards)); + options.CustomFinalCondition = [&]() { + auto statsCount = schemeshard->TabletCounters->Cumulative()[COUNTER_STATS_WRITTEN].Get() - statsCountBaseline; + Cerr << "TEST waitForFullStatsUpdate, schemeshard " << (void*)schemeshard << ", stats written " << statsCount << Endl; + return statsCount >= count; + }; runtime.DispatchEvents(options); }; @@ -3158,6 +3187,14 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { return result; }; + auto compareDiskSpaceUsage = [&](TString* diff, const NKikimrSubDomains::TDiskSpaceUsage& a, const NKikimrSubDomains::TDiskSpaceUsage& b) -> bool { + using google::protobuf::util::MessageDifferencer; + MessageDifferencer d; + d.ReportDifferencesToString(diff); + d.set_repeated_field_comparison(MessageDifferencer::RepeatedFieldComparison::AS_SET); + return d.Compare(a, b); + }; + ui64 tabletId = TTestTxConfig::FakeHiveTablets; ui64 txId = 100; @@ -3172,13 +3209,15 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { env.TestWaitNotification(runtime, txId); UpdateRow(runtime, "Table1", 1, "value1", tabletId); - waitForTableStats(1); + waitForFullStatsUpdate(1); auto du = getDiskSpaceUsage(); UNIT_ASSERT_C(du.GetTables().GetTotalSize() > 0, du.ShortDebugString()); RebootTablet(runtime, TTestTxConfig::SchemeShard, sender); - UNIT_ASSERT_VALUES_EQUAL(du.ShortDebugString(), getDiskSpaceUsage().ShortDebugString()); + waitForFullStatsUpdate(1); + TString diff; + UNIT_ASSERT_C(compareDiskSpaceUsage(&diff, du, getDiskSpaceUsage()), diff); } // multi-shard table @@ -3196,16 +3235,486 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { UpdateRow(runtime, "Table2", 1, "value1", tabletId + 0); UpdateRow(runtime, "Table2", 2, "value2", tabletId + 1); - waitForTableStats(1 /* Table1 */ + 2 /* Table2 */); + const ui32 shardCount = 1 /* Table1 */ + 2 /* Table2 */; + waitForFullStatsUpdate(shardCount); auto du = getDiskSpaceUsage(); UNIT_ASSERT_C(du.GetTables().GetTotalSize() > 0, du.ShortDebugString()); RebootTablet(runtime, TTestTxConfig::SchemeShard, sender); - UNIT_ASSERT_VALUES_EQUAL(du.ShortDebugString(), getDiskSpaceUsage().ShortDebugString()); + waitForFullStatsUpdate(shardCount); + TString diff; + UNIT_ASSERT_C(compareDiskSpaceUsage(&diff, du, getDiskSpaceUsage()), diff); + } + } + + Y_UNIT_TEST_FLAG(DiskSpaceUsageWithPersistedLeftovers, DisableStatsBatching) { + TTestBasicRuntime runtime; + + TTestEnvOptions opts; + opts.DisableStatsBatching(DisableStatsBatching); + opts.EnableBackgroundCompaction(false); // make sure background compaction will not interfere + opts.DataShardStatsReportIntervalSeconds(0); // make sure stats will be reported swiftly + + TSchemeShard* schemeshard = nullptr; + TTestEnv env(runtime, opts, + /*TSchemeShardFactory ssFactory*/ + [&schemeshard](const TActorId& tablet, TTabletStorageInfo* info) { + schemeshard = new TSchemeShard(tablet, info); + Cerr << "TEST create schemeshard, " << (void*)schemeshard << Endl; + return schemeshard; + } + ); + + NDataShard::gDbStatsDataSizeResolution = 1; + NDataShard::gDbStatsRowCountResolution = 1; + + if (DisableStatsBatching == false) { + runtime.GetAppData().SchemeShardConfig.SetStatsMaxBatchSize(2); + } + + const auto sender = runtime.AllocateEdgeActor(); + + auto waitForFullStatsUpdate = [&](const ui32 count) { + ui64 statsCountBaseline = schemeshard->TabletCounters->Cumulative()[COUNTER_STATS_WRITTEN].Get(); + TDispatchOptions options; + options.CustomFinalCondition = [&]() { + auto statsCount = schemeshard->TabletCounters->Cumulative()[COUNTER_STATS_WRITTEN].Get() - statsCountBaseline; + Cerr << "TEST waitForFullStatsUpdate, schemeshard " << (void*)schemeshard << ", stats written " << statsCount << Endl; + return statsCount >= count; + }; + runtime.DispatchEvents(options); + }; + + auto getDiskSpaceUsage = [&]() { + NKikimrSubDomains::TDiskSpaceUsage result; + + TestDescribeResult( + DescribePath(runtime, "/MyRoot"), { + NLs::PathExist, + NLs::Finished, [&result] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + result = record.GetPathDescription().GetDomainDescription().GetDiskSpaceUsage(); + } + } + ); + + return result; + }; + + auto compareDiskSpaceUsage = [&](TString* diff, const NKikimrSubDomains::TDiskSpaceUsage& a, const NKikimrSubDomains::TDiskSpaceUsage& b) -> bool { + using google::protobuf::util::MessageDifferencer; + MessageDifferencer d; + d.ReportDifferencesToString(diff); + d.set_repeated_field_comparison(MessageDifferencer::RepeatedFieldComparison::AS_SET); + return d.Compare(a, b); + }; + + ui64 tabletId = TTestTxConfig::FakeHiveTablets; + ui64 txId = 100; + + // multi-shard table + { + runtime.GetAppData().FeatureFlags.SetEnablePersistentPartitionStats(true); + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table2" + Columns { Name: "key" Type: "Uint32"} + Columns { Name: "value" Type: "Utf8"} + KeyColumnNames: ["key"] + UniformPartitionsCount: 2 + )"); + env.TestWaitNotification(runtime, txId); + + UpdateRow(runtime, "Table2", 1, "value1", tabletId + 0); + UpdateRow(runtime, "Table2", 2, "value2", tabletId + 1); + + const ui32 shardCount = 2; + waitForFullStatsUpdate(shardCount); + + auto du = getDiskSpaceUsage(); + UNIT_ASSERT_C(du.GetTables().GetTotalSize() > 0, du.ShortDebugString()); + + runtime.GetAppData().FeatureFlags.SetEnablePersistentPartitionStats(false); + + RebootTablet(runtime, TTestTxConfig::SchemeShard, sender); + + waitForFullStatsUpdate(shardCount); + TString diff; + UNIT_ASSERT_C(compareDiskSpaceUsage(&diff, du, getDiskSpaceUsage()), diff); + } + } + + Y_UNIT_TEST_FLAGS(DiskSpaceUsageWithTable, DisableStatsBatching, EnablePersistentPartitionStats) { + TTestBasicRuntime runtime; + + TTestEnvOptions opts; + opts.DisableStatsBatching(DisableStatsBatching); + opts.EnablePersistentPartitionStats(EnablePersistentPartitionStats); + opts.EnableBackgroundCompaction(false); // make sure background compaction will not interfere + opts.DataShardStatsReportIntervalSeconds(0); // make sure stats will be reported swiftly + + TSchemeShard* schemeshard = nullptr; + TTestEnv env(runtime, opts, + /*TSchemeShardFactory ssFactory*/ + [&schemeshard](const TActorId& tablet, TTabletStorageInfo* info) { + schemeshard = new TSchemeShard(tablet, info); + Cerr << "TEST create schemeshard, " << (void*)schemeshard << Endl; + return schemeshard; + } + ); + + NDataShard::gDbStatsDataSizeResolution = 1; + NDataShard::gDbStatsRowCountResolution = 1; + + if (DisableStatsBatching == false) { + runtime.GetAppData().SchemeShardConfig.SetStatsMaxBatchSize(2); + } + + const auto sender = runtime.AllocateEdgeActor(); + + auto waitForFullStatsUpdate = [&](const ui32 count) { + ui64 statsCountBaseline = schemeshard->TabletCounters->Cumulative()[COUNTER_STATS_WRITTEN].Get(); + TDispatchOptions options; + options.CustomFinalCondition = [&]() { + auto statsCount = schemeshard->TabletCounters->Cumulative()[COUNTER_STATS_WRITTEN].Get() - statsCountBaseline; + Cerr << "TEST waitForFullStatsUpdate, schemeshard " << (void*)schemeshard << ", stats written " << statsCount << Endl; + return statsCount >= count; + }; + runtime.DispatchEvents(options); + }; + + auto compareProto = [&](TString* diff, const auto& a, const auto& b) -> bool { + using google::protobuf::util::MessageDifferencer; + MessageDifferencer d; + d.ReportDifferencesToString(diff); + d.set_repeated_field_comparison(MessageDifferencer::RepeatedFieldComparison::AS_SET); + return d.Compare(a, b); + }; + + ui64 txId = 100; + + // test body + + // 1. create object and fill it with data + const ui32 shardCount = 2; + { + TestCreateTable(runtime, ++txId, "/MyRoot", Sprintf(R"( + Name: "Table" + Columns { Name: "key" Type: "Uint32"} + Columns { Name: "value" Type: "Utf8"} + KeyColumnNames: ["key"] + UniformPartitionsCount: %d + )", shardCount + )); + env.TestWaitNotification(runtime, txId); + + const ui64 tabletId = TTestTxConfig::FakeHiveTablets; + UpdateRow(runtime, "Table", 1, "value1", tabletId + 0); + UpdateRow(runtime, "Table", 2, "value2", tabletId + 1); } + + // 2. wait for all shard stats to be processed + waitForFullStatsUpdate(shardCount); + + // 3. check that disk space usage at subdomain level and table level is the same + auto getUsage = [](const auto& describe) { + return std::make_pair( + describe.GetPathDescription().GetDomainDescription().GetDiskSpaceUsage(), + describe.GetPathDescription().GetTableStats() + ); + }; + auto describeBefore = DescribePath(runtime, "/MyRoot/Table"); + const auto& [subdomainDiskUsageBefore, storeUsageBefore] = getUsage(describeBefore); + UNIT_ASSERT_GT_C(subdomainDiskUsageBefore.GetTables().GetDataSize(), 0, subdomainDiskUsageBefore.DebugString()); + UNIT_ASSERT_GT_C(storeUsageBefore.GetDataSize(), 0, storeUsageBefore.DebugString()); + UNIT_ASSERT_VALUES_EQUAL(subdomainDiskUsageBefore.GetTables().GetDataSize(), storeUsageBefore.GetDataSize()); + + // 4. reboot schemeshard + RebootTablet(runtime, TTestTxConfig::SchemeShard, sender); + + // 5. wait for all shard stats to be processed + waitForFullStatsUpdate(shardCount); + + // 6. check that disk space usage levels is the same as before reboot + auto describeAfter = DescribePath(runtime, "/MyRoot/Table"); + const auto& [subdomainDiskUsageAfter, storeUsageAfter] = getUsage(describeAfter); + TString diff; + UNIT_ASSERT_C(compareProto(&diff, subdomainDiskUsageBefore, subdomainDiskUsageAfter), diff); + UNIT_ASSERT_C(compareProto(&diff, storeUsageBefore, storeUsageAfter), diff); + UNIT_ASSERT_VALUES_EQUAL(subdomainDiskUsageAfter.GetTables().GetDataSize(), storeUsageAfter.GetDataSize()); } + Y_UNIT_TEST_FLAGS(DiskSpaceUsageWithColumnTableInStore, DisableStatsBatching, EnablePersistentPartitionStats) { + TTestBasicRuntime runtime; + + TTestEnvOptions opts; + opts.DisableStatsBatching(DisableStatsBatching); + opts.EnablePersistentPartitionStats(EnablePersistentPartitionStats); + opts.EnableBackgroundCompaction(false); // make sure background compaction will not interfere + opts.DataShardStatsReportIntervalSeconds(0); // make sure stats will be reported swiftly + + TSchemeShard* schemeshard = nullptr; + TTestEnv env(runtime, opts, + /*TSchemeShardFactory ssFactory*/ + [&schemeshard](const TActorId& tablet, TTabletStorageInfo* info) { + schemeshard = new TSchemeShard(tablet, info); + Cerr << "TEST create schemeshard, " << (void*)schemeshard << Endl; + return schemeshard; + } + ); + + NDataShard::gDbStatsDataSizeResolution = 1; + NDataShard::gDbStatsRowCountResolution = 1; + + if (DisableStatsBatching == false) { + runtime.GetAppData().SchemeShardConfig.SetStatsMaxBatchSize(2); + } + + const auto sender = runtime.AllocateEdgeActor(); + + auto waitForFullStatsUpdate = [&](const ui32 count) { + ui64 statsCountBaseline = schemeshard->TabletCounters->Cumulative()[COUNTER_STATS_WRITTEN].Get(); + TDispatchOptions options; + options.CustomFinalCondition = [&]() { + auto statsCount = schemeshard->TabletCounters->Cumulative()[COUNTER_STATS_WRITTEN].Get() - statsCountBaseline; + Cerr << "TEST waitForFullStatsUpdate, schemeshard " << (void*)schemeshard << ", stats written " << statsCount << Endl; + return statsCount >= count; + }; + runtime.DispatchEvents(options); + }; + + auto compareProto = [&](TString* diff, const auto& a, const auto& b) -> bool { + using google::protobuf::util::MessageDifferencer; + MessageDifferencer d; + d.ReportDifferencesToString(diff); + d.set_repeated_field_comparison(MessageDifferencer::RepeatedFieldComparison::AS_SET); + return d.Compare(a, b); + }; + + ui64 txId = 100; + + // test body + + // 1. create object and fill it with data + const ui32 shardCount = 1; + { + TestCreateOlapStore(runtime, ++txId, "/MyRoot", Sprintf(R"( + Name: "Store" + ColumnShardCount: 1 + SchemaPresets { + Name: "default" + Schema { + Columns { Name: "timestamp" Type: "Timestamp" NotNull: true } + Columns { Name: "data" Type: "Utf8" } + KeyColumnNames: "timestamp" + } + } + )", shardCount + )); + env.TestWaitNotification(runtime, txId); + + TestCreateColumnTable(runtime, ++txId, "/MyRoot/Store", Sprintf(R"( + Name: "ColumnTable" + ColumnShardCount: %d + Schema { + Columns { Name: "timestamp" Type: "Timestamp" NotNull: true } + Columns { Name: "data" Type: "Utf8" } + KeyColumnNames: "timestamp" + } + )", shardCount + )); + env.TestWaitNotification(runtime, txId); + + ui64 pathId = 0; + ui64 shardId = 0; + { + auto describe = DescribePath(runtime, "/MyRoot/Store/ColumnTable"); + TestDescribeResult(describe, {NLs::PathExist}); + pathId = describe.GetPathId(); + const auto& sharding = describe.GetPathDescription().GetColumnTableDescription().GetSharding(); + shardId = sharding.GetColumnShards()[0]; + } + UNIT_ASSERT(shardId); + + { // Write data directly into shard + TActorId sender = runtime.AllocateEdgeActor(); + const ui32 rowsInBatch = 100000; + + const TVector ydbSchema = { + NArrow::NTest::TTestColumn("timestamp", NScheme::TTypeInfo(NScheme::NTypeIds::Timestamp)).SetNullable(false), + NArrow::NTest::TTestColumn("data", NScheme::TTypeInfo(NScheme::NTypeIds::Utf8) ) + }; + const auto& data = NTxUT::MakeTestBlob({ 0, rowsInBatch }, ydbSchema, {}, { "timestamp" }); + ui64 writeId = 0; + std::vector writeIds; + ++txId; + NTxUT::WriteData(runtime, sender, shardId, ++writeId, pathId, data, ydbSchema, &writeIds, NEvWrite::EModificationType::Upsert, txId); + NTxUT::TPlanStep planStep = NTxUT::ProposeCommit(runtime, sender, shardId, txId, writeIds, txId); + NTxUT::PlanCommit(runtime, sender, shardId, planStep, { txId }); + + } + } + + // 2. wait for all shard stats to be processed + waitForFullStatsUpdate(shardCount); + + // 3. check that disk space usage at subdomain level and column store level is the same + auto getUsage = [](const auto& describe) { + return std::make_pair( + describe.GetPathDescription().GetDomainDescription().GetDiskSpaceUsage(), + describe.GetPathDescription().GetTableStats() + ); + }; + auto describeBefore = DescribePath(runtime, "/MyRoot/Store"); + const auto& [subdomainDiskUsageBefore, storeUsageBefore] = getUsage(describeBefore); + UNIT_ASSERT_GT_C(subdomainDiskUsageBefore.GetTables().GetDataSize(), 0, subdomainDiskUsageBefore.DebugString()); + UNIT_ASSERT_GT_C(storeUsageBefore.GetDataSize(), 0, storeUsageBefore.DebugString()); + UNIT_ASSERT_VALUES_EQUAL(subdomainDiskUsageBefore.GetTables().GetDataSize(), storeUsageBefore.GetDataSize()); + + // 4. reboot schemeshard + RebootTablet(runtime, TTestTxConfig::SchemeShard, sender); + + // 5. wait for all shard stats to be processed + waitForFullStatsUpdate(shardCount); + + // 6. check that disk space usage levels is the same as before reboot + auto describeAfter = DescribePath(runtime, "/MyRoot/Store"); + const auto& [subdomainDiskUsageAfter, storeUsageAfter] = getUsage(describeAfter); + TString diff; + UNIT_ASSERT_C(compareProto(&diff, subdomainDiskUsageBefore, subdomainDiskUsageAfter), diff); + UNIT_ASSERT_C(compareProto(&diff, storeUsageBefore, storeUsageAfter), diff); + UNIT_ASSERT_VALUES_EQUAL(subdomainDiskUsageAfter.GetTables().GetDataSize(), storeUsageAfter.GetDataSize()); + } + + Y_UNIT_TEST_FLAGS(DiskSpaceUsageWithStandaloneColumnTable, DisableStatsBatching, EnablePersistentPartitionStats) { + TTestBasicRuntime runtime; + + TTestEnvOptions opts; + opts.DisableStatsBatching(DisableStatsBatching); + opts.EnablePersistentPartitionStats(EnablePersistentPartitionStats); + opts.EnableBackgroundCompaction(false); // make sure background compaction will not interfere + opts.DataShardStatsReportIntervalSeconds(0); // make sure stats will be reported swiftly + + TSchemeShard* schemeshard = nullptr; + TTestEnv env(runtime, opts, + /*TSchemeShardFactory ssFactory*/ + [&schemeshard](const TActorId& tablet, TTabletStorageInfo* info) { + schemeshard = new TSchemeShard(tablet, info); + Cerr << "TEST create schemeshard, " << (void*)schemeshard << Endl; + return schemeshard; + } + ); + + NDataShard::gDbStatsDataSizeResolution = 1; + NDataShard::gDbStatsRowCountResolution = 1; + + if (DisableStatsBatching == false) { + runtime.GetAppData().SchemeShardConfig.SetStatsMaxBatchSize(2); + } + + const auto sender = runtime.AllocateEdgeActor(); + + auto waitForFullStatsUpdate = [&](const ui32 count) { + ui64 statsCountBaseline = schemeshard->TabletCounters->Cumulative()[COUNTER_STATS_WRITTEN].Get(); + TDispatchOptions options; + options.CustomFinalCondition = [&]() { + auto statsCount = schemeshard->TabletCounters->Cumulative()[COUNTER_STATS_WRITTEN].Get() - statsCountBaseline; + Cerr << "TEST waitForFullStatsUpdate, schemeshard " << (void*)schemeshard << ", stats written " << statsCount << Endl; + return statsCount >= count; + }; + runtime.DispatchEvents(options); + }; + + auto compareProto = [&](TString* diff, const auto& a, const auto& b) -> bool { + using google::protobuf::util::MessageDifferencer; + MessageDifferencer d; + d.ReportDifferencesToString(diff); + d.set_repeated_field_comparison(MessageDifferencer::RepeatedFieldComparison::AS_SET); + return d.Compare(a, b); + }; + + ui64 txId = 100; + + // test body + + // 1. create object and fill it with data + const ui32 shardCount = 1; + { + TestCreateColumnTable(runtime, ++txId, "/MyRoot", Sprintf(R"( + Name: "ColumnTable" + ColumnShardCount: %d + Schema { + Columns { Name: "timestamp" Type: "Timestamp" NotNull: true } + Columns { Name: "data" Type: "Utf8" } + KeyColumnNames: "timestamp" + } + )", shardCount + )); + env.TestWaitNotification(runtime, txId); + + ui64 pathId = 0; + ui64 shardId = 0; + { + auto describe = DescribePath(runtime, "/MyRoot/ColumnTable"); + TestDescribeResult(describe, {NLs::PathExist}); + pathId = describe.GetPathId(); + const auto& sharding = describe.GetPathDescription().GetColumnTableDescription().GetSharding(); + shardId = sharding.GetColumnShards()[0]; + } + UNIT_ASSERT(shardId); + + { // Write data directly into shard + TActorId sender = runtime.AllocateEdgeActor(); + const ui32 rowsInBatch = 100000; + + const TVector ydbSchema = { + NArrow::NTest::TTestColumn("timestamp", NScheme::TTypeInfo(NScheme::NTypeIds::Timestamp)).SetNullable(false), + NArrow::NTest::TTestColumn("data", NScheme::TTypeInfo(NScheme::NTypeIds::Utf8) ) + }; + const auto& data = NTxUT::MakeTestBlob({ 0, rowsInBatch }, ydbSchema, {}, { "timestamp" }); + ui64 writeId = 0; + std::vector writeIds; + ++txId; + NTxUT::WriteData(runtime, sender, shardId, ++writeId, pathId, data, ydbSchema, &writeIds, NEvWrite::EModificationType::Upsert, txId); + NTxUT::TPlanStep planStep = NTxUT::ProposeCommit(runtime, sender, shardId, txId, writeIds, txId); + NTxUT::PlanCommit(runtime, sender, shardId, planStep, { txId }); + + } + } + + // 2. wait for all shard stats to be processed + waitForFullStatsUpdate(shardCount); + + // 3. check that disk space usage at subdomain level and column store level is the same + auto getUsage = [](const auto& describe) { + return std::make_pair( + describe.GetPathDescription().GetDomainDescription().GetDiskSpaceUsage(), + describe.GetPathDescription().GetTableStats() + ); + }; + auto describeBefore = DescribePath(runtime, "/MyRoot/ColumnTable"); + const auto& [subdomainDiskUsageBefore, storeUsageBefore] = getUsage(describeBefore); + UNIT_ASSERT_GT_C(subdomainDiskUsageBefore.GetTables().GetDataSize(), 0, subdomainDiskUsageBefore.DebugString()); + UNIT_ASSERT_GT_C(storeUsageBefore.GetDataSize(), 0, storeUsageBefore.DebugString()); + UNIT_ASSERT_VALUES_EQUAL(subdomainDiskUsageBefore.GetTables().GetDataSize(), storeUsageBefore.GetDataSize()); + + // 4. reboot schemeshard + RebootTablet(runtime, TTestTxConfig::SchemeShard, sender); + + // 5. wait for all shard stats to be processed + waitForFullStatsUpdate(shardCount); + + // 6. check that disk space usage levels is the same as before reboot + auto describeAfter = DescribePath(runtime, "/MyRoot/ColumnTable"); + const auto& [subdomainDiskUsageAfter, storeUsageAfter] = getUsage(describeAfter); + TString diff; + UNIT_ASSERT_C(compareProto(&diff, subdomainDiskUsageBefore, subdomainDiskUsageAfter), diff); + UNIT_ASSERT_C(compareProto(&diff, storeUsageBefore, storeUsageAfter), diff); + UNIT_ASSERT_VALUES_EQUAL(subdomainDiskUsageAfter.GetTables().GetDataSize(), storeUsageAfter.GetDataSize()); + } + + //TODO: add DiskSpaceUsage test for topics + Y_UNIT_TEST(TableDiskSpaceQuotas) { TTestBasicRuntime runtime; TTestEnvOptions opts; diff --git a/ydb/core/tx/schemeshard/ut_subdomain/ya.make b/ydb/core/tx/schemeshard/ut_subdomain/ya.make index 7e491032fdb7..a258826d8a03 100644 --- a/ydb/core/tx/schemeshard/ut_subdomain/ya.make +++ b/ydb/core/tx/schemeshard/ut_subdomain/ya.make @@ -18,6 +18,8 @@ PEERDIR( ydb/core/testlib/default ydb/core/tx ydb/core/tx/schemeshard/ut_helpers + ydb/core/tx/columnshard/hooks/testing + ydb/core/tx/columnshard/test_helper yql/essentials/public/udf/service/exception_policy )