Skip to content

Commit bbf072b

Browse files
victoria-mcgrathvinser52
victoria-mcgrath
authored and
vinser52
committed
Fixed total cache size in CacheMemoryStats (#38)
Return a sum of sizes of each tier instead of just 1st tier's size.
1 parent c57827e commit bbf072b

File tree

2 files changed

+135
-64
lines changed

2 files changed

+135
-64
lines changed

‎cachelib/allocator/CacheAllocator-inl.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3762,7 +3762,10 @@ GlobalCacheStats CacheAllocator<CacheTrait>::getGlobalCacheStats() const {
37623762

37633763
template <typename CacheTrait>
37643764
CacheMemoryStats CacheAllocator<CacheTrait>::getCacheMemoryStats() const {
3765-
const auto totalCacheSize = allocator_[currentTier()]->getMemorySize();
3765+
size_t totalCacheSize = 0;
3766+
for(auto& allocator: allocator_) {
3767+
totalCacheSize += allocator->getMemorySize();
3768+
}
37663769

37673770
auto addSize = [this](size_t a, PoolId pid) {
37683771
return a + allocator_[currentTier()]->getPool(pid).getPoolSize();

‎cachelib/allocator/tests/MemoryTiersTest.cpp‎

Lines changed: 131 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,14 @@
1515
*/
1616

1717
#include <numeric>
18+
1819
#include "cachelib/allocator/CacheAllocator.h"
1920
#include "cachelib/allocator/tests/TestBase.h"
2021

2122
namespace facebook {
2223
namespace cachelib {
2324
namespace tests {
2425

25-
2626
using LruAllocatorConfig = CacheAllocatorConfig<LruAllocator>;
2727
using LruMemoryTierConfigs = LruAllocatorConfig::MemoryTierConfigs;
2828
using Strings = std::vector<std::string>;
@@ -33,66 +33,95 @@ const std::string defaultCacheDir{"/var/metadataDir"};
3333
const std::string defaultPmemPath{"/dev/shm/p1"};
3434
const std::string defaultDaxPath{"/dev/dax0.0"};
3535

36+
const size_t metaDataSize = 4194304;
37+
constexpr size_t MB = 1024ULL * 1024ULL;
38+
constexpr size_t GB = MB * 1024ULL;
39+
3640
template <typename Allocator>
3741
class MemoryTiersTest: public AllocatorTest<Allocator> {
38-
public:
39-
void basicCheck(
40-
LruAllocatorConfig& actualConfig,
41-
const Strings& expectedPaths = {defaultPmemPath},
42-
size_t expectedTotalCacheSize = defaultTotalCacheSize,
43-
const std::string& expectedCacheDir = defaultCacheDir) {
44-
EXPECT_EQ(actualConfig.getCacheSize(), expectedTotalCacheSize);
45-
EXPECT_EQ(actualConfig.getMemoryTierConfigs().size(), expectedPaths.size());
46-
EXPECT_EQ(actualConfig.getCacheDir(), expectedCacheDir);
47-
auto configs = actualConfig.getMemoryTierConfigs();
48-
49-
size_t sum_ratios = std::accumulate(configs.begin(), configs.end(), 0,
50-
[](const size_t i, const MemoryTierCacheConfig& config) { return i + config.getRatio();});
51-
size_t sum_sizes = std::accumulate(configs.begin(), configs.end(), 0,
52-
[&](const size_t i, const MemoryTierCacheConfig& config) { return i + config.calculateTierSize(actualConfig.getCacheSize(), sum_ratios);});
53-
54-
55-
size_t partition_size = 0;
56-
if (sum_ratios) {
57-
partition_size = actualConfig.getCacheSize() / sum_ratios;
58-
/* Sum of sizes can be lower due to rounding down to partition_size. */
59-
EXPECT_GE(sum_sizes, expectedTotalCacheSize - partition_size);
60-
}
42+
public:
43+
void basicCheck(LruAllocatorConfig& actualConfig,
44+
const Strings& expectedPaths = {defaultPmemPath},
45+
size_t expectedTotalCacheSize = defaultTotalCacheSize,
46+
const std::string& expectedCacheDir = defaultCacheDir) {
47+
EXPECT_EQ(actualConfig.getCacheSize(), expectedTotalCacheSize);
48+
EXPECT_EQ(actualConfig.getMemoryTierConfigs().size(), expectedPaths.size());
49+
EXPECT_EQ(actualConfig.getCacheDir(), expectedCacheDir);
50+
auto configs = actualConfig.getMemoryTierConfigs();
51+
52+
size_t sum_ratios = std::accumulate(configs.begin(), configs.end(), 0,
53+
[](const size_t i, const MemoryTierCacheConfig& config) { return i + config.getRatio();});
54+
size_t sum_sizes = std::accumulate(configs.begin(), configs.end(), 0,
55+
[&](const size_t i, const MemoryTierCacheConfig& config) {
56+
return i + config.calculateTierSize(actualConfig.getCacheSize(), sum_ratios);
57+
});
58+
59+
60+
size_t partition_size = 0;
61+
if (sum_ratios) {
62+
partition_size = actualConfig.getCacheSize() / sum_ratios;
63+
/* Sum of sizes can be lower due to rounding down to partition_size. */
64+
EXPECT_GE(sum_sizes, expectedTotalCacheSize - partition_size);
65+
}
6166

62-
for(auto i = 0; i < configs.size(); ++i) {
63-
auto tierSize = configs[i].calculateTierSize(actualConfig.getCacheSize(), sum_ratios);
64-
auto &opt = std::get<FileShmSegmentOpts>(configs[i].getShmTypeOpts());
65-
EXPECT_EQ(opt.path, expectedPaths[i]);
66-
EXPECT_GT(tierSize, 0);
67-
if (configs[i].getRatio() && (i < configs.size() - 1)) {
68-
EXPECT_EQ(tierSize, partition_size * configs[i].getRatio());
69-
}
67+
for(auto i = 0; i < configs.size(); ++i) {
68+
auto tierSize = configs[i].calculateTierSize(actualConfig.getCacheSize(), sum_ratios);
69+
auto &opt = std::get<FileShmSegmentOpts>(configs[i].getShmTypeOpts());
70+
EXPECT_EQ(opt.path, expectedPaths[i]);
71+
EXPECT_GT(tierSize, 0);
72+
if (configs[i].getRatio() && (i < configs.size() - 1)) {
73+
EXPECT_EQ(tierSize, partition_size * configs[i].getRatio());
7074
}
7175
}
76+
}
77+
78+
LruAllocatorConfig createTestCacheConfig(
79+
const Strings& tierPaths = {defaultPmemPath},
80+
const Ratios& tierRatios = {1},
81+
bool setPosixForShm = true,
82+
size_t cacheSize = defaultTotalCacheSize,
83+
const std::string& cacheDir = defaultCacheDir) {
84+
EXPECT_EQ(tierPaths.size(), tierRatios.size());
85+
LruAllocatorConfig cfg;
86+
cfg.setCacheSize(cacheSize)
87+
.enableCachePersistence(cacheDir);
88+
89+
if (setPosixForShm)
90+
cfg.usePosixForShm();
91+
92+
LruMemoryTierConfigs tierConfigs;
93+
tierConfigs.reserve(tierPaths.size());
94+
for(auto i = 0; i < tierPaths.size(); ++i) {
95+
tierConfigs.push_back(MemoryTierCacheConfig::fromFile(tierPaths[i])
96+
.setRatio(tierRatios[i]));
97+
}
7298

73-
LruAllocatorConfig createTestCacheConfig(
74-
const Strings& tierPaths = {defaultPmemPath},
75-
const Ratios& tierRatios = {1},
76-
bool setPosixForShm = true,
77-
size_t cacheSize = defaultTotalCacheSize,
78-
const std::string& cacheDir = defaultCacheDir) {
79-
EXPECT_EQ(tierPaths.size(), tierRatios.size());
80-
LruAllocatorConfig cfg;
81-
cfg.setCacheSize(cacheSize)
82-
.enableCachePersistence(cacheDir);
83-
84-
if (setPosixForShm)
85-
cfg.usePosixForShm();
86-
87-
LruMemoryTierConfigs tierConfigs;
88-
tierConfigs.reserve(tierPaths.size());
89-
for(auto i = 0; i < tierPaths.size(); ++i) {
90-
tierConfigs.push_back(MemoryTierCacheConfig::fromFile(tierPaths[i])
91-
.setRatio(tierRatios[i]));
92-
}
93-
cfg.configureMemoryTiers(tierConfigs);
94-
return cfg;
99+
cfg.configureMemoryTiers(tierConfigs);
100+
return cfg;
101+
}
102+
103+
LruAllocatorConfig createTieredCacheConfig(size_t totalCacheSize,
104+
size_t numTiers = 2) {
105+
LruAllocatorConfig tieredCacheConfig{};
106+
std::vector<MemoryTierCacheConfig> configs;
107+
for (auto i = 1; i <= numTiers; ++i) {
108+
configs.push_back(MemoryTierCacheConfig::fromFile(
109+
folly::sformat("/tmp/tier{}-{}", i, ::getpid()))
110+
.setRatio(1));
95111
}
112+
tieredCacheConfig.setCacheSize(totalCacheSize)
113+
.enableCachePersistence(
114+
folly::sformat("/tmp/multi-tier-test/{}", ::getpid()))
115+
.usePosixForShm()
116+
.configureMemoryTiers(configs);
117+
return tieredCacheConfig;
118+
}
119+
120+
LruAllocatorConfig createDramCacheConfig(size_t totalCacheSize) {
121+
LruAllocatorConfig dramConfig{};
122+
dramConfig.setCacheSize(totalCacheSize);
123+
return dramConfig;
124+
}
96125
};
97126

98127
using LruMemoryTiersTest = MemoryTiersTest<LruAllocator>;
@@ -114,15 +143,17 @@ TEST_F(LruMemoryTiersTest, TestValid2TierDaxPmemConfig) {
114143
}
115144

116145
TEST_F(LruMemoryTiersTest, TestValid2TierDaxPmemRatioConfig) {
117-
LruAllocatorConfig cfg = createTestCacheConfig({defaultDaxPath, defaultPmemPath},
118-
{5, 2});
146+
LruAllocatorConfig cfg =
147+
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
148+
{5, 2});
119149
basicCheck(cfg, {defaultDaxPath, defaultPmemPath});
120150
}
121151

122152
TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigPosixShmNotSet) {
123-
LruAllocatorConfig cfg = createTestCacheConfig({defaultDaxPath, defaultPmemPath},
124-
{1, 1},
125-
/* setPosixShm */ false);
153+
LruAllocatorConfig cfg =
154+
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
155+
{1, 1},
156+
/* setPosixShm */ false);
126157
}
127158

128159
TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigNumberOfPartitionsTooLarge) {
@@ -132,10 +163,47 @@ TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigNumberOfPartitionsTooLarge) {
132163
}
133164

134165
TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigRatiosCacheSizeNotSet) {
135-
EXPECT_THROW(createTestCacheConfig({defaultDaxPath, defaultPmemPath},
136-
{1, 1},
137-
/* setPosixShm */ true, /* cacheSize */ 0).validate(),
138-
std::invalid_argument);
166+
EXPECT_THROW(
167+
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
168+
{1, 1},
169+
/* setPosixShm */ true, /* cacheSize */ 0)
170+
.validate(),
171+
std::invalid_argument);
172+
}
173+
174+
TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigRatioNotSet) {
175+
EXPECT_THROW(
176+
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
177+
{1, 0}),
178+
std::invalid_argument);
179+
}
180+
181+
TEST_F(LruMemoryTiersTest, TestTieredCacheSize) {
182+
size_t totalSizes[] = {50 * MB, 77 * MB, 100 * MB, 101 * MB + MB / 2,
183+
1 * GB, 4 * GB, 8 * GB, 9 * GB};
184+
size_t numTiers[] = {2};
185+
186+
auto getCacheSize = [&](size_t cacheSize, size_t tiers) {
187+
std::unique_ptr<LruAllocator> alloc;
188+
if (tiers < 2) {
189+
alloc = std::unique_ptr<LruAllocator>(
190+
new LruAllocator(createDramCacheConfig(cacheSize)));
191+
} else {
192+
alloc = std::unique_ptr<LruAllocator>(
193+
new LruAllocator(LruAllocator::SharedMemNew,
194+
createTieredCacheConfig(cacheSize, tiers)));
195+
}
196+
return alloc->getCacheMemoryStats().cacheSize;
197+
};
198+
199+
for (auto totalSize : totalSizes) {
200+
auto dramCacheSize = getCacheSize(totalSize, 1);
201+
for (auto n : numTiers) {
202+
auto tieredCacheSize = getCacheSize(totalSize, n);
203+
EXPECT_GT(dramCacheSize, tieredCacheSize);
204+
EXPECT_GE(metaDataSize * n * 2, dramCacheSize - tieredCacheSize);
205+
}
206+
}
139207
}
140208

141209
} // namespace tests

0 commit comments

Comments
 (0)