Skip to content

Commit 3477453

Browse files
byrnedjbyrnedj
committed
Tests and fix tier sizing
------------------------- There are two parts to this commit and we can split them up. Part 1) This commit contains the additional memory tiers tests for different pool sizes. We also use getPoolSize(pid), to get total size from all pools across allocators. Part 2) This part can be merged with the initial multi-tier part 1. It fixes the tiering sizes (pulls changes from what was issue75 rebased commit that did not make it into upstream commits).
1 parent 245ec1b commit 3477453

File tree

4 files changed

+156
-14
lines changed

4 files changed

+156
-14
lines changed

‎cachelib/allocator/CacheAllocator.h‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2210,6 +2210,8 @@ class CacheAllocator : public CacheBase {
22102210
return config_.memoryTierConfigs.size();
22112211
}
22122212

2213+
size_t memoryTierSize(TierId tid) const;
2214+
22132215
// Whether the memory allocator for this cache allocator was created on shared
22142216
// memory. The hash table, chained item hash table etc is also created on
22152217
// shared memory except for temporary shared memory mode when they're created
@@ -2496,6 +2498,16 @@ ShmSegmentOpts CacheAllocator<CacheTrait>::createShmCacheOpts(TierId tid) {
24962498
return opts;
24972499
}
24982500

2501+
template <typename CacheTrait>
2502+
size_t CacheAllocator<CacheTrait>::memoryTierSize(TierId tid) const {
2503+
auto partitions = std::accumulate(config_.memoryTierConfigs.begin(), config_.memoryTierConfigs.end(), 0UL,
2504+
[](const size_t i, const MemoryTierCacheConfig& config){
2505+
return i + config.getRatio();
2506+
});
2507+
2508+
return config_.memoryTierConfigs[tid].calculateTierSize(config_.getCacheSize(), partitions);
2509+
}
2510+
24992511
template <typename CacheTrait>
25002512
std::vector<std::unique_ptr<MemoryAllocator>>
25012513
CacheAllocator<CacheTrait>::createPrivateAllocator() {
@@ -2518,14 +2530,15 @@ CacheAllocator<CacheTrait>::createPrivateAllocator() {
25182530
template <typename CacheTrait>
25192531
std::unique_ptr<MemoryAllocator>
25202532
CacheAllocator<CacheTrait>::createNewMemoryAllocator(TierId tid) {
2533+
size_t tierSize = memoryTierSize(tid);
25212534
return std::make_unique<MemoryAllocator>(
25222535
getAllocatorConfig(config_),
25232536
shmManager_
25242537
->createShm(detail::kShmCacheName + std::to_string(tid),
2525-
config_.getCacheSize(), config_.slabMemoryBaseAddr,
2538+
tierSize, config_.slabMemoryBaseAddr,
25262539
createShmCacheOpts(tid))
25272540
.addr,
2528-
config_.getCacheSize());
2541+
tierSize);
25292542
}
25302543

25312544
template <typename CacheTrait>
@@ -2536,7 +2549,7 @@ CacheAllocator<CacheTrait>::restoreMemoryAllocator(TierId tid) {
25362549
shmManager_
25372550
->attachShm(detail::kShmCacheName + std::to_string(tid),
25382551
config_.slabMemoryBaseAddr, createShmCacheOpts(tid)).addr,
2539-
config_.getCacheSize(),
2552+
memoryTierSize(tid),
25402553
config_.disableFullCoredump);
25412554
}
25422555

@@ -4830,6 +4843,16 @@ const std::string CacheAllocator<CacheTrait>::getCacheName() const {
48304843
return config_.cacheName;
48314844
}
48324845

4846+
template <typename CacheTrait>
4847+
size_t CacheAllocator<CacheTrait>::getPoolSize(PoolId poolId) const {
4848+
size_t poolSize = 0;
4849+
for (auto& allocator: allocator_) {
4850+
const auto& pool = allocator->getPool(poolId);
4851+
poolSize += pool.getPoolSize();
4852+
}
4853+
return poolSize;
4854+
}
4855+
48334856
template <typename CacheTrait>
48344857
PoolStats CacheAllocator<CacheTrait>::getPoolStats(PoolId poolId) const {
48354858
stats().numExpensiveStatsPolled.inc();
@@ -5782,9 +5805,12 @@ GlobalCacheStats CacheAllocator<CacheTrait>::getGlobalCacheStats() const {
57825805

57835806
template <typename CacheTrait>
57845807
CacheMemoryStats CacheAllocator<CacheTrait>::getCacheMemoryStats() const {
5785-
const auto totalCacheSize = allocator_[0]->getMemorySize();
5786-
const auto configuredTotalCacheSize = allocator_[0]->getMemorySizeInclAdvised();
5787-
5808+
size_t totalCacheSize = 0;
5809+
size_t configuredTotalCacheSize = 0;
5810+
for(auto& allocator: allocator_) {
5811+
totalCacheSize += allocator->getMemorySize();
5812+
configuredTotalCacheSize += allocator->getMemorySizeInclAdvised();
5813+
}
57885814
auto addSize = [this](size_t a, PoolId pid) {
57895815
return a + allocator_[0]->getPool(pid).getPoolSize();
57905816
};

‎cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ namespace tests {
2323
using LruAllocatorMemoryTiersTest = AllocatorMemoryTiersTest<LruAllocator>;
2424

2525
// TODO(MEMORY_TIER): add more tests with different eviction policies
26-
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValid1) {
27-
this->testMultiTiersValid1();
28-
}
26+
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersInvalid) { this->testMultiTiersInvalid(); }
27+
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValid) { this->testMultiTiersValid(); }
28+
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValidMixed) { this->testMultiTiersValidMixed(); }
2929

3030
} // end of namespace tests
3131
} // end of namespace cachelib

‎cachelib/allocator/tests/AllocatorMemoryTiersTest.h‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ namespace tests {
2727
template <typename AllocatorT>
2828
class AllocatorMemoryTiersTest : public AllocatorTest<AllocatorT> {
2929
public:
30-
void testMultiTiersValid1() {
30+
void testMultiTiersInvalid() {
3131
typename AllocatorT::Config config;
3232
config.setCacheSize(100 * Slab::kSize);
3333
ASSERT_NO_THROW(config.configureMemoryTiers(
@@ -36,6 +36,44 @@ class AllocatorMemoryTiersTest : public AllocatorTest<AllocatorT> {
3636
MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind(
3737
std::string("0"))}));
3838
}
39+
40+
void testMultiTiersValid() {
41+
typename AllocatorT::Config config;
42+
config.setCacheSize(100 * Slab::kSize);
43+
config.enableCachePersistence("/tmp");
44+
ASSERT_NO_THROW(config.configureMemoryTiers(
45+
{MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind(
46+
std::string("0")),
47+
MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind(
48+
std::string("0"))}));
49+
50+
auto alloc = std::make_unique<AllocatorT>(AllocatorT::SharedMemNew, config);
51+
ASSERT(alloc != nullptr);
52+
53+
auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().ramCacheSize);
54+
auto handle = alloc->allocate(pool, "key", std::string("value").size());
55+
ASSERT(handle != nullptr);
56+
ASSERT_NO_THROW(alloc->insertOrReplace(handle));
57+
}
58+
59+
void testMultiTiersValidMixed() {
60+
typename AllocatorT::Config config;
61+
config.setCacheSize(100 * Slab::kSize);
62+
config.enableCachePersistence("/tmp");
63+
ASSERT_NO_THROW(config.configureMemoryTiers(
64+
{MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind(
65+
std::string("0")),
66+
MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind(
67+
std::string("0"))}));
68+
69+
auto alloc = std::make_unique<AllocatorT>(AllocatorT::SharedMemNew, config);
70+
ASSERT(alloc != nullptr);
71+
72+
auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().ramCacheSize);
73+
auto handle = alloc->allocate(pool, "key", std::string("value").size());
74+
ASSERT(handle != nullptr);
75+
ASSERT_NO_THROW(alloc->insertOrReplace(handle));
76+
}
3977
};
4078
} // namespace tests
4179
} // namespace cachelib

‎cachelib/allocator/tests/MemoryTiersTest.cpp‎

Lines changed: 82 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ constexpr size_t MB = 1024ULL * 1024ULL;
3434
constexpr size_t GB = MB * 1024ULL;
3535

3636
const size_t defaultTotalCacheSize{1 * GB};
37-
const std::string defaultCacheDir{"/var/metadataDir"};
37+
const std::string defaultCacheDir{"/tmp/metadataDir"};
3838

3939
template <typename Allocator>
4040
class MemoryTiersTest : public AllocatorTest<Allocator> {
@@ -109,7 +109,7 @@ class MemoryTiersTest : public AllocatorTest<Allocator> {
109109
void validatePoolSize(PoolId poolId,
110110
std::unique_ptr<LruAllocator>& allocator,
111111
size_t expectedSize) {
112-
size_t actualSize = allocator->getPool(poolId).getPoolSize();
112+
size_t actualSize = allocator->getPoolSize(poolId);
113113
EXPECT_EQ(actualSize, expectedSize);
114114
}
115115

@@ -119,9 +119,9 @@ class MemoryTiersTest : public AllocatorTest<Allocator> {
119119
size_t numTiers = 2) {
120120
if (isSizeValid) {
121121
auto pool = alloc->addPool("validPoolSize", poolSize);
122-
EXPECT_LE(alloc->getPool(pool).getPoolSize(), poolSize);
122+
EXPECT_LE(alloc->getPoolSize(pool), poolSize);
123123
if (poolSize >= numTiers * Slab::kSize)
124-
EXPECT_GE(alloc->getPool(pool).getPoolSize(),
124+
EXPECT_GE(alloc->getPoolSize(pool),
125125
poolSize - numTiers * Slab::kSize);
126126
} else {
127127
EXPECT_THROW(alloc->addPool("invalidPoolSize", poolSize),
@@ -172,6 +172,84 @@ TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigRatioNotSet) {
172172
TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigSizesNeCacheSize) {
173173
EXPECT_THROW(createTestCacheConfig({0, 0}), std::invalid_argument);
174174
}
175+
176+
TEST_F(LruMemoryTiersTest, TestPoolAllocations) {
177+
std::vector<size_t> totalCacheSizes = {8 * GB, 2 * GB};
178+
179+
static const size_t numExtraSizes = 4;
180+
static const size_t numExtraSlabs = 20;
181+
182+
for (size_t i = 0; i < numExtraSizes; i++) {
183+
totalCacheSizes.push_back(totalCacheSizes.back() +
184+
(folly::Random::rand64() % numExtraSlabs) *
185+
Slab::kSize);
186+
}
187+
188+
size_t min_ratio = 1;
189+
size_t max_ratio = 111;
190+
191+
static const size_t numCombinations = 10;
192+
193+
for (auto totalCacheSize : totalCacheSizes) {
194+
for (size_t k = 0; k < numCombinations; k++) {
195+
const size_t i = folly::Random::rand32() % max_ratio + min_ratio;
196+
const size_t j = folly::Random::rand32() % max_ratio + min_ratio;
197+
LruAllocatorConfig cfg =
198+
createTestCacheConfig({i, j},
199+
/* usePoisx */ true, totalCacheSize);
200+
basicCheck(cfg, totalCacheSize);
201+
202+
std::unique_ptr<LruAllocator> alloc = std::unique_ptr<LruAllocator>(
203+
new LruAllocator(LruAllocator::SharedMemNew, cfg));
204+
205+
size_t size = (folly::Random::rand64() %
206+
(alloc->getCacheMemoryStats().ramCacheSize - Slab::kSize)) +
207+
Slab::kSize;
208+
testAddPool(alloc, size, true);
209+
}
210+
}
211+
}
212+
213+
TEST_F(LruMemoryTiersTest, TestPoolInvalidAllocations) {
214+
std::vector<size_t> totalCacheSizes = {48 * MB, 51 * MB, 256 * MB,
215+
1 * GB, 5 * GB, 8 * GB};
216+
size_t min_ratio = 1;
217+
size_t max_ratio = 111;
218+
219+
static const size_t numCombinations = 10;
220+
221+
for (auto totalCacheSize : totalCacheSizes) {
222+
for (size_t k = 0; k < numCombinations; k++) {
223+
const size_t i = folly::Random::rand32() % max_ratio + min_ratio;
224+
const size_t j = folly::Random::rand32() % max_ratio + min_ratio;
225+
LruAllocatorConfig cfg =
226+
createTestCacheConfig({i, j},
227+
/* usePoisx */ true, totalCacheSize);
228+
229+
std::unique_ptr<LruAllocator> alloc = nullptr;
230+
try {
231+
alloc = std::unique_ptr<LruAllocator>(
232+
new LruAllocator(LruAllocator::SharedMemNew, cfg));
233+
} catch(...) {
234+
// expection only if cache too small
235+
size_t sum_ratios = std::accumulate(
236+
cfg.getMemoryTierConfigs().begin(), cfg.getMemoryTierConfigs().end(), 0UL,
237+
[](const size_t i, const MemoryTierCacheConfig& config) {
238+
return i + config.getRatio();
239+
});
240+
auto tier1slabs = cfg.getMemoryTierConfigs()[0].calculateTierSize(cfg.getCacheSize(), sum_ratios) / Slab::kSize;
241+
auto tier2slabs = cfg.getMemoryTierConfigs()[1].calculateTierSize(cfg.getCacheSize(), sum_ratios) / Slab::kSize;
242+
EXPECT_TRUE(tier1slabs <= 2 || tier2slabs <= 2);
243+
244+
continue;
245+
}
246+
247+
size_t size = (folly::Random::rand64() % (100 * GB)) +
248+
alloc->getCacheMemoryStats().ramCacheSize;
249+
testAddPool(alloc, size, false);
250+
}
251+
}
252+
}
175253
} // namespace tests
176254
} // namespace cachelib
177255
} // namespace facebook

0 commit comments

Comments
 (0)