improved background eviction

Author: byrnedj

cachelib/allocator/BackgroundMover-inl.h

@@ -36,6 +36,22 @@ BackgroundMover<CacheT>::BackgroundMover(
   }
 }
 
+template <typename CacheT>
+void BackgroundMover<CacheT>::TraversalStats::recordTraversalTime(uint64_t msTaken) {
+  lastTraversalTimeMs_.store(msTaken, std::memory_order_relaxed);
+  minTraversalTimeMs_.store(std::min(minTraversalTimeMs_.load(), msTaken),
+                            std::memory_order_relaxed);
+  maxTraversalTimeMs_.store(std::max(maxTraversalTimeMs_.load(), msTaken),
+                            std::memory_order_relaxed);
+  totalTraversalTimeMs_.fetch_add(msTaken, std::memory_order_relaxed);
+}
+
+template <typename CacheT>
+uint64_t BackgroundMover<CacheT>::TraversalStats::getAvgTraversalTimeMs(
+    uint64_t numTraversals) const {
+  return numTraversals ? totalTraversalTimeMs_ / numTraversals : 0;
+}
+
 template <typename CacheT>
 BackgroundMover<CacheT>::~BackgroundMover() {
   stop(std::chrono::seconds(0));
@@ -73,6 +89,7 @@ void BackgroundMover<CacheT>::checkAndRun() {
   std::set<ClassId> classes{};
   auto batches = strategy_->calculateBatchSizes(cache_, assignedMemory);
 
+  const auto begin = util::getCurrentTimeNs();
   //const auto& mpStats = cache_.getPoolByTid(0, 0).getStats();
   for (size_t i = 0; i < batches.size(); i++) {
     const auto [tid, pid, cid] = assignedMemory[i];
@@ -87,28 +104,39 @@ void BackgroundMover<CacheT>::checkAndRun() {
     // try moving BATCH items from the class in order to reach free target
     auto moved = moverFunc(cache_, tid, pid, cid, batch);
     moves += moved;
-    moves_per_class_[tid][pid][cid] += moved;
+    moves_per_class_[assignedMemory[i]] += moved;
     //totalBytesMoved.add(moved * mpStats.acStats.at(cid).allocSize);
   }
+  auto end = util::getCurrentTimeNs();
+  if (moves > 0) {
+    traversalStats_.recordTraversalTime(end > begin ? end - begin : 0);
+    numMovedItems.add(moves);
+    numTraversals.inc();
+    totalClasses.add(classes.size());
+  }
 
-  numTraversals.inc();
-  numMovedItems.add(moves);
-  totalClasses.add(classes.size());
 }
 
 template <typename CacheT>
 BackgroundMoverStats BackgroundMover<CacheT>::getStats() const noexcept {
   BackgroundMoverStats stats;
   stats.numMovedItems = numMovedItems.get();
-  stats.runCount = numTraversals.get();
   stats.totalBytesMoved = totalBytesMoved.get();
   stats.totalClasses = totalClasses.get();
+  auto runCount = getRunCount();
+  stats.runCount = runCount;
+  stats.numTraversals = numTraversals.get();
+  stats.avgItemsMoved = (double) stats.numMovedItems / (double)runCount;
+  stats.lastTraversalTimeMs = traversalStats_.getLastTraversalTimeMs();
+  stats.avgTraversalTimeMs = traversalStats_.getAvgTraversalTimeMs(runCount);
+  stats.minTraversalTimeMs = traversalStats_.getMinTraversalTimeMs();
+  stats.maxTraversalTimeMs = traversalStats_.getMaxTraversalTimeMs();
 
   return stats;
 }
 
 template <typename CacheT>
-std::map<TierId, std::map<PoolId, std::map<ClassId, uint64_t>>>
+std::map<MemoryDescriptorType,uint64_t>
 BackgroundMover<CacheT>::getClassStats() const noexcept {
   return moves_per_class_;
 }

cachelib/allocator/BackgroundMover.h

@@ -74,14 +74,34 @@ class BackgroundMover : public PeriodicWorker {
   ~BackgroundMover() override;
 
   BackgroundMoverStats getStats() const noexcept;
-  std::map<TierId, std::map<PoolId, std::map<ClassId, uint64_t>>>
-  getClassStats() const noexcept;
+  std::map<MemoryDescriptorType,uint64_t> getClassStats() const noexcept;
 
   void setAssignedMemory(std::vector<MemoryDescriptorType>&& assignedMemory);
 
  private:
-  std::map<TierId, std::map<PoolId, std::map<ClassId, uint64_t>>>
-      moves_per_class_;
+  std::map<MemoryDescriptorType,uint64_t> moves_per_class_;
+  
+  struct TraversalStats {
+    // record a traversal and its time taken
+    void recordTraversalTime(uint64_t msTaken);
+
+    uint64_t getAvgTraversalTimeMs(uint64_t numTraversals) const;
+    uint64_t getMinTraversalTimeMs() const { return minTraversalTimeMs_; }
+    uint64_t getMaxTraversalTimeMs() const { return maxTraversalTimeMs_; }
+    uint64_t getLastTraversalTimeMs() const { return lastTraversalTimeMs_; }
+    uint64_t getNumTraversals() const { return numTraversals_; }
+
+   private:
+    // time it took us the last time to traverse the cache.
+    std::atomic<uint64_t> lastTraversalTimeMs_{0};
+    std::atomic<uint64_t> minTraversalTimeMs_{
+        std::numeric_limits<uint64_t>::max()};
+    std::atomic<uint64_t> maxTraversalTimeMs_{0};
+    std::atomic<uint64_t> totalTraversalTimeMs_{0};
+    std::atomic<uint64_t> numTraversals_{0};
+  };
+  
+  TraversalStats traversalStats_;
   // cache allocator's interface for evicting
   using Item = typename Cache::Item;
 

cachelib/allocator/BackgroundMoverStrategy.h

@@ -22,13 +22,6 @@
 namespace facebook {
 namespace cachelib {
 
-struct MemoryDescriptorType {
-    MemoryDescriptorType(TierId tid, PoolId pid, ClassId cid) : 
-        tid_(tid), pid_(pid), cid_(cid) {}
-    TierId tid_;
-    PoolId pid_;
-    ClassId cid_;
-};
 
 // Base class for background eviction strategy.
 class BackgroundMoverStrategy {
@@ -38,5 +31,27 @@ class BackgroundMoverStrategy {
       std::vector<MemoryDescriptorType> acVec) = 0;
 };
 
+class DefaultBackgroundMoverStrategy : public BackgroundMoverStrategy {
+  public:
+    DefaultBackgroundMoverStrategy(uint64_t batchSize)
+      : batchSize_(batchSize) {}
+    ~DefaultBackgroundMoverStrategy() {}
+
+  std::vector<size_t> calculateBatchSizes(
+      const CacheBase& cache,
+      std::vector<MemoryDescriptorType> acVec) {
+    std::vector<size_t> batches{};
+    for (auto [tid, pid, cid] : acVec) {
+        auto stats = cache.getACStats(tid, pid, cid);
+        if (stats.usageFraction() > 0.90) {
+          batches.push_back(batchSize_);
+        }
+    }
+    return batches;
+  }
+  private:
+    uint64_t batchSize_{100};
+};
+
 } // namespace cachelib
 } // namespace facebook

cachelib/allocator/Cache.h

@@ -73,6 +73,38 @@ enum class DestructorContext {
   kRemovedFromNVM
 };
 
+struct MemoryDescriptorType {
+    MemoryDescriptorType(TierId tid, PoolId pid, ClassId cid) : 
+        tid_(tid), pid_(pid), cid_(cid) {}
+    TierId tid_;
+    PoolId pid_;
+    ClassId cid_;
+
+    bool operator<(const MemoryDescriptorType& rhs) {
+        if (this->tid_ <= rhs.tid_) {
+            if (this->pid_ <= rhs.pid_) {
+                if (this->cid_ < rhs.cid_) return true;
+                else return false;
+            }
+            return false;
+        }
+        return false;
+    }
+    
+    friend bool operator<(const MemoryDescriptorType& lhs, const MemoryDescriptorType& rhs);
+};
+
+inline bool operator<(const MemoryDescriptorType& lhs, const MemoryDescriptorType& rhs) {
+  if (lhs.tid_ <= rhs.tid_) {
+      if (lhs.pid_ <= rhs.pid_) {
+          if (lhs.cid_ < rhs.cid_) return true;
+          else return false;
+      }
+      return false;
+  }
+  return false;
+}
+
 // A base class of cache exposing members and status agnostic of template type.
 class CacheBase {
  public:

cachelib/allocator/CacheAllocator.h

@@ -1188,42 +1188,37 @@ class CacheAllocator : public CacheBase {
     return stats;
   }
 
-  // returns the background mover stats
-  BackgroundMoverStats getBackgroundMoverStats(MoverDir direction) const {
-    auto stats = BackgroundMoverStats{};
+  // returns the background mover stats per thread
+  std::vector<BackgroundMoverStats> getBackgroundMoverStats(MoverDir direction) const {
+    auto stats = std::vector<BackgroundMoverStats>();
     if (direction == MoverDir::Evict) {
       for (auto& bg : backgroundEvictor_)
-        stats += bg->getStats();
+        stats.push_back(bg->getStats());
     } else if (direction == MoverDir::Promote) {
       for (auto& bg : backgroundPromoter_)
-        stats += bg->getStats();
+        stats.push_back(bg->getStats());
     }
     return stats;
   }
 
-  std::map<TierId, std::map<PoolId, std::map<ClassId, uint64_t>>>
+  std::map<MemoryDescriptorType,uint64_t>
   getBackgroundMoverClassStats(MoverDir direction) const {
-    std::map<TierId, std::map<PoolId, std::map<ClassId, uint64_t>>> stats;
+    std::map<MemoryDescriptorType,uint64_t> stats;
+    auto record = [&](auto &bg) {
+      //gives a unique descriptor
+      auto classStats = bg->getClassStats();
+      for (const auto& [key,value] : classStats) {
+          stats[key] = value;
+      }
+    };
 
     if (direction == MoverDir::Evict) {
       for (auto& bg : backgroundEvictor_) {
-        for (auto& tid : bg->getClassStats()) {
-          for (auto& pid : tid.second) {
-            for (auto& cid : pid.second) {
-              stats[tid.first][pid.first][cid.first] += cid.second;
-            }
-          }
-        }
+          record(bg);
       }
     } else if (direction == MoverDir::Promote) {
       for (auto& bg : backgroundPromoter_) {
-        for (auto& tid : bg->getClassStats()) {
-          for (auto& pid : tid.second) {
-            for (auto& cid : pid.second) {
-              stats[tid.first][pid.first][cid.first] += cid.second;
-            }
-          }
-        }
+          record(bg);
       }
     }
 
@@ -2253,8 +2248,13 @@ auto& mmContainer = getMMContainer(tid, pid, cid);
           newItemHandles[index]->getSize());
         (*stats_.dsaEvictionSubmits)[tid][pid][cid].inc();
         if (config_.dsaAsync) {
-          dmlHandles.emplace_back(
-            dml::submit<dml::hardware>(dml::mem_copy, srcView, dstView));
+          if (cid < 11) {
+            dmlHandles.emplace_back(
+              dml::submit<dml::software>(dml::mem_copy, srcView, dstView));
+          } else {
+            dmlHandles.emplace_back(
+              dml::submit<dml::hardware>(dml::mem_copy, srcView, dstView));
+          }
         } else {
           auto dmlHandle = dml::submit<dml::hardware>(dml::mem_copy, srcView, dstView);
           auto result = dmlHandle.get();
@@ -2804,7 +2804,9 @@ auto& mmContainer = getMMContainer(tid, pid, cid);
   // free memory monitor
   std::unique_ptr<MemoryMonitor> memMonitor_;
 
-  // background evictor
+  // background data movement
+
+
   std::vector<std::unique_ptr<BackgroundMover<CacheT>>> backgroundEvictor_;
   std::vector<std::unique_ptr<BackgroundMover<CacheT>>> backgroundPromoter_;
 

cachelib/allocator/CacheStats.h

@@ -311,18 +311,34 @@ struct BackgroundMoverStats {
   uint64_t numMovedItems{0};
   // number of times we went executed the thread //TODO: is this def correct?
   uint64_t runCount{0};
+  
+  double avgItemsMoved{0.0};
+
+  uint64_t numTraversals{0};
   // total number of classes
   uint64_t totalClasses{0};
   // eviction size
   uint64_t totalBytesMoved{0};
+  
+  // indicates the time in ms for the last iteration across the entire cache
+  uint64_t lastTraversalTimeMs{0};
 
-  BackgroundMoverStats& operator+=(const BackgroundMoverStats& rhs) {
-    numMovedItems += rhs.numMovedItems;
-    runCount += rhs.runCount;
-    totalClasses += rhs.totalClasses;
-    totalBytesMoved += rhs.totalBytesMoved;
-    return *this;
-  }
+  // indicates the maximum of all traversals
+  uint64_t minTraversalTimeMs{0};
+
+  // indicates the minimum of all traversals
+  uint64_t maxTraversalTimeMs{0};
+
+  // indicates the average of all traversals
+  uint64_t avgTraversalTimeMs{0};
+
+  //BackgroundMoverStats& operator+=(const BackgroundMoverStats& rhs) {
+  //  numMovedItems += rhs.numMovedItems;
+  //  runCount += rhs.runCount;
+  //  totalClasses += rhs.totalClasses;
+  //  totalBytesMoved += rhs.totalBytesMoved;
+  //  return *this;
+  //}
 };
 
 
@@ -347,9 +363,9 @@ struct Stats;
 // the ones that are aggregated over all pools
 struct GlobalCacheStats {
   // background eviction stats
-  BackgroundMoverStats evictionStats;
+  std::vector<BackgroundMoverStats> evictionStats;
 
-  BackgroundMoverStats promotionStats;
+  std::vector<BackgroundMoverStats> promotionStats;
 
   // number of calls to CacheAllocator::find
   uint64_t numCacheGets{0};

cachelib/cachebench/cache/Cache-inl.h

@@ -646,13 +646,15 @@ Stats Cache<Allocator>::getStats() const {
     ret.numItems.push_back(aggregate.numItems());
   }
 
-  std::map<TierId, std::map<PoolId, std::map<ClassId, ACStats>>> allocationClassStats{};
+  std::map<MemoryDescriptorType, ACStats> allocationClassStats{};
 
   for (size_t pid = 0; pid < pools_.size(); pid++) {
     auto cids = cache_->getPoolStats(static_cast<PoolId>(pid)).getClassIds();
     for (TierId tid = 0; tid < cache_->getNumTiers(); tid++) {
-      for (auto cid : cids)
-        allocationClassStats[tid][pid][cid] = cache_->getACStats(tid, pid, cid);
+      for (auto cid : cids) {
+        MemoryDescriptorType md(tid,pid,cid);
+        allocationClassStats[md] = cache_->getACStats(tid, pid, cid);
+      }
     }
   }
 
@@ -662,19 +664,8 @@ Stats Cache<Allocator>::getStats() const {
 
   ret.allocationClassStats = allocationClassStats;
 
-  ret.backgndEvicStats.nEvictedItems =
-            cacheStats.evictionStats.numMovedItems;
-  ret.backgndEvicStats.nTraversals =
-            cacheStats.evictionStats.runCount;
-  ret.backgndEvicStats.nClasses =
-            cacheStats.evictionStats.totalClasses;
-  ret.backgndEvicStats.evictionSize =
-            cacheStats.evictionStats.totalBytesMoved;
-  
-  ret.backgndPromoStats.nPromotedItems =
-            cacheStats.promotionStats.numMovedItems;
-  ret.backgndPromoStats.nTraversals =
-            cacheStats.promotionStats.runCount;
+  ret.backgroundEvictorStats = cacheStats.evictionStats;
+  ret.backgroundPromoStats = cacheStats.promotionStats;
 
   ret.evictAttempts = cacheStats.evictionAttempts;
   ret.dsaEvictionSubmits = cacheStats.dsaEvictionSubmits;