added per tier pool class rolling average latency

Author: guptask

cachelib/allocator/Cache.h

@@ -84,7 +84,7 @@ class CacheBase {
   CacheBase& operator=(CacheBase&&) = default;
 
   // TODO: come up with some reasonable number
-  static constexpr unsigned kMaxTiers = 8;
+  static constexpr unsigned kMaxTiers = 2;
 
   // Get a string referring to the cache name for this cache
   virtual const std::string getCacheName() const = 0;

cachelib/allocator/CacheAllocator-inl.h

@@ -382,6 +382,7 @@ CacheAllocator<CacheTrait>::allocateInternalTier(TierId tid,
 
   // the allocation class in our memory allocator.
   const auto cid = allocator_[tid]->getAllocationClassId(pid, requiredSize);
+  util::RollingLatencyTracker rollTracker{(*stats_.classAllocLatency)[tid][pid][cid]};
 
   // TODO: per-tier
   (*stats_.allocAttempts)[pid][cid].inc();
@@ -480,6 +481,8 @@ CacheAllocator<CacheTrait>::allocateChainedItemInternal(
   const auto pid = allocator_[tid]->getAllocInfo(parent->getMemory()).poolId;
   const auto cid = allocator_[tid]->getAllocationClassId(pid, requiredSize);
 
+  util::RollingLatencyTracker rollTracker{(*stats_.classAllocLatency)[tid][pid][cid]};
+
   // TODO: per-tier? Right now stats_ are not used in any public periodic
   // worker
   (*stats_.allocAttempts)[pid][cid].inc();
@@ -2540,6 +2543,7 @@ AllocationClassBaseStat CacheAllocator<CacheTrait>::getAllocationClassStats(
   } else {
     stats.approxFreePercent = ac.approxFreePercentage();
   }
+  stats.allocLatencyNs = (*stats_.classAllocLatency)[tid][pid][cid];
 
   return stats;
 }

cachelib/allocator/CacheStats.cpp

@@ -42,6 +42,8 @@ void Stats::init() {
   initToZero(*fragmentationSize);
   initToZero(*chainedItemEvictions);
   initToZero(*regularItemEvictions);
+
+  classAllocLatency = std::make_unique<PerTierPoolClassRollingStats>();
 }
 
 template <int>

cachelib/allocator/CacheStats.h

@@ -25,6 +25,7 @@
 #include "cachelib/allocator/memory/Slab.h"
 #include "cachelib/common/FastStats.h"
 #include "cachelib/common/PercentileStats.h"
+#include "cachelib/common/RollingStats.h"
 #include "cachelib/common/Time.h"
 
 namespace facebook {
@@ -107,6 +108,9 @@ struct AllocationClassBaseStat {
 
   // percent of free memory in this class
   double approxFreePercent{0.0};
+
+  // Rolling allocation latency (in ns)
+  util::RollingStats allocLatencyNs;
 };
 
 // cache related stats for a given allocation class.

cachelib/allocator/CacheStatsInternal.h

@@ -21,6 +21,7 @@
 #include "cachelib/allocator/Cache.h"
 #include "cachelib/allocator/memory/MemoryAllocator.h"
 #include "cachelib/common/AtomicCounter.h"
+#include "cachelib/common/RollingStats.h"
 
 namespace facebook {
 namespace cachelib {
@@ -221,6 +222,15 @@ struct Stats {
   std::unique_ptr<PerPoolClassAtomicCounters> chainedItemEvictions{};
   std::unique_ptr<PerPoolClassAtomicCounters> regularItemEvictions{};
 
+  using PerTierPoolClassRollingStats =
+      std::array<std::array<std::array<util::RollingStats,
+                MemoryAllocator::kMaxClasses>,
+                MemoryPoolManager::kMaxPools>,
+                CacheBase::kMaxTiers>;
+
+  // rolling latency tracking for every alloc class in every pool
+  std::unique_ptr<PerTierPoolClassRollingStats> classAllocLatency{};
+
   // Eviction failures due to parent cannot be removed from access container
   AtomicCounter evictFailParentAC{0};
 

cachelib/cachebench/cache/CacheStats.h

@@ -163,6 +163,12 @@ struct Stats {
         out << folly::sformat("tid{:2} pid{:2} cid{:4} {:8.2f}{} free: {:4.2f}%",
           tid, pid, cid, allocSize, allocSizeSuffix, stats.approxFreePercent) << std::endl;
       });
+
+      foreachAC([&](auto tid, auto pid, auto cid, auto stats){
+        auto [allocSizeSuffix, allocSize] = formatMemory(stats.allocSize);
+        out << folly::sformat("tid{:2} pid{:2} cid{:4} {:8.2f}{} latency: {:8.2f}%",
+          tid, pid, cid, allocSize, allocSizeSuffix, stats.allocLatencyNs.estimate()) << std::endl;
+      });
     }
 
     if (numCacheGets > 0) {

cachelib/common/RollingStats.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <folly/Range.h>
+#include <folly/logging/xlog.h>
+
+#include "cachelib/common/Utils.h"
+
+namespace facebook {
+namespace cachelib {
+namespace util {
+
+class RollingStats {
+ public:
+  // track latency by taking the value of duration directly.
+  void trackValue(double value) {
+    auto ratio = static_cast<double>(cnt_) / (cnt_ + 1);
+    avg_ *= ratio;
+    ++cnt_;
+    avg_ += value / cnt_;
+  }
+
+  // Return the rolling average.
+  uint64_t estimate() {
+    return static_cast<uint64_t>(avg_);
+  }
+
+ private:
+  double avg_ = 0;
+  uint64_t cnt_ = 0;
+};
+
+class RollingLatencyTracker {
+ public:
+  explicit RollingLatencyTracker(RollingStats& stats)
+      : stats_(&stats), begin_(std::chrono::steady_clock::now()) {}
+  RollingLatencyTracker() {}
+  ~RollingLatencyTracker() {
+    if (stats_) {
+      auto tp = std::chrono::steady_clock::now();
+      auto diffNanos =
+          std::chrono::duration_cast<std::chrono::nanoseconds>(tp - begin_)
+              .count();
+      stats_->trackValue(static_cast<double>(diffNanos));
+    }
+  }
+
+  RollingLatencyTracker(const RollingLatencyTracker&) = delete;
+  RollingLatencyTracker& operator=(const RollingLatencyTracker&) = delete;
+
+  RollingLatencyTracker(RollingLatencyTracker&& rhs) noexcept
+      : stats_(rhs.stats_), begin_(rhs.begin_) {
+    rhs.stats_ = nullptr;
+  }
+
+  RollingLatencyTracker& operator=(RollingLatencyTracker&& rhs) noexcept {
+    if (this != &rhs) {
+      this->~RollingLatencyTracker();
+      new (this) RollingLatencyTracker(std::move(rhs));
+    }
+    return *this;
+  }
+
+ private:
+  RollingStats* stats_{nullptr};
+  std::chrono::time_point<std::chrono::steady_clock> begin_;
+};
+} // namespace util
+} // namespace cachelib
+} // namespace facebook