fix: cpu utilization information may be bogus; switch to cgroup-based stats

This also adds a CPU Throttling metric.

Author: starpit

guidebooks/ml/codeflare/custodian/containers/cpu/cpu.awk

@@ -1,14 +1,32 @@
-$NF !~ /timestamp/ && $3 !="swpd" {
-    printf("\x1b[1;33m[CPU Utilization %4.1f%%] \x1b[0;33m%-30s \x1b[2m%14s %2s %2s %2s %2s %2s %sT%sZ\x1b[0m\n",
-           $13+$14,
-           pod,
-           $4,
-           $13,
-           $14,
-           $15,
-           $16,
-           $17,
-           $(NF-1),
-           $NF);
-    fflush();
+$1=="usage_usec"     {to=$2}
+$1=="user_usec"      {us=$2}
+$1=="system_usec"    {sy=$2}
+$1=="throttled_usec" {th=$2}
+$1=="nr_periods"     {np=$2}
+$1=="nr_throttled"   {nt=$2}
+$1=="nanos"          {nanos=$2}
+$1=="timestamp"      {now=$2}
+$1=="------" {
+    if (length(prev) > 0) {
+        delta = (nanos - prev) / 1000;
+
+        printf("\x1b[1;33m[CPU Utilization %4.1f%%] \x1b[0;33m%-30s \x1b[2m%s %s %s %s %s\x1b[0m\n",
+               to/delta,
+               pod,
+               us,
+               sy,
+               to,
+               th,
+               timestamp);
+
+        printf("\x1b[1;31m[CPU Throttling %4.1f%%] \x1b[0;31m%-30s \x1b[2m%s %s %s\x1b[0m\n",
+               nt/np,
+               pod,
+               nt,
+               np,
+               timestamp);
+        fflush();
+    }
+
+    prev = nanos;
 }

guidebooks/ml/codeflare/custodian/containers/cpu/cpu.sh

@@ -3,4 +3,4 @@
 kubectl get pod -l ${KUBE_PODFULL_LABEL_SELECTOR} -o name \
         --field-selector=status.phase==Running \
     | xargs -P512 -I {} -n1 \
-            sh -c 'kubectl exec {} -- sh -c "TZ=UTC vmstat --timestamp 5" | awk -v pod={} -f /tmp/cpu.awk' 2> /dev/null
+            sh -c 'kubectl exec {} -- sh -c "while true; do echo \"timestamp \\\$(date -u)\"; echo \"nanos \\\$(date +%s%N)\"; cat /sys/fs/cgroup/cpu.stat; echo ------; sleep 1; done" | awk -v pod={} -f /tmp/cpu.awk'