diff --git a/perfscale_regression_ci/scripts/performance/node/psi/analyze_results.py b/perfscale_regression_ci/scripts/performance/node/psi/analyze_results.py new file mode 100644 index 00000000000..8dd14bfcb72 --- /dev/null +++ b/perfscale_regression_ci/scripts/performance/node/psi/analyze_results.py @@ -0,0 +1,134 @@ +import json +import os +import re +import sys +from collections import defaultdict + +if len(sys.argv) < 2: + print("Usage: python3 analyze_results.py ") + sys.exit(1) + +LOG_DIR = sys.argv[1] + +def parse_top_node_log(file_path): + """Parses the output of 'kubectl top node'.""" + with open(file_path, 'r') as f: + lines = f.readlines() + if len(lines) < 2: + return None, None + + # Skip header + parts = lines[1].split() + + cpu_raw = parts[1] + mem_raw = parts[3] + + cpu_cores = int(cpu_raw.replace('m', '')) + + mem_kib = 0 + if 'Ki' in mem_raw: + mem_kib = int(mem_raw.replace('Ki', '')) + elif 'Mi' in mem_raw: + mem_mib_raw = int(mem_raw.replace('Mi', '')) + mem_kib = mem_mib_raw * 1024 + + mem_mib = mem_kib / 1024 + + return cpu_cores, mem_mib + +def parse_stats_summary(file_path): + """Parses the kubelet stats/summary JSON.""" + with open(file_path, 'r') as f: + data = json.load(f) + + kubelet_stats = next((sc for sc in data.get("node", {}).get("systemContainers", []) if sc["name"] == "kubelet"), None) + if not kubelet_stats: + return None, None + + cpu_nanocores = kubelet_stats.get("cpu", {}).get("usageNanoCores", 0) + mem_workingset_bytes = kubelet_stats.get("memory", {}).get("workingSetBytes", 0) + + return cpu_nanocores, mem_workingset_bytes / (1024**2) # Convert to MiB + +# Helper to calculate delta +def get_delta(base, test, unit, is_cpu=False): + if base is None or test is None: + return "N/A" + delta = test - base + percent = (delta / base * 100) if base != 0 else 0 + sign = "+" if delta >= 0 else "" + + # Convert nanocores to millicores for CPU + if is_cpu: + base /= 1_000_000 + test /= 1_000_000 + delta /= 1_000_000 + + return f"{base:.2f} -> {test:.2f} ({sign}{delta:.2f} / {sign}{percent:.2f}%)" + +def main(): + results = defaultdict(lambda: defaultdict(dict)) + + for filename in os.listdir(LOG_DIR): + if not filename.endswith(".log") and not filename.endswith(".json"): + continue + + parts = filename.split('_') + + feature_status = "enabled" if "enabled" in filename else "disabled" + stress_type = "unknown" + if "cpu" in filename: + stress_type = "cpu" + elif "memory" in filename: + stress_type = "memory" + elif "io" in filename: + stress_type = "io" + + condition = "unknown" + if "idle" in filename: + condition = "idle" + elif "load" in filename: + condition = "load" + + file_path = os.path.join(LOG_DIR, filename) + + if "top_node" in filename: + cpu, mem = parse_top_node_log(file_path) + results[stress_type][feature_status][f"{condition}_node_cpu_m"] = cpu + results[stress_type][feature_status][f"{condition}_node_mem_mib"] = mem + elif "stats_summary" in filename: + cpu, mem = parse_stats_summary(file_path) + results[stress_type][feature_status][f"{condition}_kubelet_cpu_nanocores"] = cpu + results[stress_type][feature_status][f"{condition}_kubelet_mem_mib"] = mem + + print("--- Performance Analysis Summary by kubectl top node and /proxy/stats/summary---") + + for stress_type, data in sorted(results.items()): + print(f"\n### Stress Type: {stress_type.upper()}\n") + + print("| Metric | Condition | Result (Baseline -> Test) |") + print("|------------------------|-----------|---------------------------|") + # Node Metrics + idle_node_cpu = get_delta(data["disabled"].get("idle_node_cpu_m"), data["enabled"].get("idle_node_cpu_m"), "m") + load_node_cpu = get_delta(data["disabled"].get("load_node_cpu_m"), data["enabled"].get("load_node_cpu_m"), "m") + idle_node_mem = get_delta(data["disabled"].get("idle_node_mem_mib"), data["enabled"].get("idle_node_mem_mib"), "MiB") + load_node_mem = get_delta(data["disabled"].get("load_node_mem_mib"), data["enabled"].get("load_node_mem_mib"), "MiB") + + print(f"| **Node CPU (m)** | Idle | {idle_node_cpu} |") + print(f"| **Node CPU (m)** | Load | {load_node_cpu} |") + print(f"| **Node Memory (MiB)** | Idle | {idle_node_mem} |") + print(f"| **Node Memory (MiB)** | Load | {load_node_mem} |") + + # Kubelet Metrics + idle_kubelet_cpu = get_delta(data["disabled"].get("idle_kubelet_cpu_nanocores"), data["enabled"].get("idle_kubelet_cpu_nanocores"), "m", is_cpu=True) + load_kubelet_cpu = get_delta(data["disabled"].get("load_kubelet_cpu_nanocores"), data["enabled"].get("load_kubelet_cpu_nanocores"), "m", is_cpu=True) + idle_kubelet_mem = get_delta(data["disabled"].get("idle_kubelet_mem_mib"), data["enabled"].get("idle_kubelet_mem_mib"), "MiB") + load_kubelet_mem = get_delta(data["disabled"].get("load_kubelet_mem_mib"), data["enabled"].get("load_kubelet_mem_mib"), "MiB") + + print(f"| **Kubelet CPU (m)** | Idle | {idle_kubelet_cpu} |") + print(f"| **Kubelet CPU (m)** | Load | {load_kubelet_cpu} |") + print(f"| **Kubelet Memory (MiB)**| Idle | {idle_kubelet_mem} |") + print(f"| **Kubelet Memory (MiB)**| Load | {load_kubelet_mem} |") + +if __name__ == "__main__": + main() diff --git a/perfscale_regression_ci/scripts/performance/node/psi/cluster-monitoring-config.yaml b/perfscale_regression_ci/scripts/performance/node/psi/cluster-monitoring-config.yaml new file mode 100644 index 00000000000..8c1a0e1c692 --- /dev/null +++ b/perfscale_regression_ci/scripts/performance/node/psi/cluster-monitoring-config.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: ConfigMap +data: + config.yaml: | + prometheusK8s: + retention: ${PROMETHEUS_RETENTION_PERIOD} + nodeSelector: + node-role.kubernetes.io/worker: "" + volumeClaimTemplate: + spec: + storageClassName: ${STORAGE_CLASS} + resources: + requests: + storage: ${PROMETHEUS_STORAGE_SIZE} + alertmanagerMain: + nodeSelector: + node-role.kubernetes.io/worker: "" + volumeClaimTemplate: + spec: + storageClassName: ${STORAGE_CLASS} + resources: + requests: + storage: ${ALERTMANAGER_STORAGE_SIZE} +metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring diff --git a/perfscale_regression_ci/scripts/performance/node/psi/enable_psi.sh b/perfscale_regression_ci/scripts/performance/node/psi/enable_psi.sh new file mode 100755 index 00000000000..95690426c35 --- /dev/null +++ b/perfscale_regression_ci/scripts/performance/node/psi/enable_psi.sh @@ -0,0 +1,266 @@ +#!/bin/bash +# +# enable-psi.sh - Enable PSI (Pressure Stall Information) on OpenShift Worker Nodes +# +# This script: +# 1. Checks if PSI is already enabled +# 2. Creates and applies MachineConfig to enable PSI +# 3. Waits for worker nodes to be updated +# 4. Verifies PSI is enabled on all workers +# +# Requirements: +# - oc CLI installed and logged in +# - Cluster admin permissions +# +# Usage: ./enable-psi.sh + +set -e # Exit on error +set -o pipefail # Catch errors in pipelines + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +MACHINECONFIG_NAME="99-worker-enable-psi" +MACHINECONFIG_FILE="99-worker-enable-psi.yaml" +CHECK_INTERVAL=30 # seconds between checks +MAX_WAIT_TIME=3600 # 1 hour timeout + +# Functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +check_prerequisites() { + log_info "Checking prerequisites..." + + # Check if oc is installed + if ! command -v oc &> /dev/null; then + log_error "oc CLI is not installed. Please install it first." + exit 1 + fi + + # Check if logged in + if ! oc whoami &> /dev/null; then + log_error "Not logged in to OpenShift. Please run 'oc login' first." + exit 1 + fi + + # Check cluster admin permissions + if ! oc auth can-i create machineconfigs &> /dev/null; then + log_error "Insufficient permissions. Cluster admin access required." + exit 1 + fi + + log_success "Prerequisites check passed" +} + +check_psi_status() { + local node=$1 + # Check if PSI files exist + psi_check=$( + oc debug node/"$node" -- chroot /host bash -c ' + [ -f /proc/pressure/cpu ] && [ -f /proc/pressure/memory ] && [ -f /proc/pressure/io ] \ + && echo enabled || echo disabled + ' 2>/dev/null | grep -E "enabled|disabled" + ) + + echo "$psi_check" +} + +check_all_workers_psi() { + log_info "Checking PSI status on all worker nodes..." + + local workers=$(oc get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[*].metadata.name}') + local all_enabled=true + + for worker in $workers; do + local status=$(check_psi_status "$worker") + if [ "$status" == "enabled" ]; then + log_success "PSI is enabled on $worker" + else + log_warning "PSI is NOT enabled on $worker" + all_enabled=false + fi + done + + if [ "$all_enabled" = true ]; then + return 0 + else + return 1 + fi +} + +create_machineconfig_yaml() { + log_info "Creating MachineConfig YAML: $MACHINECONFIG_FILE" + + cat > "$MACHINECONFIG_FILE" << 'EOF' +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: "worker" + name: 99-worker-enable-psi +spec: + kernelArguments: + - psi=1 +EOF + + log_success "MachineConfig YAML created" +} + +apply_machineconfig() { + log_info "Applying MachineConfig to cluster..." + + if oc get machineconfig "$MACHINECONFIG_NAME" &> /dev/null; then + log_error "MachineConfig $MACHINECONFIG_NAME already exists!" + log_error "Please delete it first with: oc delete machineconfig $MACHINECONFIG_NAME" + exit 1 + else + oc apply -f "$MACHINECONFIG_FILE" + log_success "MachineConfig applied" + fi + rm -rf $MACHINECONFIG_FILE +} + +wait_for_mcp_update() { + log_info "Waiting for MachineConfigPool 'worker' to update..." + log_info "This may take 30-60 minutes. Workers will reboot one by one." + + local elapsed=0 + local mcp_name="worker" + + while [ $elapsed -lt $MAX_WAIT_TIME ]; do + # Get MCP status (with default values if fields don't exist) + local updated=$(oc get mcp "$mcp_name" -o jsonpath='{.status.updatedMachineCount}' 2>/dev/null || echo "0") + local total=$(oc get mcp "$mcp_name" -o jsonpath='{.status.machineCount}' 2>/dev/null || echo "0") + local ready=$(oc get mcp "$mcp_name" -o jsonpath='{.status.readyMachineCount}' 2>/dev/null || echo "0") + local degraded=$(oc get mcp "$mcp_name" -o jsonpath='{.status.degradedMachineCount}' 2>/dev/null || echo "0") + + # Set to 0 if empty + updated=${updated:-0} + total=${total:-0} + ready=${ready:-0} + degraded=${degraded:-0} + + log_info "MCP Status: Updated=$updated/$total, Ready=$ready/$total, Degraded=$degraded" + + # Check if all nodes are updated and ready + if [ "$updated" -eq "$total" ] && [ "$ready" -eq "$total" ]; then + log_success "All worker nodes have been updated!" + return 0 + fi + + # Check for degraded nodes + if [ "$degraded" -gt 0 ]; then + log_warning "Some nodes are degraded. Checking details..." + oc get mcp "$mcp_name" -o yaml | grep -A 5 "conditions:" + fi + + log_info "Waiting $CHECK_INTERVAL seconds before next check... (elapsed: ${elapsed}s)" + sleep $CHECK_INTERVAL + elapsed=$((elapsed + CHECK_INTERVAL)) + done + + log_error "Timeout waiting for worker nodes to update after ${MAX_WAIT_TIME}s" + return 1 +} + +show_node_status() { + log_info "Current worker node status:" + oc get nodes -l node-role.kubernetes.io/worker -o wide +} + +main() { + echo "==========================================" + echo " OpenShift PSI Enablement Script" + echo "==========================================" + echo + + # Step 0: Check prerequisites + check_prerequisites + echo + + # Step 1: Check if PSI is already enabled + log_info "Step 1: Checking current PSI status on worker nodes..." + if check_all_workers_psi; then + log_success "PSI is already enabled on all worker nodes!" + exit 0 + else + log_info "PSI is not fully enabled. Proceeding with enablement..." + fi + echo + + # Step 2: Create MachineConfig YAML + log_info "Step 2: Creating MachineConfig YAML..." + create_machineconfig_yaml + echo + + # Step 3: Apply MachineConfig + log_info "Step 3: Applying MachineConfig to cluster..." + apply_machineconfig + echo + + # Show current node status + show_node_status + echo + echo "--- Wait 120 seconds for mcp to start updating ---" + sleep 120 + + # Step 4: Wait for workers to update + log_info "Step 4: Waiting for all worker nodes to be updated..." + if wait_for_mcp_update; then + log_success "Worker node update completed successfully!" + else + log_error "Failed to update all worker nodes within timeout" + log_info "You can check the status manually with: oc get mcp worker -w" + exit 1 + fi + echo + echo "--- Wait 60 seconds for PSI to be ready on nodes---" + sleep 60 + + # Step 5: Verify PSI is enabled + log_info "Step 5: Verifying PSI is enabled on all worker nodes..." + sleep 10 # Wait a bit for nodes to stabilize + + if check_all_workers_psi; then + log_success "✅ PSI has been successfully enabled on all worker nodes!" + else + log_error "❌ PSI verification failed on some nodes" + log_info "Please check individual node status manually" + exit 1 + fi + echo + + # Final status + echo "==========================================" + log_success "PSI Enablement Complete!" + echo "==========================================" + log_info "You can verify PSI on any worker node with:" + echo " oc debug node/ -- chroot /host cat /proc/pressure/cpu" + echo + log_info "MachineConfig created: $MACHINECONFIG_FILE" + log_info "To remove this configuration later, run:" + echo " oc delete machineconfig $MACHINECONFIG_NAME" + echo +} + +# Run main function +main "$@" \ No newline at end of file diff --git a/perfscale_regression_ci/scripts/performance/node/psi/run_perf_test.sh b/perfscale_regression_ci/scripts/performance/node/psi/run_perf_test.sh new file mode 100755 index 00000000000..22d64636629 --- /dev/null +++ b/perfscale_regression_ci/scripts/performance/node/psi/run_perf_test.sh @@ -0,0 +1,313 @@ +#!/bin/bash +set -e # Exit immediately if a command exits with a non-zero status. + +# Change to the script's directory to ensure outputs are created locally. +cd "$(dirname "$0")" + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOG_DIR="./perf_logs_${TIMESTAMP}" +mkdir -p "$LOG_DIR" + +# --- Function to generate stress pod definition --- +generate_stress_pod_yaml() { + local stress_type=$1 + local pod_name="${stress_type}-stress-pod" + local command + local args + local resources="" # Default to empty + + case "$stress_type" in + "cpu") + command='["/agnhost"]' + args='["stress", "--cpus", "2"]' + resources=$(printf " resources:\n requests:\n cpu: \"1\"\n limits:\n cpu: \"2\"") + ;; + "memory") + command='["/agnhost"]' + args='["stress", "--mem-total", "8589934592", "--mem-alloc-size", "1073741824"]' + resources=$(printf " resources:\n requests:\n memory: \"8Gi\"\n limits:\n memory: \"9Gi\"") + ;; + "io") + # This command runs an infinite loop to generate sustained I/O pressure. + command='["/bin/sh", "-c"]' + args='["while true; do dd if=/dev/zero of=testfile bs=1M count=128 &>/dev/null; sync; rm testfile &>/dev/null; done"]' + resources=$(printf " resources:\n requests:\n cpu: \"250m\"") + ;; + *) + echo "Invalid stress type: ${stress_type}" + exit 1 + ;; + esac + + cat > "${pod_name}.yaml" < "${pod_name}.yaml" < "test-pod.yaml" < "${LOG_DIR}/${log_prefix}_idle_top_node.log" + kubectl get --raw "/api/v1/nodes/${NODE_NAME}/proxy/stats/summary" > "${LOG_DIR}/${log_prefix}_idle_stats_summary.json" + + # --- Apply Stress Workload --- + echo "Applying ${stress_type} stress workload..." + # YAML is already generated and validated by the smoke test + kubectl apply -f "${pod_yaml}" + kubectl wait --for=condition=Ready pod/"${stress_type}-stress-pod" --timeout=120s + echo "${stress_type} stress workload is ready" + date + + # --- Metric Collection (Under Load) --- + echo "Collecting proxy metrics under load after 180 seconds..." + sleep 180 + kubectl top node "${NODE_NAME}" > "${LOG_DIR}/${log_prefix}_load_top_node.log" + kubectl get --raw "/api/v1/nodes/${NODE_NAME}/proxy/stats/summary" > "${LOG_DIR}/${log_prefix}_load_stats_summary.json" + kubectl get --raw "/api/v1/nodes/${NODE_NAME}/proxy/metrics/cadvisor" > "${LOG_DIR}/${log_prefix}_load_metrics_cadvisor.log" + + # --- Cleanup --- + echo "Cleaning up workload..." + date + kubectl delete -f "${pod_yaml}" --ignore-not-found=true + + echo "--- Finished Test: ${log_prefix} ---" +} + +persist_monitoring() { + set +e + oc get cm -n openshift-monitoring cluster-monitoring-config -o yaml | grep volumeClaimTemplate > /dev/null 2>&1 + if [[ $? -eq 1 ]]; then + echo "--- Configure persistence for monitoring ---" + export PROMETHEUS_RETENTION_PERIOD=20d + export PROMETHEUS_STORAGE_SIZE=50Gi + export ALERTMANAGER_STORAGE_SIZE=2Gi + envsubst < cluster-monitoring-config.yaml | oc apply -f - + echo "--- Sleep 60s to wait for monitoring to take the new config map ---" + sleep 60 + oc rollout status -n openshift-monitoring deploy/cluster-monitoring-operator + oc rollout status -n openshift-monitoring sts/prometheus-k8s + token=$(oc create token -n openshift-monitoring prometheus-k8s --duration=6h) + URL=https://$(oc get route -n openshift-monitoring prometheus-k8s -o jsonpath="{.spec.host}") + prom_status="not_started" + echo "Sleep 30s to wait for prometheus status to become success." + sleep 30 + retry=20 + while [[ "$prom_status" != "success" && $retry -gt 0 ]]; do + retry=$(($retry-1)) + echo "--- Prometheus status is not success yet, retrying in 10s, retries left: $retry ---" + sleep 10 + prom_status=$(curl -s -g -k -X GET -H "Authorization: Bearer $token" -H 'Accept: application/json' -H 'Content-Type: application/json' "$URL/api/v1/query?query=up" | jq -r '.status') + done + if [[ "$prom_status" != "success" ]]; then + prom_status=$(curl -s -g -k -X GET -H "Authorization: Bearer $token" -H 'Accept: application/json' -H 'Content-Type: application/json' "$URL/api/v1/query?query=up" | jq -r '.status') + echo "--- Prometheus status is '$prom_status'. 'success' is expected ---" + exit 1 + else + echo "--- Prometheus is success now. ---" + echo "--- Sleep 5m to wait for nodes to become stable as persis_monitoring may cause monitoring to be relocated ---" + sleep 5m + fi + else + echo "--- Monitoring persistence is already configured ---" + fi + set -e +} + +function install_dittybopper(){ + oc get ns dittybopper + if [[ $? -eq 0 ]]; then + echo "--- Dittybopper is already installed ---" + return + else + echo "--- Install dittybopper ---" + if [[ ! -d performance-dashboards ]]; then + git clone git@github.com:cloud-bulldozer/performance-dashboards.git --depth 1 + fi + cd performance-dashboards/dittybopper + ./deploy.sh -i "$IMPORT_DASHBOARD" + + if [[ $? -eq 0 ]];then + log "info" "dittybopper installed successfully." + else + log "error" "dittybopper install failed." + fi + fi +} + +# --- Main Execution --- + +if [[ "$1" == "--dry-run" ]]; then + echo "--- YAML Generation Dry Run ---" + generate_stress_pod_yaml "cpu" + echo "--- cpu-stress-pod.yaml ---" + cat cpu-stress-pod.yaml + echo + generate_memory_stress_pod_yaml "memory" + echo "--- memory-stress-pod.yaml ---" + cat memory-stress-pod.yaml + echo + generate_stress_pod_yaml "io" + echo "--- io-stress-pod.yaml ---" + cat io-stress-pod.yaml + exit 0 +fi + +persist_monitoring +install_dittybopper + +STRESS_TYPES=("cpu" "memory" "io") + +prepare_test +for stress in "${STRESS_TYPES[@]}"; do + run_test false "${stress}" + echo "--- Sleep 180s to let the previous stress to cool down ---" + sleep 180 +done + +./enable_psi.sh +echo "--- Sleep 10m to let the cluster to become stable after nodes reboot after enabling PSI ---" +sleep 10m + +for stress in "${STRESS_TYPES[@]}"; do + run_test true "${stress}" + echo "--- Sleep 180s to let the previous stress to cool down ---" + sleep 180 +done +cleanup_test + +echo "--- All tests completed. Logs are in ${LOG_DIR} ---" +echo "--- Analyzing results... ---" +python3 analyze_results.py "${LOG_DIR}" \ No newline at end of file