Skip to content

Commit fd78ffe

Browse files
milindlmilindl
authored
Soak tests on Kubernetes (#1821)
soak tests can now run on k8s with different test ids corresponding to different clusters
1 parent 5347020 commit fd78ffe

16 files changed

+488
-120
lines changed

‎tests/soak/Dockerfile

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
FROM debian:bookworm
2+
ARG LK_VERSION
3+
ARG CKPY_VERSION
4+
RUN test -n "${LK_VERSION}" || (echo "LK_VERSION env variable required" && exit 1)
5+
RUN test -n "${CKPY_VERSION}" || (echo "CKPY_VERSION env variable required" && exit 1)
6+
ENV DEBIAN_FRONTEND=noninteractive
7+
RUN apt update && apt install -y sudo
8+
RUN mkdir -p /soaktests
9+
COPY bootstrap.sh /soaktests
10+
WORKDIR /soaktests
11+
RUN /soaktests/bootstrap.sh ${CKPY_VERSION} ${LK_VERSION}
12+
ENTRYPOINT [ "/soaktests/confluent-kafka-python/tests/soak/run.sh" ]

‎tests/soak/README.md

+9-21
Original file line numberDiff line numberDiff line change
@@ -6,26 +6,14 @@ of time, typically 2+ weeks, to vet out any resource leaks, etc.
66
The soak testing client is made up of a producer, producing messages to
77
the configured topic, and a consumer, consuming the same messages back.
88

9-
DataDog reporting supported by setting datadog.api_key a and datadog.app_key
10-
in the soak client configuration file.
9+
OpenTelemetry reporting supported through OTLP.
1110

11+
# Installation
1212

13-
There are some convenience script to get you started.
14-
15-
On the host (ec2) where you aim to run the soaktest, do:
16-
17-
$ git clone https://github.com/confluentinc/librdkafka
18-
$ git clone https://github.com/confluentinc/confluent-kafka-python
19-
20-
# Build librdkafka and python
21-
$ ~/confluent-kafka-python/tests/soak/build.sh <librdkafka-version> <cfl-python-version>
22-
23-
# Set up config:
24-
$ cp ~/confluent-kafka-python/tests/soak/ccloud.config.example ~/confluent-kafka-python/ccloud.config
25-
26-
# Start a screen session
27-
$ screen bash
28-
29-
# Within the screen session, run the soak client
30-
(screen)$ ~/run.sh
31-
(screen)$ Ctrl-A d # to detach
13+
TESTID=normal \
14+
LK_VERSION=v2.2.0 \
15+
CKPY_VERSION=v2.2.0 \
16+
CC_BOOSTRAP_SERVERS=_ \
17+
CC_USERNAME=_ \
18+
CC_PASSWORD=_ \
19+
DOCKER_REPOSITORY=_ ./install.sh

‎tests/soak/ubuntu-bootstrap.sh renamed to ‎tests/soak/bootstrap.sh

+7-10
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,11 @@ fi
1616

1717
python_branch=$1
1818
librdkafka_branch=$2
19+
venv=$PWD/venv
1920

2021
sudo apt update
21-
sudo apt install -y make gcc g++ zlib1g-dev libssl-dev libzstd-dev screen \
22-
python3.6-dev python3-pip python3-virtualenv
23-
24-
pushd $HOME
22+
sudo apt install -y git curl make gcc g++ zlib1g-dev libssl-dev libzstd-dev \
23+
python3-dev python3-pip python3-venv
2524

2625
if [[ ! -d confluent-kafka-python ]]; then
2726
git clone https://github.com/confluentinc/confluent-kafka-python
@@ -33,14 +32,14 @@ git checkout $python_branch
3332

3433
echo "Installing librdkafka $librdkafka_branch"
3534
tools/bootstrap-librdkafka.sh --require-ssl $librdkafka_branch /usr
35+
rm -rf tmp-build
3636

37-
echo "Installing interceptors"
38-
tools/install-interceptors.sh
37+
# echo "Installing interceptors"
38+
# tools/install-interceptors.sh
3939

40-
venv=$HOME/venv
4140
echo "Setting up virtualenv in $venv"
4241
if [[ ! -d $venv ]]; then
43-
virtualenv -p python3.6 $venv
42+
python3 -m venv $venv
4443
fi
4544
source $venv/bin/activate
4645

@@ -57,8 +56,6 @@ python -c "import confluent_kafka; print(confluent_kafka.version(), confluent_ka
5756

5857
deactivate
5958

60-
popd # $HOME
61-
6259
echo "All done, activate the virtualenv in $venv before running the client:"
6360
echo "source $venv/bin/activate"
6461

‎tests/soak/ccloud.config.example

-14
This file was deleted.

‎tests/soak/install.sh

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/bash
2+
set -e
3+
4+
DOCKER_REPOSITORY_DEFAULT=${DOCKER_REPOSITORY:-docker.io/library/njc-py-soak-tests}
5+
NAMESPACE=njc-soak-tests
6+
NOCACHE=${NOCACHE:---no-cache}
7+
8+
for var in LK_VERSION CKPY_VERSION CC_BOOSTRAP_SERVERS CC_USERNAME CC_PASSWORD TESTID \
9+
DOCKER_REPOSITORY_DEFAULT; do
10+
VAR_VALUE=$(eval echo \$$var)
11+
if [ -z "$VAR_VALUE" ]; then
12+
echo "env variable $var is required"
13+
exit 1
14+
fi
15+
done
16+
17+
TAG=${LK_VERSION}-${CKPY_VERSION}
18+
19+
COMMAND="docker build . $NOCACHE --build-arg LK_VERSION=${LK_VERSION} \
20+
--build-arg CKPY_VERSION=${CKPY_VERSION} \
21+
-t ${DOCKER_REPOSITORY_DEFAULT}:${TAG}"
22+
echo $COMMAND
23+
$COMMAND
24+
25+
if [ ! -z "$DOCKER_REPOSITORY" ]; then
26+
COMMAND="docker push ${DOCKER_REPOSITORY}:${TAG}"
27+
echo $COMMAND
28+
$COMMAND
29+
fi
30+
31+
if [ "$(uname -p)" = "x86_64" ]; then
32+
NODE_ARCH="amd64"
33+
else
34+
NODE_ARCH="arm64"
35+
fi
36+
37+
COMMAND="helm upgrade --install njc-py-soak-tests-${TESTID} ./njc-py-soak-tests \
38+
--set "cluster.bootstrapServers=${CC_BOOSTRAP_SERVERS}" \
39+
--set "cluster.username=${CC_USERNAME}" \
40+
--set "cluster.password=${CC_PASSWORD}" \
41+
--set "image.repository=${DOCKER_REPOSITORY_DEFAULT}" \
42+
--set "testid=${TESTID}" \
43+
--set "fullnameOverride=njc-py-soak-tests-${TESTID}" \
44+
--set "image.tag=${TAG}" \
45+
--set "nodeSelector.kubernetes\\.io/arch=${NODE_ARCH}" \
46+
--namespace "${NAMESPACE}" --create-namespace"
47+
echo $COMMAND
48+
$COMMAND

‎tests/soak/njc-py-soak-tests/.helmignore

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Patterns to ignore when building packages.
2+
# This supports shell glob matching, relative path matching, and
3+
# negation (prefixed with !). Only one pattern per line.
4+
.DS_Store
5+
# Common VCS dirs
6+
.git/
7+
.gitignore
8+
.bzr/
9+
.bzrignore
10+
.hg/
11+
.hgignore
12+
.svn/
13+
# Common backup files
14+
*.swp
15+
*.bak
16+
*.tmp
17+
*.orig
18+
*~
19+
# Various IDEs
20+
.project
21+
.idea/
22+
*.tmproj
23+
.vscode/

‎tests/soak/njc-py-soak-tests/Chart.yaml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
apiVersion: v2
2+
name: njc-py-soak-tests
3+
description: A Helm chart for Kubernetes
4+
5+
# A chart can be either an 'application' or a 'library' chart.
6+
#
7+
# Application charts are a collection of templates that can be packaged into versioned archives
8+
# to be deployed.
9+
#
10+
# Library charts provide useful utilities or functions for the chart developer. They're included as
11+
# a dependency of application charts to inject those utilities and functions into the rendering
12+
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
13+
type: application
14+
15+
# This is the chart version. This version number should be incremented each time you make changes
16+
# to the chart and its templates, including the app version.
17+
# Versions are expected to follow Semantic Versioning (https://semver.org/)
18+
version: 0.1.0
19+
20+
# This is the version number of the application being deployed. This version number should be
21+
# incremented each time you make changes to the application. Versions are not expected to
22+
# follow Semantic Versioning. They should reflect the version the application is using.
23+
# It is recommended to use it with quotes.
24+
appVersion: "1.0.0"

‎tests/soak/njc-py-soak-tests/templates/NOTES.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
NJC Python soak tests installed!

‎tests/soak/njc-py-soak-tests/templates/_helpers.tpl

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
{{/*
2+
Expand the name of the chart.
3+
*/}}
4+
{{- define "njc-py-soak-tests.name" -}}
5+
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
6+
{{- end }}
7+
8+
{{/*
9+
Create a default fully qualified app name.
10+
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
11+
If release name contains chart name it will be used as a full name.
12+
*/}}
13+
{{- define "njc-py-soak-tests.fullname" -}}
14+
{{- if .Values.fullnameOverride }}
15+
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
16+
{{- else }}
17+
{{- $name := default .Chart.Name .Values.nameOverride }}
18+
{{- if contains $name .Release.Name }}
19+
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
20+
{{- else }}
21+
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
22+
{{- end }}
23+
{{- end }}
24+
{{- end }}
25+
26+
{{/*
27+
Create chart name and version as used by the chart label.
28+
*/}}
29+
{{- define "njc-py-soak-tests.chart" -}}
30+
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
31+
{{- end }}
32+
33+
{{/*
34+
Common labels
35+
*/}}
36+
{{- define "njc-py-soak-tests.labels" -}}
37+
helm.sh/chart: {{ include "njc-py-soak-tests.chart" . }}
38+
{{ include "njc-py-soak-tests.selectorLabels" . }}
39+
{{- if .Chart.AppVersion }}
40+
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
41+
{{- end }}
42+
app.kubernetes.io/managed-by: {{ .Release.Service }}
43+
{{- end }}
44+
45+
{{/*
46+
Selector labels
47+
*/}}
48+
{{- define "njc-py-soak-tests.selectorLabels" -}}
49+
app.kubernetes.io/name: {{ include "njc-py-soak-tests.name" . }}
50+
app.kubernetes.io/instance: {{ .Release.Name }}
51+
{{- end }}
52+
53+
{{/*
54+
Create the name of the service account to use
55+
*/}}
56+
{{- define "njc-py-soak-tests.serviceAccountName" -}}
57+
{{- if .Values.serviceAccount.create }}
58+
{{- default (include "njc-py-soak-tests.fullname" .) .Values.serviceAccount.name }}
59+
{{- else }}
60+
{{- default "default" .Values.serviceAccount.name }}
61+
{{- end }}
62+
{{- end }}

‎tests/soak/njc-py-soak-tests/templates/deployment.yaml

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: {{ include "njc-py-soak-tests.fullname" . }}
5+
labels:
6+
{{- include "njc-py-soak-tests.labels" . | nindent 4 }}
7+
spec:
8+
replicas: {{ .Values.replicaCount }}
9+
selector:
10+
matchLabels:
11+
{{- include "njc-py-soak-tests.selectorLabels" . | nindent 6 }}
12+
strategy:
13+
type: RollingUpdate
14+
rollingUpdate:
15+
maxSurge: 0
16+
maxUnavailable: 1
17+
template:
18+
metadata:
19+
{{- with .Values.podAnnotations }}
20+
annotations:
21+
{{- toYaml . | nindent 8 }}
22+
{{- end }}
23+
labels:
24+
{{- include "njc-py-soak-tests.selectorLabels" . | nindent 8 }}
25+
spec:
26+
{{- with .Values.imagePullSecrets }}
27+
imagePullSecrets:
28+
{{- toYaml . | nindent 8 }}
29+
{{- end }}
30+
serviceAccountName: {{ include "njc-py-soak-tests.serviceAccountName" . }}
31+
securityContext:
32+
{{- toYaml .Values.podSecurityContext | nindent 8 }}
33+
volumes:
34+
- name: secret
35+
secret:
36+
secretName: {{ include "njc-py-soak-tests.fullname" $ }}-secret
37+
containers:
38+
- name: {{ .Chart.Name }}
39+
securityContext:
40+
{{- toYaml .Values.securityContext | nindent 12 }}
41+
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
42+
imagePullPolicy: {{ .Values.image.pullPolicy }}
43+
env:
44+
- name: NODEIP
45+
valueFrom:
46+
fieldRef:
47+
apiVersion: v1
48+
fieldPath: status.hostIP
49+
- name: OTEL_METRICS_EXPORTER
50+
value: "otlp"
51+
- name: OTEL_RESOURCE_ATTRIBUTES
52+
value: "service.name={{ include "njc-py-soak-tests.fullname" . }},service.version={{ .Values.image.tag | default .Chart.AppVersion }}"
53+
- name: OTEL_TRACES_EXPORTER
54+
value: "none"
55+
- name: OTEL_EXPORTER_OTLP_ENDPOINT
56+
value: "http://$(NODEIP):14317"
57+
- name: OTEL_METRIC_EXPORT_INTERVAL
58+
value: "10000"
59+
- name: TESTID
60+
value: "{{ .Values.testid }}"
61+
volumeMounts:
62+
- name: "secret"
63+
mountPath: "/soaktests/confluent-kafka-python/ccloud.config"
64+
subPath: ccloud.config
65+
readOnly: true
66+
# livenessProbe:
67+
# httpGet:
68+
# path: /
69+
# port: http
70+
# readinessProbe:
71+
# httpGet:
72+
# path: /
73+
# port: http
74+
resources:
75+
{{- toYaml .Values.resources | nindent 12 }}
76+
{{- with .Values.nodeSelector }}
77+
nodeSelector:
78+
{{- toYaml . | nindent 8 }}
79+
{{- end }}
80+
{{- with .Values.affinity }}
81+
affinity:
82+
{{- toYaml . | nindent 8 }}
83+
{{- end }}
84+
{{- with .Values.tolerations }}
85+
tolerations:
86+
{{- toYaml . | nindent 8 }}
87+
{{- end }}

‎tests/soak/njc-py-soak-tests/templates/secrets.yaml

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
{{- range $nameSuffix, $values := .Values.secrets }}
2+
---
3+
apiVersion: v1
4+
kind: Secret
5+
metadata:
6+
name: {{ include "njc-py-soak-tests.fullname" $ }}-{{ $nameSuffix }}
7+
{{- with $values.annotations }}
8+
annotations:
9+
{{- range $key, $value := . }}
10+
{{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }}
11+
{{- end }}
12+
{{- end }}
13+
labels:
14+
{{- range $key, $value := $values.labels }}
15+
{{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }}
16+
{{- end }}
17+
type: {{ default "Opaque" $values.type }}
18+
{{- with $values.data }}
19+
data:
20+
{{- toYaml . | nindent 2 }}
21+
{{- end }}
22+
stringData:
23+
ccloud.config: |-
24+
bootstrap.servers={{ $.Values.cluster.bootstrapServers }}
25+
sasl.mechanisms=PLAIN
26+
security.protocol=SASL_SSL
27+
sasl.username={{ $.Values.cluster.username }}
28+
sasl.password={{ $.Values.cluster.password }}
29+
{{- $.Values.properties | nindent 4 -}}
30+
{{- with $values.stringData }}
31+
{{- range $key, $value := . }}
32+
{{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 2 }}
33+
{{- end }}
34+
{{- end }}
35+
{{- end -}}

0 commit comments

Comments
 (0)