Skip to content

Run benchmarks with feature flags for dev #237

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 10, 2025
Merged

Run benchmarks with feature flags for dev #237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7c4c6be
Run benchmarks with QRANT__FEATURE_FLAGS__ALL=true for dev only
tellet-q Apr 9, 2025
b6e080b
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/continuous-benchmark-2.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,12 @@ jobs:

# Benchmark the dev branch:
export QDRANT_VERSION=ghcr/dev
export QDRANT__FEATURE_FLAGS__ALL=true
timeout 30m bash -x tools/run_ci.sh

# Benchmark the master branch:
export QDRANT_VERSION=docker/master
export QDRANT__FEATURE_FLAGS__ALL=false
timeout 30m bash -x tools/run_ci.sh
done

Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/continuous-benchmark.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,12 @@ jobs:

# Benchmark the dev branch:
export QDRANT_VERSION=ghcr/dev
export QDRANT__FEATURE_FLAGS__ALL=true
timeout 30m bash -x tools/run_ci.sh

# Benchmark the master branch:
export QDRANT_VERSION=docker/master
export QDRANT__FEATURE_FLAGS__ALL=false
timeout 30m bash -x tools/run_ci.sh
done

Expand Down Expand Up @@ -140,10 +142,12 @@ jobs:

# Benchmark the dev branch:
export QDRANT_VERSION=ghcr/dev
export QDRANT__FEATURE_FLAGS__ALL=true
timeout 30m bash -x tools/run_ci.sh

# Benchmark the master branch:
export QDRANT_VERSION=docker/master
export QDRANT__FEATURE_FLAGS__ALL=false
timeout 30m bash -x tools/run_ci.sh

set -e
Expand Down Expand Up @@ -229,10 +233,12 @@ jobs:

# Benchmark the dev branch:
export QDRANT_VERSION=ghcr/dev
export QDRANT__FEATURE_FLAGS__ALL=true
timeout 30m bash -x tools/run_ci.sh

# Benchmark the master branch:
export QDRANT_VERSION=docker/master
export QDRANT__FEATURE_FLAGS__ALL=false
timeout 30m bash -x tools/run_ci.sh

set -e
Expand Down
2 changes: 2 additions & 0 deletions engine/servers/qdrant-continuous-benchmarks/docker-compose.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,5 @@ services:
resources:
limits:
memory: 25Gb
environment:
- QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL:-false}
3 changes: 2 additions & 1 deletion tools/run_server_container.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ SCRIPT_PATH=$(dirname "$SCRIPT")
BENCH_SERVER_NAME=${SERVER_NAME:-"benchmark-server-1"}

QDRANT_VERSION=${QDRANT_VERSION:-"dev"}
QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL:-"false"}

IP_OF_THE_SERVER=$(bash "${SCRIPT_PATH}/${CLOUD_NAME}/get_public_ip.sh" "$BENCH_SERVER_NAME")

Expand All @@ -34,7 +35,7 @@ if [[ ${QDRANT_VERSION} == docker/* ]] || [[ ${QDRANT_VERSION} == ghcr/* ]]; the
CONTAINER_REGISTRY='ghcr.io'
fi

DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true ; docker compose up -d; docker container ls -a"
DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; export QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true ; docker compose up -d; docker container ls -a"
ssh -t -o ServerAliveInterval=60 -o ServerAliveCountMax=3 "${SERVER_USERNAME}@${IP_OF_THE_SERVER}" "cd ./projects/vector-db-benchmark/engine/servers/${CONTAINER_NAME} ; $DOCKER_COMPOSE"
else
echo "Error: unknown version ${QDRANT_VERSION}. Version name should start with 'docker/' or 'ghcr/'"
Expand Down
5 changes: 3 additions & 2 deletions tools/run_server_container_with_volume.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ SCRIPT_PATH=$(dirname "$SCRIPT")
BENCH_SERVER_NAME=${SERVER_NAME:-"benchmark-server-1"}

QDRANT_VERSION=${QDRANT_VERSION:-"dev"}
QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL:-"false"}

IP_OF_THE_SERVER=$(bash "${SCRIPT_PATH}/${CLOUD_NAME}/get_public_ip.sh" "$BENCH_SERVER_NAME")

Expand All @@ -37,11 +38,11 @@ if [[ ${QDRANT_VERSION} == docker/* ]] || [[ ${QDRANT_VERSION} == ghcr/* ]]; the

if [[ "$EXECUTION_MODE" == "init" ]]; then
echo "Initialize qdrant from scratch, with qdrant_storage volume"
DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; export CONTAINER_MEM_LIMIT=${CONTAINER_MEM_LIMIT}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true; docker volume rm -f qdrant_storage || true; docker compose up -d; docker container ls -a"
DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; export CONTAINER_MEM_LIMIT=${CONTAINER_MEM_LIMIT}; export QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true; docker volume rm -f qdrant_storage || true; docker compose up -d; docker container ls -a"
elif [[ "$EXECUTION_MODE" == "continue" ]]; then
# suggest that volume qdrant_storage exist and start qdrant
echo "Reload qdrant with existing data"
DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; export CONTAINER_MEM_LIMIT=${CONTAINER_MEM_LIMIT}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true ; sudo bash -c 'sync; echo 1 > /proc/sys/vm/drop_caches'; docker compose up -d; docker container ls -a"
DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; export CONTAINER_MEM_LIMIT=${CONTAINER_MEM_LIMIT}; export QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true ; sudo bash -c 'sync; echo 1 > /proc/sys/vm/drop_caches'; docker compose up -d; docker container ls -a"
else
echo "Error: unknown execution mode ${EXECUTION_MODE}. Execution mode should be 'init' or 'continue'"
exit 1
Expand Down