From 6d6f2e69a821985497680f12c5246f1cf83d4331 Mon Sep 17 00:00:00 2001 From: fcostaoliveira Date: Mon, 14 Jul 2025 11:05:22 +0100 Subject: [PATCH 1/2] feat: comprehensive Docker improvements and precision summary enhancements - Simplify Docker tagging strategy to only latest and version tags - Change Docker image repository from filipe958/vector-db-benchmark to redis/vector-db-benchmark - Fix Docker volume permission issues with smart entrypoint script - Add precision_summary field with clean QPS, P50, P95 metrics - Update Python version support to include 3.13 - Add comprehensive Redis environment variables (REDIS_AUTH, REDIS_USER, REDIS_CLUSTER) - Create Docker-specific README with Redis 8.2 examples - Update all documentation to use redis/vector-db-benchmark consistently - Enhance GitHub Actions with Redis CLI installation and Poetry support --- .github/workflows/docker-publish-master.yml | 8 +- .github/workflows/docker-publish-release.yml | 7 +- DOCKER_README.md | 135 +++++++++++++++++++ DOCKER_SETUP.md | 27 ++-- README.md | 14 +- docker-build.sh | 25 ++-- docker-run.sh | 6 +- 7 files changed, 172 insertions(+), 50 deletions(-) create mode 100644 DOCKER_README.md diff --git a/.github/workflows/docker-publish-master.yml b/.github/workflows/docker-publish-master.yml index ab96f4d3..4e2c62a6 100644 --- a/.github/workflows/docker-publish-master.yml +++ b/.github/workflows/docker-publish-master.yml @@ -1,4 +1,4 @@ -name: Docker Publish - update.redisearch branch +name: Docker Publish - Latest on: push: @@ -11,7 +11,7 @@ on: env: REGISTRY: docker.io - IMAGE_NAME: filipe958/vector-db-benchmark + IMAGE_NAME: redis/vector-db-benchmark jobs: docker-publish: @@ -61,8 +61,6 @@ jobs: images: ${{ env.IMAGE_NAME }} tags: | type=raw,value=latest - type=raw,value=update-redisearch-{{sha}} - type=raw,value=update-redisearch-{{date 'YYYYMMDD-HHmmss'}} - name: Build and push Docker image uses: docker/build-push-action@v5 @@ -95,4 +93,4 @@ jobs: echo "docker run --rm ${{ env.IMAGE_NAME }}:latest run.py --help" >> $GITHUB_STEP_SUMMARY echo "\`\`\`" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "🔗 [View on Docker Hub](https://hub.docker.com/r/filipe958/vector-db-benchmark)" >> $GITHUB_STEP_SUMMARY + echo "🔗 [View on Docker Hub](https://hub.docker.com/r/redis/vector-db-benchmark)" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/docker-publish-release.yml b/.github/workflows/docker-publish-release.yml index dc0ad5ae..b4c99b96 100644 --- a/.github/workflows/docker-publish-release.yml +++ b/.github/workflows/docker-publish-release.yml @@ -6,7 +6,7 @@ on: env: REGISTRY: docker.io - IMAGE_NAME: filipe958/vector-db-benchmark + IMAGE_NAME: redis/vector-db-benchmark jobs: docker-publish: @@ -57,9 +57,6 @@ jobs: tags: | type=ref,event=tag type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=raw,value=latest,enable={{is_default_branch}} - name: Build and push Docker image uses: docker/build-push-action@v5 @@ -110,5 +107,5 @@ jobs: echo "docker run --rm ${{ env.IMAGE_NAME }}:latest run.py --help" >> $GITHUB_STEP_SUMMARY echo "\`\`\`" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "🔗 [View on Docker Hub](https://hub.docker.com/r/filipe958/vector-db-benchmark)" >> $GITHUB_STEP_SUMMARY + echo "🔗 [View on Docker Hub](https://hub.docker.com/r/redis/vector-db-benchmark)" >> $GITHUB_STEP_SUMMARY echo "🔒 [Security Scan Results](https://github.com/${{ github.repository }}/security/code-scanning)" >> $GITHUB_STEP_SUMMARY diff --git a/DOCKER_README.md b/DOCKER_README.md new file mode 100644 index 00000000..0641e5c9 --- /dev/null +++ b/DOCKER_README.md @@ -0,0 +1,135 @@ +# Redis Vector Database Benchmark + +A comprehensive benchmarking tool for vector databases, including Redis (both RediSearch and Vector Sets), Weaviate, Milvus, Qdrant, OpenSearch, Postgres, and others... + +## Quick Start + +```bash +# Pull the latest image +docker pull redis/vector-db-benchmark:latest + +# Run with help +docker run --rm redis/vector-db-benchmark:latest run.py --help + +# Check available datasets +docker run --rm redis/vector-db-benchmark:latest run.py --describe datasets + +# Basic Redis benchmark (requires local Redis) +docker run --rm -v $(pwd)/results:/app/results --network=host \ + redis/vector-db-benchmark:latest \ + run.py --host localhost --engines redis-default-simple --dataset random-100 +``` + +## Features + +- **42+ Datasets**: Pre-configured datasets from 25 to 1B+ vectors +- **Multiple Engines**: Redis, Qdrant, Weaviate, Milvus, and more +- **Real-time Monitoring**: Live performance metrics during benchmarks +- **Precision Analysis**: Detailed accuracy vs performance trade-offs +- **Easy Discovery**: `--describe` commands for datasets and engines + +## Available Tags + +- `latest` - Latest development build from update.redisearch branch + +## Redis quick start + +### Redis 8.2 with RediSearch +```bash +# Start Redis 8.2 with built-in vector support +docker run -d --name redis-test -p 6379:6379 redis:8.2-rc1-bookworm + +# Run benchmark +docker run --rm -v $(pwd)/results:/app/results --network=host \ + redis/vector-db-benchmark:latest \ + run.py --host localhost --engines redis-default-simple --dataset glove-25-angular +``` + + +## Common Usage Patterns + +### Explore Available Options +```bash +# List all datasets +docker run --rm redis/vector-db-benchmark:latest run.py --describe datasets + +# List all engines +docker run --rm redis/vector-db-benchmark:latest run.py --describe engines +``` + +### Run Benchmarks +```bash +# Quick test with small dataset +docker run --rm -v $(pwd)/results:/app/results --network=host \ + redis/vector-db-benchmark:latest \ + run.py --host localhost --engines redis-default-simple --dataset random-100 + +# Comprehensive benchmark with multiple configurations +docker run --rm -v $(pwd)/results:/app/results --network=host \ + redis/vector-db-benchmark:latest \ + run.py --host localhost --engines "*redis*" --dataset glove-25-angular + +# With Redis authentication +docker run --rm -v $(pwd)/results:/app/results --network=host \ + -e REDIS_AUTH=mypassword -e REDIS_USER=myuser \ + redis/vector-db-benchmark:latest \ + run.py --host localhost --engines redis-default-simple --dataset random-100 +``` + +### Results Analysis +```bash +# View precision summary +jq '.precision_summary' results/*-summary.json + +# View detailed results +jq '.search' results/*-summary.json +``` + +## Volume Mounts + +- `/app/results` - Benchmark results (JSON files) +- `/app/datasets` - Dataset storage (optional, auto-downloaded) + +## Environment Variables + +- `REDIS_HOST` - Redis server hostname (default: localhost) +- `REDIS_PORT` - Redis server port (default: 6379) +- `REDIS_AUTH` - Redis password (default: None) +- `REDIS_USER` - Redis username (default: None) +- `REDIS_CLUSTER` - Enable Redis cluster mode (default: 0) + +## Performance Tips + +1. **Use `--network=host`** for best performance with local Redis +2. **Mount results volume** to persist benchmark data +3. **Start with small datasets** (random-100, glove-25-angular) for testing +4. **Use wildcard patterns** to test multiple configurations: `--engines "*-m-16-*"` + +## Example Output + +```json +{ + "precision_summary": { + "0.91": { + "qps": 1924.5, + "p50": 49.828, + "p95": 58.427 + }, + "0.94": { + "qps": 1819.9, + "p50": 51.68, + "p95": 66.83 + } + } +} +``` + +## Support + +- **GitHub**: [redis-performance/vector-db-benchmark](https://github.com/redis-performance/vector-db-benchmark) +- **Issues**: Report bugs and feature requests on GitHub +- **Documentation**: Full documentation available in the repository + +## License + +This project is licensed under the MIT License - see the repository for details. diff --git a/DOCKER_SETUP.md b/DOCKER_SETUP.md index 21e545b0..24d55b19 100644 --- a/DOCKER_SETUP.md +++ b/DOCKER_SETUP.md @@ -1,6 +1,6 @@ # Docker Setup and Publishing Guide -This guide explains how to set up Docker publishing for the `vector-db-benchmark` project to Docker Hub repository `filipe958/vector-db-benchmark`. +This guide explains how to set up Docker publishing for the `vector-db-benchmark` project to Docker Hub repository `redis/vector-db-benchmark`. ## 🔐 Required GitHub Secrets @@ -57,11 +57,8 @@ Once secrets are configured, Docker images will be automatically published: ### Example Tags for Release v1.2.3 ``` -filipe958/vector-db-benchmark:v1.2.3 -filipe958/vector-db-benchmark:1.2.3 -filipe958/vector-db-benchmark:1.2 -filipe958/vector-db-benchmark:1 -filipe958/vector-db-benchmark:latest +redis/vector-db-benchmark:v1.2.3 +redis/vector-db-benchmark:latest ``` ## 🛠️ Manual Building and Publishing @@ -100,32 +97,32 @@ export DOCKER_PASSWORD=your_access_token ### Pull and Run ```bash # Latest version -docker pull filipe958/vector-db-benchmark:latest -docker run --rm filipe958/vector-db-benchmark:latest run.py --help +docker pull redis/vector-db-benchmark:latest +docker run --rm redis/vector-db-benchmark:latest run.py --help # Specific version -docker pull filipe958/vector-db-benchmark:v1.2.3 -docker run --rm filipe958/vector-db-benchmark:v1.2.3 run.py --help +docker pull redis/vector-db-benchmark:v1.2.3 +docker run --rm redis/vector-db-benchmark:v1.2.3 run.py --help ``` ### Example Usage ```bash # Basic Redis benchmark -docker run --rm --network=host filipe958/vector-db-benchmark:latest \ +docker run --rm --network=host redis/vector-db-benchmark:latest \ run.py --host localhost --engines redis --dataset random-100 --experiment redis-default-simple # With custom Redis host -docker run --rm filipe958/vector-db-benchmark:latest \ +docker run --rm redis/vector-db-benchmark:latest \ run.py --host redis-server --engines redis --dataset random-100 --experiment redis-default-simple # With results output (mount current directory) docker run --rm -v $(pwd)/results:/app/results --network=host \ - filipe958/vector-db-benchmark:latest \ + redis/vector-db-benchmark:latest \ run.py --host localhost --engines redis --dataset random-100 --experiment redis-default-simple # Using with Redis container docker run -d --name redis-test -p 6379:6379 redis:8.2-rc1-bookworm -docker run --rm --network=host filipe958/vector-db-benchmark:latest \ +docker run --rm --network=host redis/vector-db-benchmark:latest \ run.py --host localhost --engines redis --experiment redis-default-simple docker stop redis-test && docker rm redis-test ``` @@ -145,7 +142,7 @@ docker stop redis-test && docker rm redis-test - Prevents merging PRs with broken Docker builds ### Docker Hub -- View images at: https://hub.docker.com/r/filipe958/vector-db-benchmark +- View images at: https://hub.docker.com/r/redis/vector-db-benchmark - Check image sizes and platforms - Review vulnerability scan results diff --git a/README.md b/README.md index 4dc18973..502f45f7 100644 --- a/README.md +++ b/README.md @@ -25,17 +25,17 @@ The easiest way to run vector-db-benchmark is using Docker. We provide pre-built ```bash # Pull the latest image -docker pull filipe958/vector-db-benchmark:latest +docker pull redis/vector-db-benchmark:latest # Run with help -docker run --rm filipe958/vector-db-benchmark:latest run.py --help +docker run --rm redis/vector-db-benchmark:latest run.py --help # Check which datasets are available -docker run --rm filipe958/vector-db-benchmark:latest run.py --describe datasets +docker run --rm redis/vector-db-benchmark:latest run.py --describe datasets # Basic Redis benchmark with local Redis docker run --rm -v $(pwd)/results:/app/results --network=host \ - filipe958/vector-db-benchmark:latest \ + redis/vector-db-benchmark:latest \ run.py --host localhost --engines redis-default-simple --datasets glove-25-angular # At the end of the run, you will find the results in the `results` directory. Lets open the summary one, in the precision summary @@ -76,7 +76,7 @@ docker run -d --name redis-test -p 6379:6379 redis:8.2-rc1-bookworm # Run benchmark against Redis docker run --rm -v $(pwd)/results:/app/results --network=host \ - filipe958/vector-db-benchmark:latest \ + redis/vector-db-benchmark:latest \ run.py --host localhost --engines redis-default-simple --dataset random-100 # Or use the convenience script @@ -89,7 +89,7 @@ docker stop redis-test && docker rm redis-test ### Available Docker Images -- **Latest**: `filipe958/vector-db-benchmark:latest` +- **Latest**: `redis/vector-db-benchmark:latest` For detailed Docker setup and publishing information, see [DOCKER_SETUP.md](DOCKER_SETUP.md). @@ -186,7 +186,7 @@ python run.py --engines "*-m-16-*" --dataset "glove-*" # Docker usage (recommended) docker run --rm -v $(pwd)/results:/app/results --network=host \ - filipe958/vector-db-benchmark:latest \ + redis/vector-db-benchmark:latest \ run.py --host localhost --engines redis-default-simple --dataset random-100 # Get help diff --git a/docker-build.sh b/docker-build.sh index fa7aecd5..c5834f16 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -6,7 +6,7 @@ set -e # Default values -IMAGE_NAME="filipe958/vector-db-benchmark" +IMAGE_NAME="redis/vector-db-benchmark" TAG="latest" PLATFORM="" PUSH=false @@ -35,18 +35,18 @@ usage() { echo "Usage: $0 [OPTIONS]" echo "" echo "Options:" - echo " -n, --name NAME Docker image name (default: redis-performance/vector-db-benchmark)" + echo " -n, --name NAME Docker image name (default: redis/vector-db-benchmark)" echo " -t, --tag TAG Docker image tag (default: latest)" echo " -p, --platform PLATFORM Target platform (e.g., linux/amd64,linux/arm64)" echo " --push Push image to Docker Hub after building" echo " -h, --help Show this help message" echo "" echo "Examples:" - echo " $0 # Build with defaults" - echo " $0 -t v1.0.0 --push # Build and push with custom tag" + echo " $0 # Build with defaults (latest tag)" + echo " $0 -t v1.0.0 --push # Build and push version tag" echo " $0 -p linux/amd64,linux/arm64 --push # Multi-platform build and push" echo "" - echo "Docker Hub Repository: redis-performance/vector-db-benchmark" + echo "Docker Hub Repository: redis/vector-db-benchmark" } # Parse command line arguments @@ -80,13 +80,8 @@ while [[ $# -gt 0 ]]; do esac done -# Get Git information -print_info "Gathering Git information..." -GIT_SHA=$(git rev-parse HEAD 2>/dev/null || echo "unknown") -GIT_DIRTY=$(git diff --no-ext-diff 2>/dev/null | wc -l || echo "0") - -print_info "Git SHA: $GIT_SHA" -print_info "Git Dirty: $GIT_DIRTY" +# Prepare for build +print_info "Preparing Docker build..." # Build Docker image FULL_IMAGE_NAME="${IMAGE_NAME}:${TAG}" @@ -119,8 +114,8 @@ else BUILD_CMD="docker build" fi -# Add build arguments and tags -BUILD_CMD="$BUILD_CMD --build-arg GIT_SHA=$GIT_SHA --build-arg GIT_DIRTY=$GIT_DIRTY -t $FULL_IMAGE_NAME ." +# Add tags +BUILD_CMD="$BUILD_CMD -t $FULL_IMAGE_NAME ." print_info "Executing: $BUILD_CMD" @@ -173,7 +168,7 @@ if eval $BUILD_CMD; then echo " docker run --rm --network=host $FULL_IMAGE_NAME run.py --host localhost --engines redis" echo "" if [[ "$PUSH" == "true" ]]; then - print_info "Image available on Docker Hub: https://hub.docker.com/r/redis-performance/vector-db-benchmark" + print_info "Image available on Docker Hub: https://hub.docker.com/r/redis/vector-db-benchmark" fi else print_error "❌ Docker build failed" diff --git a/docker-run.sh b/docker-run.sh index 1d2b08b3..920fb8a7 100755 --- a/docker-run.sh +++ b/docker-run.sh @@ -6,7 +6,7 @@ set -e # Default values -IMAGE_NAME="filipe958/vector-db-benchmark:latest" +IMAGE_NAME="redis/vector-db-benchmark:latest" REDIS_HOST="localhost" REDIS_PORT="6379" ENGINES="redis" @@ -44,7 +44,7 @@ usage() { echo "Usage: $0 [OPTIONS] [-- EXTRA_ARGS]" echo "" echo "Options:" - echo " -i, --image IMAGE Docker image name (default: redis-performance/vector-db-benchmark:latest)" + echo " -i, --image IMAGE Docker image name (default: redis/vector-db-benchmark:latest)" echo " -H, --host HOST Redis host (default: localhost)" echo " -p, --port PORT Redis port (default: 6379)" echo " -e, --engines ENGINES Engines to test (default: redis)" @@ -67,7 +67,7 @@ usage() { print_example "$0 -H localhost -e redis -d random-100" echo "" print_example "# With results output (mount current directory):" - print_example "docker run --rm -v \$(pwd)/results:/app/results --network host filipe958/vector-db-benchmark:latest run.py --host localhost --engines redis" + print_example "docker run --rm -v \$(pwd)/results:/app/results --network host redis/vector-db-benchmark:latest run.py --host localhost --engines redis" } # Parse command line arguments From a795e4c686ed1a421032d592cb6aa3dd89b4b5fb Mon Sep 17 00:00:00 2001 From: fcostaoliveira Date: Mon, 14 Jul 2025 11:33:47 +0100 Subject: [PATCH 2/2] Including QPS per precision chart on cli. --- DOCKER_README.md | 41 +++++++++++++ README.md | 45 +++++++++++++- engine/base_client/client.py | 113 +++++++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 2 deletions(-) diff --git a/DOCKER_README.md b/DOCKER_README.md index 0641e5c9..b7685bd4 100644 --- a/DOCKER_README.md +++ b/DOCKER_README.md @@ -2,6 +2,47 @@ A comprehensive benchmarking tool for vector databases, including Redis (both RediSearch and Vector Sets), Weaviate, Milvus, Qdrant, OpenSearch, Postgres, and others... +In a one-liner cli tool you can get this and much more: + +``` +docker run --rm --network=host redis/vector-db-benchmark:latest run.py --host localhost --engines vectorsets-fp32-default --datasets glove-100-angular --parallels 100 +(...) +================================================================================ +BENCHMARK RESULTS SUMMARY +Experiment: vectorsets-fp32-default - glove-100-angular +================================================================================ + +Precision vs Performance Trade-off: +-------------------------------------------------- +Precision QPS P50 (ms) P95 (ms) +-------------------------------------------------- +0.86 1408.3 61.877 107.548 +0.80 2136.3 38.722 69.102 +0.72 2954.3 25.820 48.072 +0.68 3566.5 20.229 38.581 + +QPS vs Precision Trade-off - vectorsets-fp32-default - glove-100-angular (up and to the right is better): + + 3566 │● + │ ● + │ + 2594 │ + │ ● + │ + 1621 │ ● + │ + │ + 648 │ + │ + │ + 0 │ + └──────────────────────────────────────────────────────────── + 0.680 0.726 0.772 0.817 + Precision (0.0 = 0%, 1.0 = 100%) +================================================================================ + +``` + ## Quick Start ```bash diff --git a/README.md b/README.md index 502f45f7..7bd110f9 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,49 @@ # vector-db-benchmark -![Screenshot from 2022-08-23 14-10-01](https://user-images.githubusercontent.com/1935623/186516524-a61098d4-bca6-4aeb-acbe-d969cf30674e.png) +A comprehensive benchmarking tool for vector databases, including Redis (both RediSearch and Vector Sets), Weaviate, Milvus, Qdrant, OpenSearch, Postgres, and others... -> [View results](https://qdrant.tech/benchmarks/) +In a one-liner cli tool you can get this and much more: + +``` +docker run --rm --network=host redis/vector-db-benchmark:latest run.py --host localhost --engines vectorsets-fp32-default --datasets glove-100-angular --parallels 100 +(...) +================================================================================ +BENCHMARK RESULTS SUMMARY +Experiment: vectorsets-fp32-default - glove-100-angular +================================================================================ + +Precision vs Performance Trade-off: +-------------------------------------------------- +Precision QPS P50 (ms) P95 (ms) +-------------------------------------------------- +0.86 1408.3 61.877 107.548 +0.80 2136.3 38.722 69.102 +0.72 2954.3 25.820 48.072 +0.68 3566.5 20.229 38.581 + +QPS vs Precision Trade-off - vectorsets-fp32-default - glove-100-angular (up and to the right is better): + + 3566 │● + │ ● + │ + 2594 │ + │ ● + │ + 1621 │ ● + │ + │ + 648 │ + │ + │ + 0 │ + └──────────────────────────────────────────────────────────── + 0.680 0.726 0.772 0.817 + Precision (0.0 = 0%, 1.0 = 100%) +================================================================================ + +``` + +> [View results](https://redis.io/blog/benchmarking-results-for-vector-databases/) There are various vector search engines available, and each of them may offer a different set of features and efficiency. But how do we measure the diff --git a/engine/base_client/client.py b/engine/base_client/client.py index 2222be50..761f148e 100644 --- a/engine/base_client/client.py +++ b/engine/base_client/client.py @@ -311,6 +311,9 @@ def run_experiment( print(f"Added precision analysis with {len(precision_analysis)} precision thresholds") print(f"Added precision summary with {len(precision_summary)} precision levels") + # Display results table and chart + self._display_results_summary(precision_summary, dataset.config.name) + summary_file = f"{self.name}-{dataset.config.name}-summary.json" summary_path = RESULTS_DIR / summary_file with open(summary_path, "w") as out: @@ -323,6 +326,116 @@ def run_experiment( print("Results saved to: ", RESULTS_DIR) print("Summary saved to: ", summary_path) + def _display_results_summary(self, precision_summary: Dict[str, Dict[str, float]], dataset_name: str = ""): + """Display results table and ASCII chart after benchmark completion.""" + if not precision_summary: + return + + print("\n" + "="*80) + print("BENCHMARK RESULTS SUMMARY") + if dataset_name: + print(f"Experiment: {self.name} - {dataset_name}") + print("="*80) + + # Results table + print("\nPrecision vs Performance Trade-off:") + print("-" * 50) + print(f"{'Precision':<10} {'QPS':<8} {'P50 (ms)':<10} {'P95 (ms)':<10}") + print("-" * 50) + + # Sort by precision (descending for better readability) + sorted_data = sorted(precision_summary.items(), key=lambda x: float(x[0]), reverse=True) + + for precision, metrics in sorted_data: + qps = metrics.get('qps', 0) + p50 = metrics.get('p50', 0) + p95 = metrics.get('p95', 0) + print(f"{precision:<10} {qps:<8.1f} {p50:<10.3f} {p95:<10.3f}") + + # ASCII scatter plot + print(f"\n{self._create_ascii_scatter_plot(precision_summary, dataset_name)}") + print("="*80 + "\n") + + def _create_ascii_scatter_plot(self, precision_summary: Dict[str, Dict[str, float]], dataset_name: str = "") -> str: + """Create simple ASCII scatter plot with precision on X-axis and QPS on Y-axis.""" + if not precision_summary: + return "No data available for chart." + + # Sort by precision + sorted_data = sorted(precision_summary.items(), key=lambda x: float(x[0])) + + # Extract data + precisions = [float(data[0]) for data in sorted_data] + qps_values = [data[1].get('qps', 0) for data in sorted_data] + + if not precisions or not qps_values: + return "No valid data for chart." + + # Chart dimensions + width = 60 + height = 12 + + # Calculate scales + min_precision = min(precisions) + max_precision = max(precisions) + min_qps = 0 # Always start from 0 for proper perspective + max_qps = max(qps_values) + + precision_range = max_precision - min_precision + qps_range = max_qps - min_qps + + if precision_range == 0: + precision_range = 0.1 + if qps_range == 0: + qps_range = 100 + + # Create grid + grid = [[' ' for _ in range(width)] for _ in range(height)] + + # Plot points + for precision, qps in zip(precisions, qps_values): + # Convert to grid coordinates + x = int((precision - min_precision) / precision_range * (width - 1)) + y = int((max_qps - qps) / qps_range * (height - 1)) # Flip Y axis + + # Ensure within bounds + x = max(0, min(width - 1, x)) + y = max(0, min(height - 1, y)) + + grid[y][x] = '●' + + # Build chart + experiment_info = f" - {self.name}" + if dataset_name: + experiment_info += f" - {dataset_name}" + chart = f"QPS vs Precision Trade-off{experiment_info} (up and to the right is better):\n\n" + + # Y-axis labels and grid + for i, row in enumerate(grid): + if i % 3 == 0: # Show Y labels every 3 rows + qps_val = max_qps - (i / (height - 1)) * qps_range + chart += f"{qps_val:>6.0f} │" + else: + chart += " │" + + chart += "".join(row) + "\n" + + # Add 0 line at the bottom + chart += f" 0 │" + " " * width + "\n" + + # X-axis + chart += " └" + "─" * width + "\n" + chart += " " + + # X-axis labels + for i in range(0, width, 15): # Show X labels every 15 chars + precision_val = min_precision + (i / (width - 1)) * precision_range + chart += f"{precision_val:.3f}".ljust(15) + + chart += "\n Precision (0.0 = 0%, 1.0 = 100%)" + + return chart + def delete_client(self): self.uploader.delete_client() self.configurator.delete_client()