Skip to content

Commit 23948fd

Browse files
tisniktisnik
authored
Merge pull request #652 from are-ces/rhaiis-support
LCORE-378: Lightspeed core needs to fully support Red Hat AI Inference server LLM provider
2 parents 9285c1b + 46a68f5 commit 23948fd

File tree

6 files changed

+333
-8
lines changed

6 files changed

+333
-8
lines changed

‎.github/workflows/e2e_tests_rhaiis.yaml‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
# .github/workflows/e2e_tests_rhaiis.yaml
2+
name: RHAIIS E2E Tests
3+
4+
on:
5+
schedule:
6+
- cron: "0 0 * * *" # Runs once a day at midnight UTC
7+
workflow_dispatch:
8+
9+
10+
jobs:
11+
e2e_tests:
12+
runs-on: ubuntu-latest
13+
strategy:
14+
matrix:
15+
environment: [ "rhaiis" ]
16+
env:
17+
RHAIIS_URL: ${{ secrets.RHAIIS_URL }}
18+
RHAIIS_API_KEY: ${{ secrets.RHAIIS_API_KEY }}
19+
20+
steps:
21+
- uses: actions/checkout@v4
22+
with:
23+
# On PR_TARGET → the fork (or same repo) that opened the PR.
24+
# On push → falls back to the current repository.
25+
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
26+
27+
# On PR_TARGET → the PR head *commit* (reproducible).
28+
# On push → the pushed commit that triggered the workflow.
29+
ref: ${{ github.event.pull_request.head.ref || github.sha }}
30+
31+
# Don’t keep credentials when running untrusted PR code under PR_TARGET.
32+
persist-credentials: ${{ github.event_name != 'pull_request_target' }}
33+
34+
- name: Verify actual git checkout result
35+
run: |
36+
echo "=== Git Status After Checkout ==="
37+
echo "Remote URLs:"
38+
git remote -v
39+
echo ""
40+
echo "Current branch: $(git branch --show-current 2>/dev/null || echo 'detached HEAD')"
41+
echo "Current commit: $(git rev-parse HEAD)"
42+
echo "Current commit message: $(git log -1 --oneline)"
43+
echo ""
44+
echo "=== Recent commits (should show setup-metrics commits) ==="
45+
git log --oneline -5
46+
47+
- uses: 1arp/[email protected]
48+
with:
49+
path: '.'
50+
isAbsolutePath: false
51+
file: 'lightspeed-stack.yaml'
52+
content: |
53+
name: Lightspeed Core Service (LCS)
54+
service:
55+
host: 0.0.0.0
56+
port: 8080
57+
auth_enabled: false
58+
workers: 1
59+
color_log: true
60+
access_log: true
61+
llama_stack:
62+
# Uses a remote llama-stack service
63+
# The instance would have already been started with a llama-stack-run.yaml file
64+
use_as_library_client: false
65+
# Alternative for "as library use"
66+
# use_as_library_client: true
67+
# library_client_config_path: <path-to-llama-stack-run.yaml-file>
68+
url: http://llama-stack:8321
69+
api_key: xyzzy
70+
user_data_collection:
71+
feedback_enabled: true
72+
feedback_storage: "/tmp/data/feedback"
73+
transcripts_enabled: true
74+
transcripts_storage: "/tmp/data/transcripts"
75+
76+
authentication:
77+
module: "noop"
78+
79+
- name: Select and configure run.yaml
80+
env:
81+
CONFIG_ENVIRONMENT: ${{ matrix.environment || 'rhaiis' }}
82+
run: |
83+
CONFIGS_DIR="tests/e2e/configs"
84+
ENVIRONMENT="$CONFIG_ENVIRONMENT"
85+
86+
echo "Looking for configurations in $CONFIGS_DIR/"
87+
88+
# List available configurations
89+
if [ -d "$CONFIGS_DIR" ]; then
90+
echo "Available configurations:"
91+
ls -la "$CONFIGS_DIR"/*.yaml 2>/dev/null || echo "No YAML files found in $CONFIGS_DIR/"
92+
else
93+
echo "Configs directory '$CONFIGS_DIR' not found!"
94+
exit 1
95+
fi
96+
97+
# Determine which config file to use
98+
CONFIG_FILE="$CONFIGS_DIR/run-$ENVIRONMENT.yaml"
99+
100+
echo "Looking for: $CONFIG_FILE"
101+
102+
if [ -f "$CONFIG_FILE" ]; then
103+
echo "Found config for environment: $ENVIRONMENT"
104+
cp "$CONFIG_FILE" run.yaml
105+
else
106+
echo "Configuration file not found: $CONFIG_FILE"
107+
echo "Available files in $CONFIGS_DIR:"
108+
ls -la "$CONFIGS_DIR/"
109+
exit 1
110+
fi
111+
112+
# Update paths for container environment (relative -> absolute)
113+
sed -i 's|db_path: \.llama/distributions|db_path: /app-root/.llama/distributions|g' run.yaml
114+
sed -i 's|db_path: tmp/|db_path: /app-root/.llama/distributions/|g' run.yaml
115+
116+
echo "Successfully configured for environment: $ENVIRONMENT"
117+
echo "Using configuration: $(basename "$CONFIG_FILE")"
118+
119+
- name: Test RHAIIS connectivity
120+
env:
121+
RHAIIS_URL: ${{ secrets.RHAIIS_URL }}
122+
RHAIIS_API_KEY: ${{ secrets.RHAIIS_API_KEY }}
123+
run: |
124+
curl ${RHAIIS_URL}:8000/v1/models -H "Authorization: Bearer ${RHAIIS_API_KEY}"
125+
126+
- name: Run service manually
127+
env:
128+
RHAIIS_URL: ${{ secrets.RHAIIS_URL }}
129+
RHAIIS_API_KEY: ${{ secrets.RHAIIS_API_KEY }}
130+
run: |
131+
docker compose version
132+
docker compose up -d
133+
134+
# Check for errors and show logs if any services failed
135+
if docker compose ps | grep -E 'Exit|exited|stopped'; then
136+
echo "Some services failed to start - showing logs:"
137+
docker compose logs
138+
exit 1
139+
else
140+
echo "All services started successfully"
141+
fi
142+
143+
- name: Wait for services
144+
run: |
145+
echo "Waiting for services to be healthy..."
146+
sleep 20 # adjust depending on boot time
147+
148+
- name: Quick connectivity test
149+
run: |
150+
echo "Testing basic connectivity before full test suite..."
151+
curl -f http://localhost:8080/v1/models || {
152+
echo "❌ Basic connectivity failed - showing logs before running full tests"
153+
docker compose logs --tail=30
154+
exit 1
155+
}
156+
157+
- name: Run e2e tests
158+
run: |
159+
echo "Installing test dependencies..."
160+
pip install uv
161+
uv sync
162+
163+
echo "Running comprehensive e2e test suite..."
164+
make test-e2e
165+
166+
- name: Show logs on failure
167+
if: failure()
168+
run: |
169+
echo "=== Test failure logs ==="
170+
echo "=== llama-stack logs ==="
171+
docker compose logs llama-stack
172+
173+
echo ""
174+
echo "=== lightspeed-stack logs ==="
175+
docker compose logs lightspeed-stack

‎README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ The service includes comprehensive user data collection capabilities for various
2222
* [Configuration](#configuration)
2323
* [LLM Compatibility](#llm-compatibility)
2424
* [Set LLM provider and model](#set-llm-provider-and-model)
25+
* [Supported providers](#supported-providers)
2526
* [Integration with Llama Stack](#integration-with-llama-stack)
2627
* [Llama Stack as separate server](#llama-stack-as-separate-server)
2728
* [MCP Server and Tool Configuration](#mcp-server-and-tool-configuration)
@@ -123,6 +124,7 @@ Lightspeed Core Stack (LCS) supports the large language models from the provider
123124
| -------- | ---------------------------------------------- | ------------ | -------------- | -------------------------------------------------------------------------- |
124125
| OpenAI | gpt-5, gpt-4o, gpt4-turbo, gpt-4.1, o1, o3, o4 | Yes | remote::openai | [1](examples/openai-faiss-run.yaml) [2](examples/openai-pgvector-run.yaml) |
125126
| OpenAI | gpt-3.5-turbo, gpt-4 | No | remote::openai | |
127+
| RHAIIS (vLLM)| meta-llama/Llama-3.1-8B-Instruct | Yes | remote::vllm | [1](tests/e2e/configs/run-rhaiis.yaml) |
126128

127129
The "provider_type" is used in the llama stack configuration file when refering to the provider.
128130

@@ -156,6 +158,9 @@ models:
156158
provider_model_id: gpt-4-turbo
157159
```
158160

161+
## Supported providers
162+
163+
For a comprehensive list of supported providers, take a look [here](docs/providers.md).
159164

160165
## Integration with Llama Stack
161166

‎docker-compose.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ services:
1414
- OPENAI_API_KEY=${OPENAI_API_KEY}
1515
- BRAVE_SEARCH_API_KEY=${BRAVE_SEARCH_API_KEY:-}
1616
- TAVILY_SEARCH_API_KEY=${TAVILY_SEARCH_API_KEY:-}
17+
- RHAIIS_URL=${RHAIIS_URL}
18+
- RHAIIS_API_KEY=${RHAIIS_API_KEY}
1719
networks:
1820
- lightspeednet
1921
healthcheck:

‎docs/providers.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,15 @@ The tables below summarize each provider category, containing the following atri
5555
| tgi | remote | `huggingface_hub`, `aiohttp` | ❌ |
5656
| together | remote | `together` | ❌ |
5757
| vertexai | remote | `litellm`, `google-cloud-aiplatform` | ❌ |
58-
| vllm | remote | `openai` | ❌ |
5958
| watsonx | remote | `ibm_watsonx_ai` | ❌ |
6059

60+
Red Hat providers:
61+
62+
| Name | Version Tested | Type | Pip Dependencies | Supported in LCS |
63+
|---|---|---|---|:---:|
64+
| RHAIIS (vllm) | 3.2.3 (on RHEL 9.20250429.0.4) | remote | `openai` | ✅ |
65+
66+
6167
---
6268

6369
## Agent Providers

‎tests/e2e/configs/run-rhaiis.yaml‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
version: '2'
2+
image_name: rhaiis-configuration
3+
4+
apis:
5+
- agents
6+
- datasetio
7+
- eval
8+
- files
9+
- inference
10+
- post_training
11+
- safety
12+
- scoring
13+
- telemetry
14+
- tool_runtime
15+
- vector_io
16+
benchmarks: []
17+
container_image: null
18+
datasets: []
19+
external_providers_dir: null
20+
inference_store:
21+
db_path: .llama/distributions/ollama/inference_store.db
22+
type: sqlite
23+
logging: null
24+
metadata_store:
25+
db_path: .llama/distributions/ollama/registry.db
26+
namespace: null
27+
type: sqlite
28+
providers:
29+
files:
30+
- config:
31+
storage_dir: /tmp/llama-stack-files
32+
metadata_store:
33+
type: sqlite
34+
db_path: .llama/distributions/ollama/files_metadata.db
35+
provider_id: localfs
36+
provider_type: inline::localfs
37+
agents:
38+
- config:
39+
persistence_store:
40+
db_path: .llama/distributions/ollama/agents_store.db
41+
namespace: null
42+
type: sqlite
43+
responses_store:
44+
db_path: .llama/distributions/ollama/responses_store.db
45+
type: sqlite
46+
provider_id: meta-reference
47+
provider_type: inline::meta-reference
48+
datasetio:
49+
- config:
50+
kvstore:
51+
db_path: .llama/distributions/ollama/huggingface_datasetio.db
52+
namespace: null
53+
type: sqlite
54+
provider_id: huggingface
55+
provider_type: remote::huggingface
56+
- config:
57+
kvstore:
58+
db_path: .llama/distributions/ollama/localfs_datasetio.db
59+
namespace: null
60+
type: sqlite
61+
provider_id: localfs
62+
provider_type: inline::localfs
63+
eval:
64+
- config:
65+
kvstore:
66+
db_path: .llama/distributions/ollama/meta_reference_eval.db
67+
namespace: null
68+
type: sqlite
69+
provider_id: meta-reference
70+
provider_type: inline::meta-reference
71+
inference:
72+
- provider_id: sentence-transformers # Can be any embedding provider
73+
provider_type: inline::sentence-transformers
74+
config: {}
75+
- provider_id: vllm
76+
provider_type: remote::vllm
77+
config:
78+
url: http://${env.RHAIIS_URL}:8000/v1/
79+
api_token: ${env.RHAIIS_API_KEY}
80+
tls_verify: false
81+
max_tokens: 2048
82+
post_training:
83+
- config:
84+
checkpoint_format: huggingface
85+
device: cpu
86+
distributed_backend: null
87+
dpo_output_dir: "."
88+
provider_id: huggingface
89+
provider_type: inline::huggingface-gpu
90+
safety:
91+
- config:
92+
excluded_categories: []
93+
provider_id: llama-guard
94+
provider_type: inline::llama-guard
95+
scoring:
96+
- config: {}
97+
provider_id: basic
98+
provider_type: inline::basic
99+
- config: {}
100+
provider_id: llm-as-judge
101+
provider_type: inline::llm-as-judge
102+
- config:
103+
openai_api_key: '********'
104+
provider_id: braintrust
105+
provider_type: inline::braintrust
106+
telemetry:
107+
- config:
108+
service_name: 'lightspeed-stack-telemetry'
109+
sinks: sqlite
110+
sqlite_db_path: .llama/distributions/ollama/trace_store.db
111+
provider_id: meta-reference
112+
provider_type: inline::meta-reference
113+
tool_runtime:
114+
- provider_id: model-context-protocol
115+
provider_type: remote::model-context-protocol
116+
config: {}
117+
scoring_fns: []
118+
server:
119+
auth: null
120+
host: null
121+
port: 8321
122+
quota: null
123+
tls_cafile: null
124+
tls_certfile: null
125+
tls_keyfile: null
126+
shields: []
127+
models:
128+
- metadata:
129+
embedding_dimension: 768 # Depends on chosen model
130+
model_id: sentence-transformers/all-mpnet-base-v2 # Example embedding model
131+
provider_id: sentence-transformers
132+
provider_model_id: sentence-transformers/all-mpnet-base-v2 # Location of embedding model
133+
model_type: embedding
134+
- model_id: meta-llama/Llama-3.1-8B-Instruct
135+
provider_id: vllm
136+
model_type: llm
137+
provider_model_id: meta-llama/Llama-3.1-8B-Instruct

0 commit comments

Comments
 (0)