Merge branch 'main' into jinja2-set-vars-incorrect

Author: srini047

.github/actionlint.yml

@@ -0,0 +1,69 @@
+name: Test Python snippets in docs
+
+on:
+  schedule:
+    - cron: '17 3 * * *'  # daily at 03:17 UTC
+  workflow_dispatch:
+    inputs:
+      haystack_version:
+        description: 'Haystack version to test against (e.g., 2.16.1, main)'
+        required: false
+        default: 'main'
+        type: string
+
+  # TEMPORARILY DISABLED
+  # push:
+  #   paths:
+  #     - 'docs-website/docs/**'
+  #     - 'docs-website/versioned_docs/**'
+  #     - 'docs-website/scripts/test_python_snippets.py'
+  #     - 'docs-website/scripts/generate_requirements.py'
+  #     - '.github/workflows/docs-website-test-docs-snippets.yml'
+
+jobs:
+  test-docs-snippets:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      # TODO: We'll properly set these after migration to core project
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+
+      - name: Install base dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests toml
+
+      - name: Generate requirements.txt
+        run: |
+          # Use input version or default to main
+          if [ "${{ github.event.inputs.haystack_version }}" != "" ]; then
+            VERSION="${{ github.event.inputs.haystack_version }}"
+          else
+            VERSION="main"
+          fi
+          echo "Generating requirements.txt for Haystack version: $VERSION"
+          python docs-website/scripts/generate_requirements.py --version "$VERSION"
+
+      - name: Install dependencies
+        run: |
+          if [ -f requirements.txt ]; then
+            echo "Installing dependencies from requirements.txt"
+            pip install -r requirements.txt
+          else
+            echo "Error: requirements.txt was not generated"
+            exit 1
+          fi
+
+      - name: Run snippet tests (verbose)
+        run: |
+          # TEMPORARY: Testing with single file to make CI green
+          # TODO: Expand to run all docs: --paths docs versioned_docs
+          python docs-website/scripts/test_python_snippets.py --paths docs-website/docs/api/haystack-api/agents_api.md

.github/utils/deepset_sync.py

@@ -0,0 +1,42 @@
+name: Vale Lint
+
+on:
+  pull_request:
+    paths:
+      - 'docs-website/**'
+  push:
+    branches: [ main ]
+    paths:
+      - 'docs-website/**'
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  vale:
+    name: Run Vale
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: '20'
+
+      - name: Install mdx2vast
+        run: npm install -g mdx2vast
+
+      - name: Vale
+        uses: errata-ai/vale-action@v2
+        with:
+          files: |
+            docs-website/docs
+            docs-website/versioned_docs
+          reporter: github-pr-review
+          fail_on_error: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        continue-on-error: true

.github/workflows/docs-website-test-docs-snippets.yml

@@ -12,9 +12,7 @@ on:
 
 env:
   HATCH_VERSION: "1.14.2"
-  PYTHON_VERSION: "3.9"
-  DOCS_REPO: "deepset-ai/haystack-docs"
-  DOCS_REPO_PATH: "reference/haystack-api"
+  PYTHON_VERSION: "3.11"
 
 jobs:
   sync:
@@ -34,41 +32,29 @@ jobs:
       - name: Install Hatch
         run: pip install hatch==${{ env.HATCH_VERSION }}
 
-      - name: Generate API docs for Docusaurus
+      - name: Generate API reference for Docusaurus
         # This command simply runs ./.github/utils/pydoc-markdown.sh with a specific config path
         run: hatch run readme:sync "../config_docusaurus/*"
 
-      - name: Checkout Docusaurus repo
-        uses: actions/checkout@v5
-        with:
-          repository: ${{ env.DOCS_REPO }}
-          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
-          path: haystack-docs
-
-      - name: Sync generated docs to Docusaurus repo
+      - name: Sync generated API reference to docs folder
         run: |
           SOURCE_PATH="docs/pydoc/temp"
-          DEST_PATH="haystack-docs/${{ env.DOCS_REPO_PATH }}"
+          DEST_PATH="docs-website/reference/haystack-api"
 
           echo "Syncing from $SOURCE_PATH to $DEST_PATH"
           mkdir -p $DEST_PATH
           # Using rsync to copy files. This will also remove files in dest that are no longer in source.
           rsync -av --delete --exclude='.git/' "$SOURCE_PATH/" "$DEST_PATH/"
 
-      - name: Commit and push changes
-        env:
-            GITHUB_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
-        run: |
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-
-          cd haystack-docs
-
-          if [[ -n $(git status -s) ]]; then
-            echo "Syncing docs with Docusaurus..."
-            git add .
-            git commit -m "docs: Sync Haystack API reference"
-            git push
-          else
-            echo "No changes to sync with Docusaurus."
-          fi
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Sync Haystack API reference on Docusaurus"
+          branch: sync-docusaurus-api-reference
+          base: main
+          title: "docs: sync Haystack API reference on Docusaurus"
+          add-paths: |
+            docs-website/reference/haystack-api
+          body: |
+            This PR syncs the Haystack API reference on Docusaurus. Just approve and merge it.

.github/workflows/docs-website-vale.yml

@@ -61,6 +61,15 @@ coverage.xml
 *.mo
 *.pot
 
+# Docs website (Docusaurus)
+docs-website/.docusaurus/
+docs-website/build/
+docs-website/node_modules/
+docs-website/.cache-loader/
+docs-website/.eslintcache
+docs-website/.netlify/
+docs-website/coverage/
+
 # Django stuff:
 *.log
 local_settings.py
@@ -144,6 +153,7 @@ dmypy.json
 haystack/document_store/qa.db
 **/mlruns/**
 src
+!docs-website/src/
 models
 saved_models
 *_build

.github/workflows/docusaurus_sync.yml

@@ -1 +1 @@
-2.19.0-rc0
+2.20.0-rc0

.gitignore

@@ -0,0 +1,3 @@
+* text=auto
+*.md text diff=markdown
+*.mdx text diff=markdown