chore(internal/ci): setup breaking change detection

Author: stainless-app[bot]

.github/workflows/detect-breaking-changes.yml

@@ -0,0 +1,42 @@
+name: CI
+on:
+  pull_request:
+    branches:
+      - main
+      - next
+
+jobs:
+  detect_breaking_changes:
+    runs-on: 'ubuntu-latest'
+    name: detect-breaking-changes
+    if: github.repository == 'openai/openai-python'
+    steps:
+      - name: Calculate fetch-depth
+        run: |
+          echo "FETCH_DEPTH=$(expr ${{ github.event.pull_request.commits }} + 1)" >> $GITHUB_ENV
+
+      - uses: actions/checkout@v4
+        with:
+          # Ensure we can check out the pull request base in the script below.
+          fetch-depth: ${{ env.FETCH_DEPTH }}
+
+      - name: Install Rye
+        run: |
+          curl -sSf https://rye.astral.sh/get | bash
+          echo "$HOME/.rye/shims" >> $GITHUB_PATH
+        env:
+          RYE_VERSION: '0.44.0'
+          RYE_INSTALL_OPTION: '--yes'
+      - name: Install dependencies
+        run: |
+          rye sync --all-features
+      - name: Detect removed symbols
+        run: |
+          rye run python scripts/detect-breaking-changes.py "${{ github.event.pull_request.base.sha }}"
+
+      - name: Detect breaking changes
+        run: |
+          # Try to check out previous versions of the breaking change detection script. This ensures that
+          # we still detect breaking changes when entire files and their tests are removed.
+          git checkout "${{ github.event.pull_request.base.sha }}" -- ./scripts/detect-breaking-changes 2>/dev/null || true
+          ./scripts/detect-breaking-changes ${{ github.event.pull_request.base.sha }}

.stats.yml

@@ -1,4 +1,4 @@
 configured_endpoints: 111
 openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-7ef7a457c3bf05364e66e48c9ca34f31bfef1f6c9b7c15b1812346105e0abb16.yml
 openapi_spec_hash: a2b1f5d8fbb62175c93b0ebea9f10063
-config_hash: 76afa3236f36854a8705f1281b1990b8
+config_hash: 4870312b04f48fd717ea4151053e7fb9

pyproject.toml

@@ -71,6 +71,7 @@ dev-dependencies = [
     "trio >=0.22.2",
     "nest_asyncio==1.6.0",
     "pytest-xdist>=3.6.1",
+    "griffe>=1",
 ]
 
 [tool.rye.scripts]

requirements-dev.lock

@@ -44,6 +44,8 @@ cffi==1.16.0
     # via sounddevice
 charset-normalizer==3.3.2
     # via requests
+colorama==0.4.6
+    # via griffe
 colorlog==6.7.0
     # via nox
 cryptography==42.0.7
@@ -68,6 +70,7 @@ filelock==3.12.4
 frozenlist==1.7.0
     # via aiohttp
     # via aiosignal
+griffe==1.12.1
 h11==0.16.0
     # via httpcore
 httpcore==1.0.9

scripts/detect-breaking-changes

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+echo "==> Detecting breaking changes"
+
+TEST_PATHS=(
+	tests/api_resources
+	tests/test_client.py
+	tests/test_response.py
+	tests/test_legacy_response.py
+)
+
+for PATHSPEC in "${TEST_PATHS[@]}"; do
+    # Try to check out previous versions of the test files
+    # with the current SDK.
+    git checkout "$1" -- "${PATHSPEC}" 2>/dev/null || true
+done
+
+# Instead of running the tests, use the linter to check if an
+# older test is no longer compatible with the latest SDK.
+./scripts/lint

scripts/detect-breaking-changes.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import sys
+from typing import Iterator
+from pathlib import Path
+
+import rich
+import griffe
+from rich.text import Text
+from rich.style import Style
+
+
+def public_members(obj: griffe.Object | griffe.Alias) -> dict[str, griffe.Object | griffe.Alias]:
+    if isinstance(obj, griffe.Alias):
+        # ignore imports for now, they're technically part of the public API
+        # but we don't have good preventative measures in place to prevent
+        # changing them
+        return {}
+
+    return {name: value for name, value in obj.all_members.items() if not name.startswith("_")}
+
+
+def find_breaking_changes(
+    new_obj: griffe.Object | griffe.Alias,
+    old_obj: griffe.Object | griffe.Alias,
+    *,
+    path: list[str],
+) -> Iterator[Text | str]:
+    new_members = public_members(new_obj)
+    old_members = public_members(old_obj)
+
+    for name, old_member in old_members.items():
+        if isinstance(old_member, griffe.Alias) and len(path) > 2:
+            # ignore imports in `/types/` for now, they're technically part of the public API
+            # but we don't have good preventative measures in place to prevent changing them
+            continue
+
+        new_member = new_members.get(name)
+        if new_member is None:
+            cls_name = old_member.__class__.__name__
+            yield Text(f"({cls_name})", style=Style(color="rgb(119, 119, 119)"))
+            yield from [" " for _ in range(10 - len(cls_name))]
+            yield f" {'.'.join(path)}.{name}"
+            yield "\n"
+            continue
+
+        yield from find_breaking_changes(new_member, old_member, path=[*path, name])
+
+
+def main() -> None:
+    try:
+        against_ref = sys.argv[1]
+    except IndexError as err:
+        raise RuntimeError("You must specify a base ref to run breaking change detection against") from err
+
+    package = griffe.load(
+        "openai",
+        search_paths=[Path(__file__).parent.parent.joinpath("src")],
+    )
+    old_package = griffe.load_git(
+        "openai",
+        ref=against_ref,
+        search_paths=["src"],
+    )
+    assert isinstance(package, griffe.Module)
+    assert isinstance(old_package, griffe.Module)
+
+    output = list(find_breaking_changes(package, old_package, path=["openai"]))
+    if output:
+        rich.print(Text("Breaking changes detected!", style=Style(color="rgb(165, 79, 87)")))
+        rich.print()
+
+        for text in output:
+            rich.print(text, end="")
+
+        sys.exit(1)
+
+
+main()