From c47007b3aa3a715f9cb8a42f3500c859a50edf30 Mon Sep 17 00:00:00 2001 From: surister Date: Tue, 8 Apr 2025 14:39:40 +0200 Subject: [PATCH 1/9] Remove CrateDB from both javascript and python test pipelines. The objective of the project is to be able to do SQL stuff without a CrateDB instance, so it does not make any sense to have one. --- .github/workflows/javascript.yml | 10 ---------- .github/workflows/python.yml | 10 ---------- 2 files changed, 20 deletions(-) diff --git a/.github/workflows/javascript.yml b/.github/workflows/javascript.yml index 4492c51..0a3276f 100644 --- a/.github/workflows/javascript.yml +++ b/.github/workflows/javascript.yml @@ -38,16 +38,6 @@ jobs: OS: ${{ matrix.os }} NODEJS: ${{ matrix.node-version }} - # https://docs.github.com/en/actions/using-containerized-services/about-service-containers - services: - cratedb: - image: crate/crate:nightly - ports: - - 4200:4200 - - 5432:5432 - env: - CRATE_HEAP_SIZE: 4g - name: Node.js ${{ matrix.node-version }} on OS ${{ matrix.os }} steps: diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 942d905..250950e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -38,16 +38,6 @@ jobs: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} - # https://docs.github.com/en/actions/using-containerized-services/about-service-containers - services: - cratedb: - image: crate/crate:nightly - ports: - - 4200:4200 - - 5432:5432 - env: - CRATE_HEAP_SIZE: 4g - name: Python ${{ matrix.python-version }} on OS ${{ matrix.os }} steps: From f260ef5e2c3034dd48fb00eb9be70cf8a01e6f7b Mon Sep 17 00:00:00 2001 From: surister Date: Tue, 8 Apr 2025 14:41:58 +0200 Subject: [PATCH 2/9] Update tested python versions. Removed 3.8 which reached EOL and ignored 3.9 which reaches it soon. https://devguide.python.org/versions/ --- .github/workflows/python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 250950e..cae522e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -32,7 +32,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - python-version: ["3.8", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13"] env: OS: ${{ matrix.os }} From 298430a6de1cbd02fe07f195a7e8750584aa32c8 Mon Sep 17 00:00:00 2001 From: surister Date: Tue, 8 Apr 2025 14:47:28 +0200 Subject: [PATCH 3/9] Overhaul javascript.yml and python.yml. * Use uv in pipelines * Add path for more fine-controlled executions fixes #39 * Change parent project to uv * Update metadata --- .github/workflows/javascript.yml | 11 +++++++---- .github/workflows/python.yml | 25 ++++++++++++------------ cratedb_sqlparse_py/pyproject.toml | 31 ++++++++++++++++-------------- pyproject.toml | 15 ++++++++++----- requirements.txt | 1 - 5 files changed, 46 insertions(+), 37 deletions(-) diff --git a/.github/workflows/javascript.yml b/.github/workflows/javascript.yml index 0a3276f..f0c1bcd 100644 --- a/.github/workflows/javascript.yml +++ b/.github/workflows/javascript.yml @@ -2,9 +2,14 @@ name: JavaScript Tests on: - pull_request: ~ + pull_request: + branches: [ main ] + paths: + - cratedb_sqlparse_js push: branches: [ main ] + paths: + - cratedb_sqlparse_js # Allow job to be triggered manually. workflow_dispatch: @@ -24,15 +29,13 @@ defaults: working-directory: cratedb_sqlparse_js jobs: - tests: - runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: ["ubuntu-latest"] - node-version: ["18", "20", "22"] + node-version: ["20", "22", "23"] env: OS: ${{ matrix.os }} diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index cae522e..9b0b0af 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -2,9 +2,15 @@ name: Python Tests on: - pull_request: ~ + pull_request: + branches: [ main ] + paths: + - cratedb_sqlparse_py push: branches: [ main ] + paths: + - cratedb_sqlparse_py + # Allow job to be triggered manually. workflow_dispatch: @@ -24,9 +30,7 @@ defaults: working-directory: cratedb_sqlparse_py jobs: - tests: - runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -49,22 +53,17 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - cache: 'pip' - cache-dependency-path: 'pyproject.toml' + + - name: Install uv + uses: astral-sh/setup-uv@v5 - name: Set up project run: | - - # `setuptools 0.64.0` adds support for editable install hooks (PEP 660). - # https://github.com/pypa/setuptools/blob/main/CHANGES.rst#v6400 - pip install "setuptools>=64" --upgrade - - # Install package in editable mode. - pip install --use-pep517 --prefer-binary --editable='.[develop,generate,test]' + uv sync --all-groups - name: Run linter and software tests run: | - poe check + uv run poe check - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 diff --git a/cratedb_sqlparse_py/pyproject.toml b/cratedb_sqlparse_py/pyproject.toml index d96f478..7d3d61d 100644 --- a/cratedb_sqlparse_py/pyproject.toml +++ b/cratedb_sqlparse_py/pyproject.toml @@ -23,7 +23,7 @@ license = { text = "Apache License 2.0" } authors = [ { name = "Ivan Sanchez Valencia", email = "ivan.sanchezvalencia@crate.io" }, ] -requires-python = ">=3.8" +requires-python = ">=3.9" classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Console", @@ -67,30 +67,34 @@ dynamic = [ dependencies = [ "antlr4-python3-runtime<4.14", ] -optional-dependencies.develop = [ + +[dependency-groups] +dev = [ "mypy<1.16", "packaging", "poethepoet<1", "pyproject-fmt<2.6", "ruff<0.12", "validate-pyproject<0.25", + "pytest<9", + "pytest-cov<7", ] -optional-dependencies.generate = [ +generate = [ "antlr4-tools<0.3", "requests<3", ] -optional-dependencies.release = [ + +release = [ "build<2", "twine<7", ] -optional-dependencies.test = [ - "pytest<9", - "pytest-cov<7", -] -urls.changelog = "https://github.com/crate/cratedb-sqlparse/blob/main/CHANGES.md" -urls.documentation = "https://github.com/crate/cratedb-sqlparse" -urls.homepage = "https://github.com/crate/cratedb-sqlparse" -urls.repository = "https://github.com/crate/cratedb-sqlparse" + +[project.urls] +Homepage = "https://github.com/crate/cratedb-sqlparse" +Documentation = "https://github.com/crate/cratedb-sqlparse" +Repository = "https://github.com/crate/cratedb-sqlparse" +Changelog = "https://github.com/crate/cratedb-sqlparse/releases" + [tool.setuptools.packages.find] namespaces = false @@ -152,8 +156,7 @@ testpaths = [ "tests", ] xfail_strict = true -markers = [ -] +markers = [] [tool.coverage.run] branch = false diff --git a/pyproject.toml b/pyproject.toml index d697afb..afa031c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,11 @@ -[tool.poe.tasks] - -generate = [ - { cmd = "python setup_grammar.py python" }, - { cmd = "python setup_grammar.py javascript" }, +[project] +name = "cratedb-sqlparse-generator" +version = "0.0.1" +description = "Dependencies to build and generate cratedb-sqlparse parsers; same for all target languages" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "antlr4-tools>=0.2.1", + "requests>=2.32.3", ] +tool.uv.package = false diff --git a/requirements.txt b/requirements.txt index 2547dfd..c246bdc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ antlr4-tools<0.3 -poethepoet<1 requests<3 From f90dddbf48c88ee00ff27123abc5a7dc5e8d947f Mon Sep 17 00:00:00 2001 From: surister Date: Tue, 8 Apr 2025 15:46:01 +0200 Subject: [PATCH 4/9] Add more ignored artifacts to gitignore --- .gitignore | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitignore b/.gitignore index d841e3a..3fe6462 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,11 @@ cython_debug/ *.tokens *.interp *.g4 + +node_modules +**/cratedb_sqlparse/**/SqlBaseParserVisitor.* +**/cratedb_sqlparse/**/SqlBaseParserListener.* + +node_modules +**/cratedb_sqlparse/**/SqlBaseParserVisitor.* +**/cratedb_sqlparse/**/SqlBaseParserListener.* \ No newline at end of file From 810424b1e985d515ba0475e471b90cc25b6202c6 Mon Sep 17 00:00:00 2001 From: surister Date: Tue, 8 Apr 2025 15:56:13 +0200 Subject: [PATCH 5/9] Set CrateDB version as 5.10.4 --- setup_grammar.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/setup_grammar.py b/setup_grammar.py index 6b76c85..f33316e 100644 --- a/setup_grammar.py +++ b/setup_grammar.py @@ -157,18 +157,15 @@ def set_version(target: Antlr4Target, version: str): if __name__ == '__main__': """ Invoke the grammar compiler / generator. - - TODO: Converge `version` into command-line argument? - TODO: Improve efficiency by generating runtime parser for all implemented languages at once. """ setup_logging() input_target = sys.argv[1] - version = '5.8.3' + version = '5.10.4' if input_target.startswith("py"): target = Antlr4Target.python - elif input_target.startswith("js") or input_target.startswith("java"): + elif input_target.startswith("js") or input_target.startswith("javascript"): target = Antlr4Target.js else: raise NotImplementedError(f"Parser generator for target {input_target} not implemented") From 4f22c45c1c25ad9dcb237b1280887562eca38e5a Mon Sep 17 00:00:00 2001 From: surister Date: Tue, 8 Apr 2025 17:59:14 +0200 Subject: [PATCH 6/9] Add js release and re-estructure into a same workflow call that to run when a Github release is published --- .github/workflows/javascript.yml | 1 - .github/workflows/python.yml | 1 - .github/workflows/release.yml | 41 ++++-------------------- .github/workflows/release_javascript.yml | 33 +++++++++++++++++++ .github/workflows/release_python.yml | 35 ++++++++++++++++++++ 5 files changed, 75 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/release_javascript.yml create mode 100644 .github/workflows/release_python.yml diff --git a/.github/workflows/javascript.yml b/.github/workflows/javascript.yml index f0c1bcd..53fcf63 100644 --- a/.github/workflows/javascript.yml +++ b/.github/workflows/javascript.yml @@ -1,4 +1,3 @@ ---- name: JavaScript Tests on: diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 9b0b0af..f32735e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -1,4 +1,3 @@ ---- name: Python Tests on: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8cb89f9..8112857 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,36 +1,9 @@ ---- -name: Release - -on: push +on: + release: + types: [published] jobs: - pypi: - name: Build and publish package to PyPI - runs-on: ubuntu-latest - if: startsWith(github.event.ref, 'refs/tags') - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Generate grammar - run: | - python -m pip install -r requirements.txt - poe generate - - - name: Build package - run: | - python -m pip install build twine - cd cratedb_sqlparse_py - python -m build - twine check dist/{*.tar.gz,*.whl} - - - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: cratedb_sqlparse_py/dist/ - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} + release_javascript: + uses: ./.github/workflows/release_javascript.yml + release_python: + uses: ./.github/workflows/release_python.yml diff --git a/.github/workflows/release_javascript.yml b/.github/workflows/release_javascript.yml new file mode 100644 index 0000000..1dfaebc --- /dev/null +++ b/.github/workflows/release_javascript.yml @@ -0,0 +1,33 @@ +name: Publish Package to npmjs + +on: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + # Setup .npmrc file to publish to npm + - uses: actions/setup-node@v4 + with: + node-version: '20.x' + registry-url: 'https://registry.npmjs.org' + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Generate grammar + run: | + python -m pip install -r requirements.txt + python setup_grammar.py javascript + + - run: npm build + - run: npm publish --provenance --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml new file mode 100644 index 0000000..fa0a608 --- /dev/null +++ b/.github/workflows/release_python.yml @@ -0,0 +1,35 @@ +name: Publish python parser to pypi + +on: + workflow_call: + +jobs: + build: + name: Build and publish package to PyPI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Generate grammar + run: | + python -m pip install -r requirements.txt + python setup_grammar.py python + + - name: Build package + run: | + python -m pip install build twine + cd cratedb_sqlparse_py + python -m build + twine check dist/{*.tar.gz,*.whl} + + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: cratedb_sqlparse_py/dist/ + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} From 9327efe81143effd7652af51b69160207289351a Mon Sep 17 00:00:00 2001 From: surister Date: Tue, 8 Apr 2025 18:37:58 +0200 Subject: [PATCH 7/9] Set npm version from tag --- .github/workflows/release_javascript.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release_javascript.yml b/.github/workflows/release_javascript.yml index 1dfaebc..97ae1b7 100644 --- a/.github/workflows/release_javascript.yml +++ b/.github/workflows/release_javascript.yml @@ -27,7 +27,12 @@ jobs: python -m pip install -r requirements.txt python setup_grammar.py javascript + - name: Set VERSION from tag + run: | + npm version ${GITHUB_REF#refs/*/v} + - run: npm build + - run: npm publish --provenance --access public env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} From 981ad11d5606ac003e35eaa11d2daa1282be39a1 Mon Sep 17 00:00:00 2001 From: surister Date: Tue, 8 Apr 2025 18:51:34 +0200 Subject: [PATCH 8/9] Update limitations section in README.md --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f5977af..23593ec 100644 --- a/README.md +++ b/README.md @@ -49,11 +49,13 @@ print(select_query.query) ``` ## Limitations +All the implemented libraries can validate SQL, split queries and parse using antrl4's listeners. -Listeners are not implemented, which means that you can only: Validate SQL syntax, -split queries and get some Tokens metadata from -the query, if you need some more information like what https://github.com/macbre/sql-metadata does ( -e.g. get the columns of this query) open a new issue. +Listeners are partially implemented, which means that we can only parse: + +* Create table statements: table name, schema, and 'with' options + +If more metadata is needed, it can be requested in a new issue. New features should preferably be implemented in all available targets. From 6542f48e4e34540347ff8d633451f2c319a14a3d Mon Sep 17 00:00:00 2001 From: surister Date: Tue, 8 Apr 2025 18:52:15 +0200 Subject: [PATCH 9/9] Simplify and update DEVELOP.md with latest changes in the repository. --- DEVELOP.md | 155 +++++++++++++++++++++++++++-------------------------- 1 file changed, 80 insertions(+), 75 deletions(-) diff --git a/DEVELOP.md b/DEVELOP.md index c2e7dbe..c8fe084 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -1,13 +1,13 @@ # Developer Guide for cratedb-sqlparse -About building locally, or using a different CrateDB version. +Libraries in this repository are not usable without building the necessary files with `setup_grammar.py`, +this is because the generated parser is not uploaded to the repository. Every target language, e.g: +Python or JavaScript, needs to be built independently. -> The generated parser is not uploaded to the repository because it is huge. -> To use the package locally or to build a different version use the build script. +The build script should not have any side effect, meaning you should be able to re-build with +a different CrateDB version without any extra work. -## Setup - -To start things off, bootstrap the sandbox environment. +## Running the build script. ### Acquire sources ```shell @@ -15,111 +15,116 @@ git clone git@github.com:crate/cratedb-sqlparse.git cd cratedb-sqlparse ``` -### Install dependencies -``` -pip install -r requirements.txt -``` +### Setup uv +[uv](https://docs.astral.sh/uv/) is used across the project and is the recommended dependency tool, both for building the +project's grammar and the development of the Python target. -### Generate grammar files +macOS and Linux: ```shell -poe generate +curl -LsSf https://astral.sh/uv/install.sh | sh ``` +Windows: +```shell +powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" +``` -## Running Tests for Python - -First, navigate to the corresponding subdirectory: - - cd cratedb_sqlparse_py - -Verify code by running all linters and software tests: - - poe check +If those options don't work, see https://docs.astral.sh/uv/getting-started/installation/ -Run specific tests: +### Install dependencies +```shell +pip install -r requirements.txt +``` - pytest -k enricher - pytest -k lexer +or uv -Format code: +```shell +uv pip install -r requirements.txt +``` - poe format +### Generate grammar files +Python: +```shell +uv run python setup_grammar.py python +``` -## Running Tests for JavaScript +Javascript: +```shell +uv run python setup_grammar.py javascript +``` -First, navigate to the corresponding subdirectory: +Now libraries that were built are ready to use, every library e.g. `cratedb_sqlparse_js` +are on themselves different projects, with their own dependencies and +dependency management systems. - cd cratedb_sqlparse_js +## Setting up cratedb_sqlparse_py -Set up project: +In `./cratedb_sqlparse_py` run - npm install -Verify code by running all linters and software tests: +### Set up dependencies - npm test +```shell +uv sync --all-groups +``` -Run specific tests: +### Run tests - ??? +```shell +uv run pytest +``` -Format code: +### Format code: +```shell +uv run poe format +``` + +## Running Tests for JavaScript +In `./cratedb_sqlparse_js` run: - ??? +## Setup dependencies +```shell +npm install +``` +## Run tests +```shell +npm test +``` +## Releasing libraries. +Releases are done on GitHub, by creating a [new release](https://github.com/crate/cratedb-sqlparse/releases) +every library will be built and published. -## Running a Release +1. Make sure that in `setup_grammar.py` the CrateDB version matches the one you want. +2. On GitHub create a new release, creating the appropriate tag and adapting the changelog. ### Python +Releases to https://pypi.org/project/cratedb-sqlparse/ Overview: - Versioning happens automatically based on the `versioningit` package. - You just need to tag the repository. -- Package building and publishing happens automatically, being staged - through GHA to PyPI. - -On branch `main`: -- Add a section for the new version in the `CHANGES.md` file. -- Commit your changes with a message like `Release vx.y.z`. -- Create a tag, and push to remote. - This will trigger a GitHub action which releases the new version to PyPI. - ```shell - git tag v0.0.3 - git push --tags - ``` -- On GitHub, designate a new release, copying in the relevant section - from the CHANGELOG. - https://github.com/crate/cratedb-sqlparse/releases +#### Manual release. Optionally, build the package and upload to PyPI manually. ```shell -poe release +uv run poe release ``` - ### JavaScript +Releases to https://www.npmjs.com/package/@cratedb/cratedb-sqlparse Overview: -- Versioning happens manually on behalf of the `package.json` file. -- Package building and publishing to npmjs.com happens manually, using - the `npm` program. - -On branch `main`: -- Make sure to run `poe generate` on the root folder first. -- Adjust version number in `package.json`. -- Generate `package-lock.json`. +- Versioning is manual. - npm install --package-lock-only +#### Manual release +Make sure to run `poe generate` on the root folder first and adjust version number in `package.json`. -- Commit your changes with a message like `Release vx.y.z`. -- Create a tag, and push to remote. -- Build package. - - npm run build - -- Publish package. - - npm login - npm publish +Then run: +```shell +npm install --package-lock-only && +npm run build && +npm login && +npm publish +```