Update feature/perf from master (#6000)

There is a conflict in 'dune' files, have to make PR from fork to fix,
and I cannot edit the origin repo on the other PR.

Author: edwintorok

.github/workflows/format.yml

@@ -6,6 +6,7 @@ on:
       - master
       - 'feature/**'
       - '*-lcm'
+  merge_group:
 
 jobs:
   ocaml-format:

.github/workflows/generate-and-build-sdks.yml

@@ -55,6 +55,12 @@ jobs:
             _build/install/default/xapi/sdk/go/*
             !_build/install/default/xapi/sdk/go/dune
 
+      - name: Store Java SDK source
+        uses: actions/upload-artifact@v4
+        with:
+          name: SDK_Source_Java
+          path: _build/install/default/xapi/sdk/java/*
+
       - name: Trim dune cache
         run: opam exec -- dune cache trim --size=2GiB
 
@@ -84,6 +90,39 @@ jobs:
             source/*
             !source/src/*.o
 
+  build-java-sdk:
+    name: Build Java SDK
+    runs-on: ubuntu-latest
+    needs: generate-sdk-sources
+    steps:
+      - name: Install dependencies
+        run: sudo apt-get install maven
+
+      - name: Retrieve Java SDK source
+        uses: actions/download-artifact@v4
+        with:
+          name: SDK_Source_Java
+          path: source/
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+
+      - name: Build Java SDK
+        shell: bash
+        run: |
+          xapi_version="${{ inputs.xapi_version }}"
+          xapi_version="${xapi_version//v/}"
+          mkdir -p target && mvn -f source/xen-api/pom.xml -B -Drevision=$xapi_version-prerelease clean package && mv source/xen-api/target/*.jar target/
+
+      - name: Store Java SDK
+        uses: actions/upload-artifact@v4
+        with:
+          name: SDK_Artifacts_Java
+          path: target/*
+
   build-csharp-sdk:
     name: Build C# SDK
     runs-on: windows-2022

.github/workflows/main.yml

@@ -6,9 +6,10 @@ on:
   schedule:
     # run daily, this refreshes the cache
     - cron: "13 2 * * *"
+  merge_group:
 
 concurrency: # On new push, cancel old workflows from the same PR, branch or tag:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{github.event_name}}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -45,7 +46,6 @@ jobs:
       - name: Make install smoketest
         run: |
           opam exec -- make install DESTDIR=$(mktemp -d)
-          opam exec -- make install DESTDIR=$(mktemp -d) BUILD_PY2=NO
 
       - name: Check disk space
         run: df -h || true

.github/workflows/other.yml

@@ -6,9 +6,10 @@ on:
   schedule:
     # run daily, this refreshes the cache
     - cron: "13 2 * * *"
+  merge_group:
 
 concurrency: # On new push, cancel old workflows from the same PR, branch or tag:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{github.event_name}}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -18,7 +19,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["2.7", "3.11"]
+        python-version: ["3.11"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -39,48 +40,11 @@ jobs:
 
       - uses: pre-commit/[email protected]
         name: Run pre-commit checks (no spaces at end of lines, etc)
-        if: ${{ matrix.python-version != '2.7' }}
         with:
           extra_args: --all-files --verbose --hook-stage commit
         env:
           SKIP: no-commit-to-branch
 
-      - name: Install dependencies only needed for python 2
-        if: ${{ matrix.python-version == '2.7' }}
-        run: pip install enum
-
-      - name: Install dependencies only needed for python 3
-        if: ${{ matrix.python-version != '2.7' }}
-        run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt
-
-      - name: Install common dependencies for Python ${{matrix.python-version}}
-        run: pip install future mock pytest-coverage pytest-mock
-
-      - name: Run Pytest for python 2 and get code coverage
-        if: ${{ matrix.python-version == '2.7' }}
-        run: >
-          pytest
-          --cov=scripts --cov=ocaml/xcp-rrdd
-          scripts/ ocaml/xcp-rrdd -vv -rA
-          --junitxml=.git/pytest${{matrix.python-version}}.xml
-          --cov-report term-missing
-          --cov-report xml:.git/coverage${{matrix.python-version}}.xml
-        env:
-          PYTHONDEVMODE: yes
-
-      - name: Run Pytest for python 3 and get code coverage
-        if: ${{ matrix.python-version != '2.7' }}
-        run: >
-          pytest
-          --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/
-          scripts/ ocaml/xcp-rrdd python3/ -vv -rA
-          --junitxml=.git/pytest${{matrix.python-version}}.xml
-          --cov-report term-missing
-          --cov-report xml:.git/coverage${{matrix.python-version}}.xml
-        env:
-          PYTHONDEVMODE: yes
-          PYTHONPATH: "python3:python3/tests/stubs"
-
       - name: Upload coverage report to Coveralls
         uses: coverallsapp/github-action@v2
         with:
@@ -90,7 +54,6 @@ jobs:
           parallel: true
 
       - uses: dciborow/[email protected]
-        if: ${{ matrix.python-version != '2.7' }}
         with:
           reporter: github-pr-review
           level: warning
@@ -99,8 +62,7 @@ jobs:
         continue-on-error: true
 
       - name: Run pytype checks
-        if: ${{ matrix.python-version != '2.7' }}
-        run: ./pytype_reporter.py
+        run: pip install pandas pytype toml && ./pytype_reporter.py
         env:
           PR_NUMBER: ${{ github.event.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -33,7 +33,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: XenAPI
-          path: scripts/examples/python/dist/
+          path: python3/examples/dist/
 
   build-sdks:
     name: Build and upload SDK artifacts
@@ -64,6 +64,12 @@ jobs:
           name: SDK_Artifacts_C
           path: libxenserver/usr/local/
 
+      - name: Retrieve Java SDK distribution artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: SDK_Artifacts_Java
+          path: dist/
+
       - name: Retrieve C# SDK distribution artifacts
         uses: actions/download-artifact@v4
         with:

.github/workflows/setup-xapi-environment/action.yml

@@ -28,9 +28,9 @@ runs:
       shell: bash
       run: sudo apt-get update
 
-    - name: Install python2
+    - name: Install python3
       shell: bash
-      run: sudo apt-get install python2
+      run: sudo apt-get install python3
 
     - name: Use disk with more space for TMPDIR and XDG_CACHE_HOME
       shell: bash

.github/workflows/shellcheck.yaml

@@ -2,9 +2,10 @@ name: ShellCheck
 
 on:
   pull_request:
+  merge_group:
 
 concurrency:  # On new push, cancel old workflows from the same PR, branch or tag:
-  group: sc-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: sc-${{ github.workflow }}-${{github.event_name}}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:

.gitignore

@@ -16,10 +16,10 @@ config.mk
 # python packaging
 **/__pycache__/
 **/*.pyc
-scripts/examples/python/setup.py
-scripts/examples/python/XenAPI.egg-info/
-scripts/examples/python/build/
-scripts/examples/python/dist/
+python3/examples/setup.py
+python3/examples/XenAPI.egg-info/
+python3/examples/build/
+python3/examples/dist/
 
 # ignore file needed for building the SDK
 ocaml/sdk-gen/csharp/XE_SR_ERRORCODES.xml

.ocamlformat

@@ -7,3 +7,4 @@ break-separators=before
 break-infix=fit-or-vertical
 break-infix-before-func=false
 sequence-blank-line=preserve-one
+ocaml-version=4.14

.pre-commit-config.yaml

@@ -14,6 +14,8 @@
 # pre-commit run -av --hook-stage pre-push
 #
 default_stages: [commit, push]
+default_language_version:
+    python: python3.11
 repos:
 # Recommendation for a minimal git pre-commit hook:
 # https://github.com/pre-commit/pre-commit-hooks/blob/main/README.md:
@@ -29,6 +31,95 @@ repos:
     -   id: check-executables-have-shebangs
         exclude: ocaml
 
+
+# Improve Python formatting incrementally:
+# https://dev.to/akaihola/improving-python-code-incrementally-3f7a
+#
+# darker checks if staged python changes are formatted according using
+# the PEP8-aligned black formatter. It also checks if the imports are sorted.
+#
+# It is a good idea to run this before committing, and it is also run in the
+# GitHub Workflow.
+#
+# Note: darker only checks the changes in files ending in .py!
+# Python scripts that don't end in .py should be renamed to have the .py extension
+# when moving them to python3/bin.
+# (remove the .py extension in the Makefile when installing the file)
+#
+-   repo: https://github.com/akaihola/darker
+    rev: 1.7.3
+    hooks:
+    -   id: darker
+        files: python3/
+        name: check changes in Python3 tree using darker and isort
+        args: [--diff, --skip-string-normalization, --isort, -tpy36]
+        additional_dependencies: [isort]
+
+#
+# Run pytest and diff-cover to check that the new /python3 test suite in passes.
+# This hook uses a local venv containing the required dependencies. When adding
+# new dependencies, they should be added to the additional_dependencies below.
+#
+-   repo: local
+    hooks:
+    -   id: pytest
+        files: python3/
+        name: check that the Python3 test suite in passes
+        entry: env PYTHONDEVMODE=yes sh -c 'coverage run && coverage xml &&
+            coverage html && coverage report &&
+            diff-cover --ignore-whitespace --compare-branch=origin/master
+            --show-uncovered --html-report .git/coverage-diff.html
+            --fail-under 50 .git/coverage3.11.xml'
+        require_serial: true
+        pass_filenames: false
+        language: python
+        types: [python]
+        additional_dependencies:
+        - coverage
+        - diff-cover
+        - future
+        - opentelemetry-api
+        - opentelemetry-exporter-zipkin-json
+        - opentelemetry-sdk
+        - pytest-mock
+        - mock
+        - wrapt
+        - XenAPI
+
+
+-   repo: https://github.com/RobertCraigie/pyright-python
+    rev: v1.1.372
+    hooks:
+    -   id: pyright
+        name: check that python3 tree passes pyright/VSCode check
+        files: python3/
+        additional_dependencies:
+        - mock
+        - opentelemetry-api
+        - opentelemetry-exporter-zipkin-json
+        - opentelemetry-sdk
+        - pytest
+        - pyudev
+        - XenAPI
+
+
+# Check that pylint passes for the changes in new /python3 code.
+-   repo: local
+    hooks:
+    -   id: pylint
+        files: python3/
+        stages: [push]
+        name: check that changes to python3 tree pass pylint
+        entry: diff-quality --violations=pylint
+            --ignore-whitespace --compare-branch=origin/master
+        pass_filenames: false
+        language: python
+        types: [python]
+        additional_dependencies: [diff-cover, pylint, pytest]
+
+
+# pre-push hook (it only runs if you install pre-commit as a pre-push hook):
+# It can be manually tested using: `pre-commit run -av --hook-stage push`
 # Recommendation for a minimal git pre-push hook:
 # While using pre-commit yields great results, it
 # is "not fast". Therefore only run it pre-push,
@@ -53,4 +144,12 @@ repos:
         # developers have such version installed, it can be configured here:
         # language_version: python3.11
         require_serial: true
-        additional_dependencies: [pandas, pytype]
+        additional_dependencies:
+        - future
+        - opentelemetry-api
+        - opentelemetry-exporter-zipkin-json
+        - opentelemetry-sdk
+        - pandas
+        - pytest
+        - pytype
+        files: python3/