Release validations using release version matrix (#1611)

atalman · web-flow · commit 252463f16d20 · 2023-11-16T17:15:07.000-05:00
* Release pypi prep change (#1587) * [aarch64] Release pypi prep script change for aarch64 builds * Release versions for testing Testing calling version (#1588) Upstream/release validations (#1589) * Testing calling version * add release matrix Upstream/release validations (#1590) * Testing calling version * add release matrix * test test (#1591) test (#1592) Release v1 (#1595) * test * test Release v1 (#1596) * test * test * test test (#1597) Test versions validations (#1598) * test * basedir Test versions validations (#1599) * test * basedir * test test (#1600) * test * test Add release versions everywhere (#1601) * test * test * test * test test (#1602) Test version validations (#1603) * test * test Test version validations (#1604) * test * test * test tests (#1605) More tests nov16 (#1606) * tests * test More tests nov16 (#1607) * tests * test * test More tests nov16 (#1608) * tests * test * test * test More tests nov16 (#1609) * tests * test * test * test * test * fix_lint
diff --git a/.github/workflows/validate-aarch64-linux-binaries.yml b/.github/workflows/validate-aarch64-linux-binaries.yml
@@ -17,6 +17,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
   workflow_dispatch:
     inputs:
       channel:
@@ -38,6 +48,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
 
 jobs:
   generate-aarch64-linux-matrix:
@@ -47,7 +67,6 @@ jobs:
       os: linux-aarch64
       channel: ${{ inputs.channel }}
       with-cuda: disable
-
   linux-aarch64:
     needs: generate-aarch64-linux-matrix
     strategy:
@@ -72,6 +91,8 @@ jobs:
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TARGET_OS="linux-aarch64"
         export TORCH_ONLY=${{ inputs.torchonly }}
+        export RELEASE_VERSION=${{ inputs.version }}
+        printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json
         eval "$(conda shell.bash hook)"
 
         # Standart case: Validate binaries
diff --git a/.github/workflows/validate-binaries.yml b/.github/workflows/validate-binaries.yml
@@ -60,44 +60,69 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate'
+        default: ""
+        required: false
+        type: string
+
 
 jobs:
+  generate-release-matrix:
+    uses: pytorch/test-infra/.github/workflows/generate_release_matrix.yml@main
+    with:
+      version: ${{ inputs.version }}
+
   win:
     if:  inputs.os == 'windows' || inputs.os == 'all'
+    needs: generate-release-matrix
     uses: ./.github/workflows/validate-windows-binaries.yml
     with:
       channel: ${{ inputs.channel }}
       ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
+      version: ${{ inputs.version }}
+      release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
 
   linux:
     if:  inputs.os == 'linux' || inputs.os == 'all'
+    needs: generate-release-matrix
     uses: ./.github/workflows/validate-linux-binaries.yml
     with:
       channel: ${{ inputs.channel }}
       ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
+      version: ${{ inputs.version }}
+      release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
 
   linux-aarch64:
     if:  inputs.os == 'linux-aarch64'
+    needs: generate-release-matrix
     uses: ./.github/workflows/validate-aarch64-linux-binaries.yml
     with:
       channel: ${{ inputs.channel }}
       ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
-
+      version: ${{ inputs.version }}
+      release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
   mac:
     if:  inputs.os == 'macos' || inputs.os == 'all'
+    needs: generate-release-matrix
     uses: ./.github/workflows/validate-macos-binaries.yml
     with:
       channel: ${{ inputs.channel }}
       ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
+      version: ${{ inputs.version }}
+      release-matrix: ${{ needs.generate-rlease-matrix.outputs.matrix }}
 
   mac-arm64:
     if:  inputs.os == 'macos' || inputs.os == 'all'
+    needs: generate-release-matrix
     uses: ./.github/workflows/validate-macos-arm64-binaries.yml
     with:
       channel: ${{ inputs.channel }}
       ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
+      version: ${{ inputs.version }}
+      release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml
@@ -17,6 +17,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
   workflow_dispatch:
     inputs:
       channel:
@@ -38,6 +48,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
 
 jobs:
   generate-linux-matrix:
@@ -64,8 +84,11 @@ jobs:
         set -ex
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TORCH_ONLY=${{ inputs.torchonly }}
+        export RELEASE_VERSION=${{ inputs.version }}
         export TARGET_OS="linux"
         eval "$(conda shell.bash hook)"
+        printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json
+        cat release_matrix.json
 
         # Special case PyPi installation package. And Install of PyPi package via poetry
         if [[ ${MATRIX_PACKAGE_TYPE} == "manywheel" && ${MATRIX_GPU_ARCH_VERSION} == "12.1" ]]; then
diff --git a/.github/workflows/validate-macos-arm64-binaries.yml b/.github/workflows/validate-macos-arm64-binaries.yml
@@ -17,6 +17,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
   workflow_dispatch:
     inputs:
       channel:
@@ -38,6 +48,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
 
 jobs:
   generate-macos-arm64-matrix:
@@ -64,4 +84,6 @@ jobs:
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TARGET_OS="macos-arm64"
         export TORCH_ONLY=${{ inputs.torchonly }}
+        export RELEASE_VERSION=${{ inputs.version }}
+        printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json
         source ./.github/scripts/validate_binaries.sh
diff --git a/.github/workflows/validate-macos-binaries.yml b/.github/workflows/validate-macos-binaries.yml
@@ -17,6 +17,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
   workflow_dispatch:
     inputs:
       channel:
@@ -38,6 +48,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
 
 jobs:
   generate-macos-matrix:
@@ -64,4 +84,6 @@ jobs:
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TARGET_OS="macos"
         export TORCH_ONLY=${{ inputs.torchonly }}
+        export RELEASE_VERSION=${{ inputs.version }}
+        printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json
         source ./.github/scripts/validate_binaries.sh
diff --git a/.github/workflows/validate-windows-binaries.yml b/.github/workflows/validate-windows-binaries.yml
@@ -17,6 +17,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
   workflow_dispatch:
     inputs:
       channel:
@@ -38,6 +48,16 @@ on:
         default: false
         required: false
         type: boolean
+      version:
+        description: 'Version to validate - optional'
+        default: ""
+        required: false
+        type: string
+      release-matrix:
+        description: 'Release matrix - optional'
+        default: ""
+        required: false
+        type: string
 
 jobs:
   generate-windows-matrix:
@@ -46,7 +66,6 @@ jobs:
       package-type: all
       os: windows
       channel: ${{ inputs.channel }}
-
   win:
     needs: generate-windows-matrix
     strategy:
@@ -66,9 +85,10 @@ jobs:
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TARGET_OS="windows"
         export TORCH_ONLY=${{ inputs.torchonly }}
+        export RELEASE_VERSION=${{ inputs.version }}
+        printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json
         source /c/Jenkins/Miniconda3/etc/profile.d/conda.sh
         if [[ ${MATRIX_GPU_ARCH_VERSION} == "12.1" ]]; then
           ./windows/internal/driver_update.bat
         fi
-
         source ./.github/scripts/validate_binaries.sh
diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py
@@ -3,18 +3,20 @@
 import sys
 import argparse
 import torch
+import json
 import importlib
 import subprocess
 import torch._dynamo
 import torch.nn as nn
 import torch.nn.functional as F
+from pathlib import Path
 
 gpu_arch_ver = os.getenv("MATRIX_GPU_ARCH_VERSION")
 gpu_arch_type = os.getenv("MATRIX_GPU_ARCH_TYPE")
 channel = os.getenv("MATRIX_CHANNEL")
-stable_version = os.getenv("MATRIX_STABLE_VERSION")
 package_type = os.getenv("MATRIX_PACKAGE_TYPE")
 target_os = os.getenv("TARGET_OS")
+BASE_DIR =  Path(__file__).parent.parent.parent
 
 is_cuda_system = gpu_arch_type == "cuda"
 NIGHTLY_ALLOWED_DELTA = 3
@@ -52,8 +54,27 @@ def forward(self, x):
         output = self.fc1(x)
         return output
 
+def load_json_from_basedir(filename: str):
+    try:
+        with open(BASE_DIR / filename) as fptr:
+            return json.load(fptr)
+    except FileNotFoundError as exc:
+        raise ImportError(f"File {filename} not found error: {exc.strerror}") from exc
+    except json.JSONDecodeError as exc:
+        raise ImportError(f"Invalid JSON {filename}") from exc
+
+def read_release_matrix():
+    return load_json_from_basedir("release_matrix.json")
 
 def check_version(package: str) -> None:
+    release_version = os.getenv("RELEASE_VERSION")
+    # if release_version is specified, use it to validate the packages
+    if(release_version):
+        release_matrix = read_release_matrix()
+        stable_version = release_matrix["torch"]
+    else:
+        stable_version = os.getenv("MATRIX_STABLE_VERSION")
+
     # only makes sense to check nightly package where dates are known
     if channel == "nightly":
         check_nightly_binaries_date(package)
@@ -62,6 +83,20 @@ def check_version(package: str) -> None:
             raise RuntimeError(
                 f"Torch version mismatch, expected {stable_version} for channel {channel}. But its {torch.__version__}"
             )
+
+        if release_version and package == "all":
+            for module in MODULES:
+                imported_module = importlib.import_module(module["name"])
+                module_version = imported_module.__version__
+                if not module_version.startswith(release_matrix[module["name"]]):
+                    raise RuntimeError(
+                        f"{module['name']} version mismatch, expected: \
+                            {release_matrix[module['name']]} for channel {channel}. But its {module_version}"
+                    )
+                else:
+                     print(f"{module['name']} version actual: {module_version} expected: \
+                        {release_matrix[module['name']]} for channel {channel}.")
+
     else:
         print(f"Skip version check for channel {channel} as stable version is None")
 
@@ -255,6 +290,7 @@ def main() -> None:
     )
     options = parser.parse_args()
     print(f"torch: {torch.__version__}")
+
     check_version(options.package)
     smoke_test_conv2d()
     smoke_test_linalg()
