Enable nightly 3.12 for all domains (#4843)

Cherry Pick of #4834
Add 3.12 to test environment

---------

Co-authored-by: Huy Do <[email protected]>

Author: atalman

tools/scripts/generate_binary_build_matrix.py

@@ -21,8 +21,10 @@
 mod = sys.modules[__name__]
 
 PYTHON_ARCHES_DICT = {
-    "nightly": ["3.8", "3.9", "3.10", "3.11"],
-    "test": ["3.8", "3.9", "3.10", "3.11"],
+    # TODO (huydhn): 3.12 is only enabled in nightly for now, test and release
+    # will come later
+    "nightly": ["3.8", "3.9", "3.10", "3.11", "3.12"],
+    "test": ["3.8", "3.9", "3.10", "3.11", "3.12"],
     "release": ["3.8", "3.9", "3.10", "3.11"],
 }
 CUDA_ARCHES_DICT = {
@@ -463,12 +465,9 @@ def generate_wheels_matrix(
 ) -> List[Dict[str, str]]:
     package_type = "wheel"
 
-    # Define default python version
-    if python_versions is None and channel == NIGHTLY:
-        # Python 3.12 is added to the nightly wheel matrix only
-        python_versions = list(mod.PYTHON_ARCHES) + ["3.12"]
-    elif python_versions is None:
-         python_versions = list(mod.PYTHON_ARCHES)
+    if python_versions is None:
+        # Define default python version
+        python_versions = list(mod.PYTHON_ARCHES)
 
     if os == LINUX:
         # NOTE: We only build manywheel packages for linux

tools/tests/assets/build_matrix_linux_conda_cuda.json

@@ -1 +1 @@
-{"include": [{"python_version": "3.8", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_8-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_8-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1", "package_type": "conda", "build_name": "conda-py3_8-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_9-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_9-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1", "package_type": "conda", "build_name": "conda-py3_9-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_10-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_10-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1", "package_type": "conda", "build_name": "conda-py3_10-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_11-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_11-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1", "package_type": "conda", "build_name": "conda-py3_11-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}]}
+{"include": [{"python_version": "3.8", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_8-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_8-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1", "package_type": "conda", "build_name": "conda-py3_8-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_9-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_9-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1", "package_type": "conda", "build_name": "conda-py3_9-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_10-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_10-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1", "package_type": "conda", "build_name": "conda-py3_10-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_11-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_11-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1", "package_type": "conda", "build_name": "conda-py3_11-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_12-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_12-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1", "package_type": "conda", "build_name": "conda-py3_12-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}]}

tools/tests/assets/build_matrix_macos_conda.json

@@ -1 +1 @@
-{"include": [{"python_version": "3.8", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_8-cpu", "validation_runner": "macos-12", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}, {"python_version": "3.9", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_9-cpu", "validation_runner": "macos-12", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}, {"python_version": "3.10", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_10-cpu", "validation_runner": "macos-12", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}, {"python_version": "3.11", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_11-cpu", "validation_runner": "macos-12", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}]}
+{"include": [{"python_version": "3.8", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_8-cpu", "validation_runner": "macos-12", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}, {"python_version": "3.9", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_9-cpu", "validation_runner": "macos-12", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}, {"python_version": "3.10", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_10-cpu", "validation_runner": "macos-12", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}, {"python_version": "3.11", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_11-cpu", "validation_runner": "macos-12", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}, {"python_version": "3.12", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_12-cpu", "validation_runner": "macos-12", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}]}