Add nightly testing to CUDA CI (#18078)

Author: carmocca

.azure/gpu-tests-fabric.yml

@@ -53,15 +53,23 @@ jobs:
       FREEZE_REQUIREMENTS: "1"
       PIP_CACHE_DIR: "/var/tmp/pip"
     container:
-      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.0-cuda11.7.1"
+      image: $(image)
       # default shm size is 64m. Increase it to avoid:
       # 'Error while creating shared memory: unhandled system error, NCCL version 2.7.8'
       options: "--gpus=all --shm-size=2gb  -v /var/tmp:/var/tmp"
     strategy:
       matrix:
-        'pkg: Fabric':
+        'Fabric | latest':
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.0-cuda11.7.1"
+          IS_NIGHTLY: "false"
           PACKAGE_NAME: "fabric"
-        'pkg: Lightning':
+        'Lightning | latest':
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.0-cuda11.7.1"
+          IS_NIGHTLY: "false"
+          PACKAGE_NAME: "lightning"
+        'Lightning | nightly':
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.0-cuda11.7.1"
+          IS_NIGHTLY: "true"
           PACKAGE_NAME: "lightning"
     workspace:
       clean: all
@@ -80,6 +88,7 @@ jobs:
         echo $CUDA_VISIBLE_DEVICES
         echo $CUDA_VERSION_MM
         echo $TORCH_URL
+        echo $(IS_NIGHTLY)
         echo $COVERAGE_SOURCE
         whereis nvidia
         nvidia-smi
@@ -96,13 +105,23 @@ jobs:
         for fpath in `ls requirements/**/*.txt`; do \
           python ./adjust-torch-versions.py $fpath ${PYTORCH_VERSION}; \
         done
+      # without succeeded this could run even if the job has already failed
+      condition: and(succeeded(), eq(variables.IS_NIGHTLY, 'false'))
       displayName: 'Adjust dependencies'
 
     - bash: |
         extra=$(python -c "print({'lightning': 'fabric-'}.get('$(PACKAGE_NAME)', ''))")
         pip install -e ".[${extra}dev]" pytest-timeout -U --find-links ${TORCH_URL}
       displayName: 'Install package & dependencies'
 
+    - bash: |
+        pip uninstall -y torch torchvision
+        pip install torch torchvision -U --pre --no-cache --index-url https://download.pytorch.org/whl/nightly/cu${CUDA_VERSION_MM%}
+        python -c "from torch import __version__ as ver; assert ver.startswith('2.1.0'), ver"
+      # without succeeded this could run even if the job has already failed
+      condition: and(succeeded(), eq(variables.IS_NIGHTLY, 'true'))
+      displayName: 'Bump to nightly'
+
     - bash: |
         set -e
         python requirements/collect_env_details.py

.azure/gpu-tests-pytorch.yml

@@ -53,11 +53,15 @@ jobs:
       matrix:
         'PyTorch | latest':
           image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.0-cuda11.7.1"
-          scope: ""
+          IS_NIGHTLY: "false"
           PACKAGE_NAME: "pytorch"
         'Lightning | latest':
           image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.0-cuda11.7.1"
-          scope: ""
+          IS_NIGHTLY: "false"
+          PACKAGE_NAME: "lightning"
+        'Lightning | nightly':
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.0-cuda11.7.1"
+          IS_NIGHTLY: "true"
           PACKAGE_NAME: "lightning"
     pool: lit-rtx-3090
     variables:
@@ -87,6 +91,7 @@ jobs:
         echo $CUDA_VISIBLE_DEVICES
         echo $CUDA_VERSION_MM
         echo $TORCH_URL
+        echo $(IS_NIGHTLY)
         echo $COVERAGE_SOURCE
         whereis nvidia
         nvidia-smi
@@ -103,18 +108,30 @@ jobs:
         for fpath in `ls requirements/**/*.txt`; do \
           python ./adjust-torch-versions.py $fpath ${PYTORCH_VERSION}; \
         done
-        # prune packages with installation issues
+      # without succeeded this could run even if the job has already failed
+      condition: and(succeeded(), eq(variables.IS_NIGHTLY, 'false'))
+      displayName: 'Adjust dependencies'
+
+    - bash: |
         pip install -q -r .actions/requirements.txt
         python .actions/assistant.py requirements_prune_pkgs \
           --packages="[lightning-colossalai,lightning-bagua]" \
           --req_files="[requirements/_integrations/strategies.txt]"
-      displayName: 'Adjust dependencies'
+      displayName: 'Prune packages'  # these have installation issues
 
     - bash: |
         extra=$(python -c "print({'lightning': 'pytorch-'}.get('$(PACKAGE_NAME)', ''))")
         pip install -e ".[${extra}dev]" -r requirements/_integrations/strategies.txt pytest-timeout -U --find-links ${TORCH_URL}
       displayName: 'Install package & dependencies'
 
+    - bash: |
+        pip uninstall -y torch torchvision
+        pip install torch torchvision -U --pre --no-cache --index-url https://download.pytorch.org/whl/nightly/cu${CUDA_VERSION_MM%}
+        python -c "from torch import __version__ as ver; assert ver.startswith('2.1.0'), ver"
+      # without succeeded this could run even if the job has already failed
+      condition: and(succeeded(), eq(variables.IS_NIGHTLY, 'true'))
+      displayName: 'Bump to nightly'
+
     - bash: pip uninstall -y lightning
       condition: eq(variables['PACKAGE_NAME'], 'pytorch')
       # Lightning is dependency of Habana or other accelerators/integrations so in case we test PL we need to remove it

examples/pytorch/basics/autoencoder.py

@@ -81,7 +81,7 @@ def _to_grid(self, images):
             nrow=self.nrow,
             padding=self.padding,
             normalize=self.normalize,
-            range=self.norm_range,
+            value_range=self.norm_range,
             scale_each=self.scale_each,
             pad_value=self.pad_value,
         )