diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh
index 88a76de86..c6a8afffc 100755
--- a/.github/scripts/validate_binaries.sh
+++ b/.github/scripts/validate_binaries.sh
@@ -6,6 +6,11 @@ else
     conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
     conda activate ${ENV_NAME}
     INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"}
+
+    # Make sure we remove previous installation if it exists
+    if [[ ${MATRIX_PACKAGE_TYPE} == 'wheel' ]]; then
+        pip3 uninstall -y torch torchaudio torchvision
+    fi
     eval $INSTALLATION
 
     if [[ ${TARGET_OS} == 'linux' ]]; then
@@ -14,7 +19,12 @@ else
         ${PWD}/check_binary.sh
     fi
 
-    python  ./test/smoke_test/smoke_test.py
+    if [[ ${TARGET_OS} == 'windows' ]]; then
+        python  ./test/smoke_test/smoke_test.py
+    else
+        python3  ./test/smoke_test/smoke_test.py
+    fi
+
     conda deactivate
     conda env remove -n ${ENV_NAME}
 fi
diff --git a/.github/workflows/validate-nightly-binaries.yml b/.github/workflows/validate-nightly-binaries.yml
index 4f59d3e9f..f3f54180b 100644
--- a/.github/workflows/validate-nightly-binaries.yml
+++ b/.github/workflows/validate-nightly-binaries.yml
@@ -17,15 +17,7 @@ on:
       - .github/workflows/validate-macos-binaries.yml
       - .github/workflows/validate-macos-arm64-binaries.yml
       - test/smoke_test/*
-  pull_request:
-    paths:
-      - .github/workflows/validate-nightly-binaries.yml
-      - .github/workflows/validate-linux-binaries.yml
-      - .github/workflows/validate-windows-binaries.yml
-      - .github/workflows/validate-macos-binaries.yml
-      - .github/workflows/validate-macos-arm64-binaries.yml
-      - .github/scripts/validate_binaries.sh
-      - test/smoke_test/*
+
 jobs:
   nightly:
     uses: ./.github/workflows/validate-binaries.yml
diff --git a/.github/workflows/validate-release-binaries.yml b/.github/workflows/validate-release-binaries.yml
index 73a8071ee..eb77e7d53 100644
--- a/.github/workflows/validate-release-binaries.yml
+++ b/.github/workflows/validate-release-binaries.yml
@@ -17,6 +17,15 @@ on:
       - .github/workflows/validate-macos-binaries.yml
       - .github/workflows/validate-macos-arm64-binaries.yml
       - test/smoke_test/*
+  pull_request:
+    paths:
+      - .github/workflows/validate-nightly-binaries.yml
+      - .github/workflows/validate-linux-binaries.yml
+      - .github/workflows/validate-windows-binaries.yml
+      - .github/workflows/validate-macos-binaries.yml
+      - .github/workflows/validate-macos-arm64-binaries.yml
+      - .github/scripts/validate_binaries.sh
+      - test/smoke_test/*
 
 jobs:
   release:
diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py
index 34aaabb8e..3ae4d9421 100644
--- a/test/smoke_test/smoke_test.py
+++ b/test/smoke_test/smoke_test.py
@@ -16,6 +16,7 @@
 channel = os.getenv("MATRIX_CHANNEL")
 stable_version = os.getenv("MATRIX_STABLE_VERSION")
 package_type = os.getenv("MATRIX_PACKAGE_TYPE")
+target_os = os.getenv("TARGET_OS")
 
 is_cuda_system = gpu_arch_type == "cuda"
 NIGHTLY_ALLOWED_DELTA = 3
@@ -24,14 +25,14 @@
     {
         "name": "torchvision",
         "repo": "https://github.com/pytorch/vision.git",
-        "smoke_test": "python ./vision/test/smoke_test.py",
+        "smoke_test": "./vision/test/smoke_test.py",
         "extension": "extension",
         "repo_name": "vision",
     },
     {
         "name": "torchaudio",
         "repo": "https://github.com/pytorch/audio.git",
-        "smoke_test": "python ./audio/test/smoke_test/smoke_test.py --no-ffmpeg",
+        "smoke_test": "./audio/test/smoke_test/smoke_test.py --no-ffmpeg",
         "extension": "_extension",
         "repo_name": "audio",
     },
@@ -212,8 +213,11 @@ def smoke_test_modules():
                 print(f"Path does not exist: {cwd}/{module['repo_name']}")
                 subprocess.check_output(f"git clone --depth 1 {module['repo']}", stderr=subprocess.STDOUT, shell=True)
             try:
+                smoke_test_command = f"python3 {module['smoke_test']}"
+                if target_os == 'windows':
+                    smoke_test_command = f"python {module['smoke_test']}"
                 output = subprocess.check_output(
-                    module["smoke_test"], stderr=subprocess.STDOUT, shell=True,
+                    smoke_test_command, stderr=subprocess.STDOUT, shell=True,
                     universal_newlines=True)
             except subprocess.CalledProcessError as exc:
                 raise RuntimeError(