diff --git a/aarch64_linux/aarch64_wheel_ci_build.py b/aarch64_linux/aarch64_wheel_ci_build.py
index 3b772847c..d3910f227 100755
--- a/aarch64_linux/aarch64_wheel_ci_build.py
+++ b/aarch64_linux/aarch64_wheel_ci_build.py
@@ -105,9 +105,6 @@ def parse_arguments():
     else:
         print("build pytorch without mkldnn backend")
 
-    # work around to fix Raspberry pie crash
-    print("Applying mkl-dnn patch to fix readdir crash")
-    os.system("cd /pytorch/third_party/ideep/mkl-dnn && patch -p1 < /builder/mkldnn_fix/aarch64-fix-readdir-crash.patch")
     os.system(f"cd /pytorch; {build_vars} python3 setup.py bdist_wheel")
     pytorch_wheel_name = complete_wheel("pytorch")
     print(f"Build Compelete. Created {pytorch_wheel_name}..")
diff --git a/aarch64_linux/build_aarch64_wheel.py b/aarch64_linux/build_aarch64_wheel.py
index 9efd2e6ae..d4fa6f8ad 100755
--- a/aarch64_linux/build_aarch64_wheel.py
+++ b/aarch64_linux/build_aarch64_wheel.py
@@ -554,7 +554,6 @@ def start_build(host: RemoteHost, *,
         build_ArmComputeLibrary(host, git_clone_flags)
         print("build pytorch with mkldnn+acl backend")
         build_vars += " USE_MKLDNN=ON USE_MKLDNN_ACL=ON"
-        host.run_cmd(f"cd $HOME && git clone https://github.com/pytorch/builder.git")
         host.run_cmd(f"cd $HOME/pytorch && export ACL_ROOT_DIR=$HOME/ComputeLibrary && {build_vars} python3 setup.py bdist_wheel{build_opts}")
         print('Repair the wheel')
         pytorch_wheel_name = host.list_dir("pytorch/dist")[0]
diff --git a/mkldnn_fix/aarch64-fix-readdir-crash.patch b/mkldnn_fix/aarch64-fix-readdir-crash.patch
deleted file mode 100644
index 81d46d406..000000000
--- a/mkldnn_fix/aarch64-fix-readdir-crash.patch
+++ /dev/null
@@ -1,14 +0,0 @@
-diff --git a/src/cpu/aarch64/xbyak_aarch64/src/util_impl.cpp b/src/cpu/aarch64/xbyak_aarch64/src/util_impl.cpp
-index cb800b2509..5516373b90 100644
---- a/src/cpu/aarch64/xbyak_aarch64/src/util_impl.cpp
-+++ b/src/cpu/aarch64/xbyak_aarch64/src/util_impl.cpp
-@@ -170,6 +170,8 @@ int Cpu::getFilePathMaxTailNumPlus1(const char *path) {
-   fflush(stdout);
- 
-   DIR *dir = opendir(dir_path);
-+  if (dir == NULL)
-+    return 0;
-   struct dirent *dp;
- 
-   dp = readdir(dir);
-