Enabled dependency isolation by default (#1055)

gavin-aguiar · Gavin Aguiar · web-flow · commit b1085b5bbba2 · 2022-06-22T09:24:38.000-05:00
* Enabled dependency isolation by default

* Unit test fixes

* Fixed flake8 tests

* Enabled dependency isolation by default

* Unit test fixes

* Fixed flake8 tests

* Reverted loader.py change

* Added unit tests

* Added unit tests

* Reverted loader.py change

* Added copyright comment

Co-authored-by: Gavin Aguiar &lt;gavin.aguiar@microsoft.com&gt;
diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py
@@ -43,8 +43,7 @@
 PYTHON_THREADPOOL_THREAD_COUNT_MAX = sys.maxsize
 PYTHON_THREADPOOL_THREAD_COUNT_MAX_37 = 32
 
-PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT = False
-PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 = False
+PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT = True
 PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT = False
 PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39 = True
 
diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py
@@ -1,21 +1,21 @@
-from azure_functions_worker.utils.common import is_true_like
-from typing import List, Optional
-from types import ModuleType
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
 import importlib
 import inspect
 import os
 import re
 import sys
+from types import ModuleType
+from typing import List, Optional
 
-from ..logging import logger
+from azure_functions_worker.utils.common import is_true_like
 from ..constants import (
     AZURE_WEBJOBS_SCRIPT_ROOT,
     CONTAINER_NAME,
     PYTHON_ISOLATE_WORKER_DEPENDENCIES,
-    PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT,
-    PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310
+    PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT
 )
-from ..utils.common import is_python_version
+from ..logging import logger
 from ..utils.wrappers import enable_feature_by
 
 
@@ -75,12 +75,7 @@ def is_in_linux_consumption(cls):
     @classmethod
     @enable_feature_by(
         flag=PYTHON_ISOLATE_WORKER_DEPENDENCIES,
-        flag_default=(
-            PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 if
-            is_python_version('3.10') else
-            PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT
-        )
-    )
+        flag_default=PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT)
     def use_worker_dependencies(cls):
         """Switch the sys.path and ensure the worker imports are loaded from
         Worker's dependenciess.
@@ -106,12 +101,7 @@ def use_worker_dependencies(cls):
     @classmethod
     @enable_feature_by(
         flag=PYTHON_ISOLATE_WORKER_DEPENDENCIES,
-        flag_default=(
-            PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 if
-            is_python_version('3.10') else
-            PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT
-        )
-    )
+        flag_default=PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT)
     def prioritize_customer_dependencies(cls, cx_working_dir=None):
         """Switch the sys.path and ensure the customer's code import are loaded
         from CX's deppendencies.
@@ -180,11 +170,7 @@ def reload_customer_libraries(cls, cx_working_dir: str):
         """
         use_new_env = os.getenv(PYTHON_ISOLATE_WORKER_DEPENDENCIES)
         if use_new_env is None:
-            use_new = (
-                PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 if
-                is_python_version('3.10') else
-                PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT
-            )
+            use_new = PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT
         else:
             use_new = is_true_like(use_new_env)
 
diff --git a/tests/endtoend/dependency_isolation_functions/dependency_isolation_stein/function_app.py b/tests/endtoend/dependency_isolation_functions/dependency_isolation_stein/function_app.py
diff --git a/tests/unittests/test_rpc_messages.py b/tests/unittests/test_rpc_messages.py
@@ -8,7 +8,6 @@
 import unittest
 
 from azure_functions_worker import protos, testutils
-from azure_functions_worker.utils.common import is_python_version
 
 
 class TestGRPC(testutils.AsyncTestCase):
@@ -127,13 +126,7 @@ def test_failed_azure_namespace_import(self):
 
     @unittest.skipIf(sys.platform == 'win32',
                      'Linux .sh script only works on Linux')
-    @unittest.skipIf(
-        is_python_version('3.10'),
-        'In Python 3.10, isolate worker dependencies is turned on by default.'
-        ' Reloading all customer dependencies on specialization is a must.'
-        ' This partially reloading namespace feature is no longer needed.'
-    )
     def test_successful_azure_namespace_import(self):
         self._verify_azure_namespace_import(
             'true',
-            'module_b is imported')
+            'module_b fails to import')
diff --git a/tests/unittests/test_utilities_dependency.py b/tests/unittests/test_utilities_dependency.py
@@ -1,13 +1,12 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
+import importlib.util
 import os
 import sys
-import importlib.util
 import unittest
 from unittest.mock import patch
 
 from azure_functions_worker import testutils
-from azure_functions_worker.utils.common import is_python_version
 from azure_functions_worker.utils.dependency import DependencyManager
 
 
@@ -229,7 +228,7 @@ def test_add_to_sys_path_no_duplication(self):
 
     def test_add_to_sys_path_import_module(self):
         DependencyManager._add_to_sys_path(self._customer_deps_path, True)
-        import common_module # NoQA
+        import common_module  # NoQA
         self.assertEqual(
             common_module.package_location,
             os.path.join(self._customer_deps_path, 'common_module')
@@ -240,7 +239,7 @@ def test_add_to_sys_path_import_namespace_path(self):
         into sys.path
         """
         DependencyManager._add_to_sys_path(self._customer_deps_path, True)
-        import common_namespace # NoQA
+        import common_namespace  # NoQA
         self.assertEqual(len(common_namespace.__path__), 1)
         self.assertEqual(
             common_namespace.__path__[0],
@@ -517,7 +516,7 @@ def test_clear_path_importer_cache_and_modules_retain_namespace(self):
         sys.path.insert(0, self._worker_deps_path)
 
         # Ensure new import is from _worker_deps_path
-        import common_module as worker_mod # NoQA
+        import common_module as worker_mod  # NoQA
         self.assertIn('common_module', sys.modules)
         self.assertEqual(
             worker_mod.package_location,
@@ -555,39 +554,6 @@ def test_use_worker_dependencies_disable(self):
         with self.assertRaises(ImportError):
             import common_module  # NoQA
 
-    @unittest.skipUnless(
-        sys.version_info.major == 3 and sys.version_info.minor != 10,
-        'Test only available for Python 3.6, 3.7, 3.8 or 3.9'
-    )
-    def test_use_worker_dependencies_default_python_36_37_38_39(self):
-        # Feature should be disabled in Python 3.6, 3.7, 3.8 and 3.9
-        # Setup paths
-        DependencyManager.worker_deps_path = self._worker_deps_path
-        DependencyManager.cx_deps_path = self._customer_deps_path
-        DependencyManager.cx_working_dir = self._customer_func_path
-
-        # The common_module cannot be imported since feature is disabled
-        DependencyManager.use_worker_dependencies()
-        with self.assertRaises(ImportError):
-            import common_module  # NoQA
-
-    @unittest.skip('Skipping since PYTHON_ISOLATE_WORKER_DEPENDENCIES is '
-                   'disabled by default')
-    def test_use_worker_dependencies_default_python_310(self):
-        # Feature should be enabled in Python 3.10 by default
-        # Setup paths
-        DependencyManager.worker_deps_path = self._worker_deps_path
-        DependencyManager.cx_deps_path = self._customer_deps_path
-        DependencyManager.cx_working_dir = self._customer_func_path
-
-        # Ensure the common_module is imported from _worker_deps_path
-        DependencyManager.use_worker_dependencies()
-        import common_module  # NoQA
-        self.assertEqual(
-            common_module.package_location,
-            os.path.join(self._worker_deps_path, 'common_module')
-        )
-
     def test_prioritize_customer_dependencies(self):
         # Setup app settings
         os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true'
@@ -626,52 +592,51 @@ def test_prioritize_customer_dependencies_disable(self):
         with self.assertRaises(ImportError):
             import common_module  # NoQA
 
-    @unittest.skipIf(is_python_version('3.10'),
-                     'Test not available for python 3.10')
-    def test_prioritize_customer_dependencies_default_python_36_37_38_39(self):
-        # Feature should be disabled in Python 3.6, 3.7, 3.8 and 3.9
+    def test_prioritize_customer_dependencies_from_working_directory(self):
+        self._initialize_scenario()
+
         # Setup paths
         DependencyManager.worker_deps_path = self._worker_deps_path
         DependencyManager.cx_deps_path = self._customer_deps_path
         DependencyManager.cx_working_dir = self._customer_func_path
 
-        # Ensure the common_module is imported from _customer_deps_path
+        # Ensure the func_specific_module is imported from _customer_func_path
         DependencyManager.prioritize_customer_dependencies()
-        with self.assertRaises(ImportError):
-            import common_module  # NoQA
+        import func_specific_module  # NoQA
+        self.assertEqual(
+            func_specific_module.package_location,
+            os.path.join(self._customer_func_path, 'func_specific_module')
+        )
 
-    @unittest.skip('Skipping since PYTHON_ISOLATE_WORKER_DEPENDENCIES is '
-                   'disabled by default')
-    def test_prioritize_customer_dependencies_default_python_310(self):
-        # Feature should be enabled in Python 3.10 by default
+    def test_reload_customer_libraries_dependency_isolation_true(self):
+        os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true'
         # Setup paths
         DependencyManager.worker_deps_path = self._worker_deps_path
         DependencyManager.cx_deps_path = self._customer_deps_path
         DependencyManager.cx_working_dir = self._customer_func_path
 
-        # Ensure the common_module is imported from _customer_deps_path
-        DependencyManager.prioritize_customer_dependencies()
+        DependencyManager.reload_customer_libraries(self._customer_deps_path)
         import common_module  # NoQA
         self.assertEqual(
             common_module.package_location,
-            os.path.join(self._customer_deps_path, 'common_module')
-        )
-
-    def test_prioritize_customer_dependencies_from_working_directory(self):
-        self._initialize_scenario()
+            os.path.join(self._customer_deps_path, 'common_module'))
 
+    def test_reload_customer_libraries_dependency_isolation_false(self):
+        os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'false'
         # Setup paths
         DependencyManager.worker_deps_path = self._worker_deps_path
         DependencyManager.cx_deps_path = self._customer_deps_path
         DependencyManager.cx_working_dir = self._customer_func_path
 
-        # Ensure the func_specific_module is imported from _customer_func_path
-        DependencyManager.prioritize_customer_dependencies()
-        import func_specific_module  # NoQA
-        self.assertEqual(
-            func_specific_module.package_location,
-            os.path.join(self._customer_func_path, 'func_specific_module')
-        )
+        DependencyManager._add_to_sys_path(self._worker_deps_path, True)
+        import azure.functions  # NoQA
+
+        DependencyManager._add_to_sys_path(self._customer_deps_path, True)
+        DependencyManager.reload_customer_libraries(self._customer_deps_path)
+        # Checking if azure.functions gets reloaded
+        self.assertIn(
+            os.path.join(self._customer_deps_path, 'azure', 'functions'),
+            sys.modules['azure.functions'].__path__)
 
     def test_remove_module_cache(self):
         # First import the common_module and create a sys.modules cache
