Configurable file name (#1323)

hallvictoria · vrdmr · gavin-aguiar · web-flow · commit 0b21dc02b563 · 2024-01-11T12:58:46.000-06:00
* configurable file name

* lint &amp; error fix

* test fix

* fix merge lint errors

* e2e and loader test

* test linting errors

* test linting errors

* e2e file name test

* test lint checks

* testing fixes

* moved tests to new file

* fixed last test

* file name, conciseness

* additional tests

* slight test changes

* updated tests

* fixing tests

* refactored tests

* lint

* moved invalid stein tests to broken tests file

* fixed test check

* sep broken stein tests

* renamed, removed prints

* edited logs to now also show script file name

* formatting

* increase timeout

* env vars method

* printing env variables for failing tests

* cls stop env

* class methods

* setting file name explicitly

* reset script file name after test ends

* lint + misc fixes

* refactor tests

* added extra tests

* debugging logs

* global var

* added method to testutils

* validate file name + tests

* fixed tests

* copyright

---------

Co-authored-by: Varad Meru &lt;vrdmr@users.noreply.github.com&gt;
Co-authored-by: gavin-aguiar &lt;80794152+gavin-aguiar@users.noreply.github.com&gt;
diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py
@@ -41,11 +41,13 @@
 PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39 = True
 PYTHON_EXTENSIONS_RELOAD_FUNCTIONS = "PYTHON_EXTENSIONS_RELOAD_FUNCTIONS"
 
+# new programming model default script file name
+PYTHON_SCRIPT_FILE_NAME = "PYTHON_SCRIPT_FILE_NAME"
+PYTHON_SCRIPT_FILE_NAME_DEFAULT = "function_app.py"
+
 # External Site URLs
 MODULE_NOT_FOUND_TS_URL = "https://aka.ms/functions-modulenotfound"
 
-# new programming model script file name
-SCRIPT_FILE_NAME = "function_app.py"
 PYTHON_LANGUAGE_RUNTIME = "python"
 
 # Settings for V2 programming model
diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py
@@ -28,14 +28,16 @@
                         PYTHON_THREADPOOL_THREAD_COUNT_MAX_37,
                         PYTHON_THREADPOOL_THREAD_COUNT_MIN,
                         PYTHON_ENABLE_DEBUG_LOGGING,
-                        SCRIPT_FILE_NAME,
+                        PYTHON_SCRIPT_FILE_NAME,
+                        PYTHON_SCRIPT_FILE_NAME_DEFAULT,
                         PYTHON_LANGUAGE_RUNTIME, CUSTOMER_PACKAGES_PATH)
 from .extension import ExtensionManager
 from .logging import disable_console_logging, enable_console_logging
 from .logging import (logger, error_logger, is_system_log_category,
                       CONSOLE_LOG_PREFIX, format_exception)
 from .utils.app_setting_manager import get_python_appsetting_state
-from .utils.common import get_app_setting, is_envvar_true
+from .utils.common import (get_app_setting, is_envvar_true,
+                           validate_script_file_name)
 from .utils.dependency import DependencyManager
 from .utils.tracing import marshall_exception_trace
 from .utils.wrappers import disable_feature_by
@@ -327,24 +329,29 @@ async def _handle__worker_status_request(self, request):
     async def _handle__functions_metadata_request(self, request):
         metadata_request = request.functions_metadata_request
         directory = metadata_request.function_app_directory
-        function_path = os.path.join(directory, SCRIPT_FILE_NAME)
+        script_file_name = get_app_setting(
+            setting=PYTHON_SCRIPT_FILE_NAME,
+            default_value=f'{PYTHON_SCRIPT_FILE_NAME_DEFAULT}')
+        function_path = os.path.join(directory, script_file_name)
 
         logger.info(
-            'Received WorkerMetadataRequest, request ID %s, directory: %s',
-            self.request_id, directory)
-
-        if not os.path.exists(function_path):
-            # Fallback to legacy model
-            logger.info("%s does not exist. "
-                        "Switching to host indexing.", SCRIPT_FILE_NAME)
-            return protos.StreamingMessage(
-                request_id=request.request_id,
-                function_metadata_response=protos.FunctionMetadataResponse(
-                    use_default_metadata_indexing=True,
-                    result=protos.StatusResult(
-                        status=protos.StatusResult.Success)))
+            'Received WorkerMetadataRequest, request ID %s, function_path: %s',
+            self.request_id, function_path)
 
         try:
+            validate_script_file_name(script_file_name)
+
+            if not os.path.exists(function_path):
+                # Fallback to legacy model
+                logger.info("%s does not exist. "
+                            "Switching to host indexing.", script_file_name)
+                return protos.StreamingMessage(
+                    request_id=request.request_id,
+                    function_metadata_response=protos.FunctionMetadataResponse(
+                        use_default_metadata_indexing=True,
+                        result=protos.StatusResult(
+                            status=protos.StatusResult.Success)))
+
             fx_metadata_results = self.index_functions(function_path)
 
             return protos.StreamingMessage(
@@ -367,8 +374,6 @@ async def _handle__function_load_request(self, request):
         function_id = func_request.function_id
         function_metadata = func_request.metadata
         function_name = function_metadata.name
-        function_path = os.path.join(function_metadata.directory,
-                                     SCRIPT_FILE_NAME)
 
         logger.info(
             'Received WorkerLoadRequest, request ID %s, function_id: %s,'
@@ -377,6 +382,14 @@ async def _handle__function_load_request(self, request):
         programming_model = "V1"
         try:
             if not self._functions.get_function(function_id):
+                script_file_name = get_app_setting(
+                    setting=PYTHON_SCRIPT_FILE_NAME,
+                    default_value=f'{PYTHON_SCRIPT_FILE_NAME_DEFAULT}')
+                validate_script_file_name(script_file_name)
+                function_path = os.path.join(
+                    function_metadata.directory,
+                    script_file_name)
+
                 if function_metadata.properties.get("worker_indexed", False) \
                         or os.path.exists(function_path):
                     # This is for the second worker and above where the worker
diff --git a/azure_functions_worker/loader.py b/azure_functions_worker/loader.py
@@ -17,8 +17,10 @@
 
 from . import protos, functions
 from .bindings.retrycontext import RetryPolicy
-from .constants import MODULE_NOT_FOUND_TS_URL, SCRIPT_FILE_NAME, \
-    PYTHON_LANGUAGE_RUNTIME, RETRY_POLICY, CUSTOMER_PACKAGES_PATH
+from .utils.common import get_app_setting
+from .constants import MODULE_NOT_FOUND_TS_URL, PYTHON_SCRIPT_FILE_NAME, \
+    PYTHON_SCRIPT_FILE_NAME_DEFAULT, PYTHON_LANGUAGE_RUNTIME, \
+    CUSTOMER_PACKAGES_PATH, RETRY_POLICY
 from .logging import logger
 from .utils.wrappers import attach_message_to_exception
 
@@ -225,7 +227,10 @@ def index_function_app(function_path: str):
                     f"level function app instances are defined.")
 
     if not app:
+        script_file_name = get_app_setting(
+            setting=PYTHON_SCRIPT_FILE_NAME,
+            default_value=f'{PYTHON_SCRIPT_FILE_NAME_DEFAULT}')
         raise ValueError("Could not find top level function app instances in "
-                         f"{SCRIPT_FILE_NAME}.")
+                         f"{script_file_name}.")
 
     return app.get_functions()
diff --git a/azure_functions_worker/utils/app_setting_manager.py b/azure_functions_worker/utils/app_setting_manager.py
@@ -10,7 +10,8 @@
                          PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT,
                          PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39,
                          PYTHON_ENABLE_DEBUG_LOGGING,
-                         FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED)
+                         FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED,
+                         PYTHON_SCRIPT_FILE_NAME)
 
 
 def get_python_appsetting_state():
@@ -21,7 +22,8 @@ def get_python_appsetting_state():
          PYTHON_ISOLATE_WORKER_DEPENDENCIES,
          PYTHON_ENABLE_DEBUG_LOGGING,
          PYTHON_ENABLE_WORKER_EXTENSIONS,
-         FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED]
+         FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED,
+         PYTHON_SCRIPT_FILE_NAME]
 
     app_setting_states = "".join(
         f"{app_setting}: {current_vars[app_setting]} | "
diff --git a/azure_functions_worker/utils/common.py b/azure_functions_worker/utils/common.py
@@ -3,6 +3,7 @@
 import importlib
 import os
 import sys
+import re
 from types import ModuleType
 from typing import Optional, Callable
 
@@ -136,3 +137,20 @@ def get_sdk_from_sys_path() -> ModuleType:
         sys.path.insert(0, CUSTOMER_PACKAGES_PATH)
 
     return importlib.import_module('azure.functions')
+
+
+class InvalidFileNameError(Exception):
+
+    def __init__(self, file_name: str) -> None:
+        super().__init__(
+            f'Invalid file name: {file_name}')
+
+
+def validate_script_file_name(file_name: str):
+    # First character can be a letter, number, or underscore
+    # Following characters can be a letter, number, underscore, hyphen, or dash
+    # Ending must be .py
+    pattern = re.compile(r'^[a-zA-Z0-9_][a-zA-Z0-9_\-]*\.py$')
+    if not pattern.match(file_name):
+        raise InvalidFileNameError(file_name)
+    return True
diff --git a/tests/endtoend/http_functions/http_functions_stein/file_name/main.py b/tests/endtoend/http_functions/http_functions_stein/file_name/main.py
@@ -0,0 +1,44 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from datetime import datetime
+import logging
+import time
+
+import azure.functions as func
+
+app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS)
+
+
+@app.route(route="default_template")
+def default_template(req: func.HttpRequest) -> func.HttpResponse:
+    logging.info('Python HTTP trigger function processed a request.')
+
+    name = req.params.get('name')
+    if not name:
+        try:
+            req_body = req.get_json()
+        except ValueError:
+            pass
+        else:
+            name = req_body.get('name')
+
+    if name:
+        return func.HttpResponse(
+            f"Hello, {name}. This HTTP triggered function "
+            f"executed successfully.")
+    else:
+        return func.HttpResponse(
+            "This HTTP triggered function executed successfully. "
+            "Pass a name in the query string or in the request body for a"
+            " personalized response.",
+            status_code=200
+        )
+
+
+@app.route(route="http_func")
+def http_func(req: func.HttpRequest) -> func.HttpResponse:
+    time.sleep(1)
+
+    current_time = datetime.now().strftime("%H:%M:%S")
+    return func.HttpResponse(f"{current_time}")
diff --git a/tests/endtoend/test_file_name_functions.py b/tests/endtoend/test_file_name_functions.py
@@ -0,0 +1,109 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+import os
+import requests
+
+from azure_functions_worker.constants import PYTHON_SCRIPT_FILE_NAME
+from tests.utils import testutils
+
+REQUEST_TIMEOUT_SEC = 10
+
+
+class TestHttpFunctionsFileName(testutils.WebHostTestCase):
+    """Test the native Http Trigger in the local webhost.
+
+    This test class will spawn a webhost from your <project_root>/build/webhost
+    folder and replace the built-in Python with azure_functions_worker from
+    your code base. Since the Http Trigger is a native suport from host, we
+    don't need to setup any external resources.
+
+    Compared to the unittests/test_http_functions.py, this file is more focus
+    on testing the E2E flow scenarios.
+    """
+    @classmethod
+    def get_script_name(cls):
+        return "main.py"
+
+    @classmethod
+    def get_script_dir(cls):
+        return testutils.E2E_TESTS_FOLDER / 'http_functions' / \
+                                            'http_functions_stein' / \
+                                            'file_name'
+
+    @testutils.retryable_test(3, 5)
+    def test_index_page_should_return_ok(self):
+        """The index page of Azure Functions should return OK in any
+        circumstances
+        """
+        r = self.webhost.request('GET', '', no_prefix=True,
+                                 timeout=REQUEST_TIMEOUT_SEC)
+        self.assertTrue(r.ok)
+
+    @testutils.retryable_test(3, 5)
+    def test_default_http_template_should_return_ok(self):
+        """Test if the default template of Http trigger in Python Function app
+        will return OK
+        """
+        r = self.webhost.request('GET', 'default_template',
+                                 timeout=REQUEST_TIMEOUT_SEC)
+        self.assertTrue(r.ok)
+
+    @testutils.retryable_test(3, 5)
+    def test_default_http_template_should_accept_query_param(self):
+        """Test if the azure.functions SDK is able to deserialize query
+        parameter from the default template
+        """
+        r = self.webhost.request('GET', 'default_template',
+                                 params={'name': 'query'},
+                                 timeout=REQUEST_TIMEOUT_SEC)
+        self.assertTrue(r.ok)
+        self.assertEqual(
+            r.content,
+            b'Hello, query. This HTTP triggered function executed successfully.'
+        )
+
+    @testutils.retryable_test(3, 5)
+    def test_default_http_template_should_accept_body(self):
+        """Test if the azure.functions SDK is able to deserialize http body
+        and pass it to default template
+        """
+        r = self.webhost.request('POST', 'default_template',
+                                 data='{ "name": "body" }'.encode('utf-8'),
+                                 timeout=REQUEST_TIMEOUT_SEC)
+        self.assertTrue(r.ok)
+        self.assertEqual(
+            r.content,
+            b'Hello, body. This HTTP triggered function executed successfully.'
+        )
+
+    @testutils.retryable_test(3, 5)
+    def test_worker_status_endpoint_should_return_ok(self):
+        """Test if the worker status endpoint will trigger
+        _handle__worker_status_request and sends a worker status response back
+        to host
+        """
+        root_url = self.webhost._addr
+        health_check_url = f'{root_url}/admin/host/ping'
+        r = requests.post(health_check_url,
+                          params={'checkHealth': '1'},
+                          timeout=REQUEST_TIMEOUT_SEC)
+        self.assertTrue(r.ok)
+
+    @testutils.retryable_test(3, 5)
+    def test_worker_status_endpoint_should_return_ok_when_disabled(self):
+        """Test if the worker status endpoint will trigger
+        _handle__worker_status_request and sends a worker status response back
+        to host
+        """
+        os.environ['WEBSITE_PING_METRICS_SCALE_ENABLED'] = '0'
+        root_url = self.webhost._addr
+        health_check_url = f'{root_url}/admin/host/ping'
+        r = requests.post(health_check_url,
+                          params={'checkHealth': '1'},
+                          timeout=REQUEST_TIMEOUT_SEC)
+        self.assertTrue(r.ok)
+
+    def test_correct_file_name(self):
+        self.assertIsNotNone(os.environ.get(PYTHON_SCRIPT_FILE_NAME))
+        self.assertEqual(os.environ.get(PYTHON_SCRIPT_FILE_NAME),
+                         'main.py')
diff --git a/tests/unittests/broken_functions/invalid_app_stein/function_app.py b/tests/unittests/broken_functions/invalid_app_stein/function_app.py
@@ -0,0 +1,5 @@
+import azure.functions as func
+
+
+def main(req: func.HttpRequest):
+    pass
diff --git a/tests/unittests/file_name_functions/default_file_name/function_app.py b/tests/unittests/file_name_functions/default_file_name/function_app.py
@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import azure.functions as func
+
+app = func.FunctionApp()
+
+
+@app.route(route="return_str")
+def return_str(req: func.HttpRequest) -> str:
+    return 'Hello World!'
diff --git a/tests/unittests/file_name_functions/invalid_file_name/main b/tests/unittests/file_name_functions/invalid_file_name/main
@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import azure.functions as func
+
+app = func.FunctionApp()
+
+
+@app.route(route="return_str")
+def return_str(req: func.HttpRequest) -> str:
+    return 'Hello World!'
diff --git a/tests/unittests/file_name_functions/new_file_name/test.py b/tests/unittests/file_name_functions/new_file_name/test.py
@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import azure.functions as func
+
+app = func.FunctionApp()
+
+
+@app.route(route="return_str")
+def return_str(req: func.HttpRequest) -> str:
+    return 'Hello World!'
diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py
@@ -20,9 +20,6 @@
 DISPATCHER_STEIN_FUNCTIONS_DIR = testutils.UNIT_TESTS_FOLDER / \
     'dispatcher_functions' / \
     'dispatcher_functions_stein'
-DISPATCHER_STEIN_INVALID_FUNCTIONS_DIR = testutils.UNIT_TESTS_FOLDER / \
-    'broken_functions' / \
-    'invalid_stein'
 
 
 class TestThreadPoolSettingsPython37(testutils.AsyncTestCase):
diff --git a/tests/unittests/test_invalid_stein.py b/tests/unittests/test_invalid_stein.py
diff --git a/tests/unittests/test_loader.py b/tests/unittests/test_loader.py
diff --git a/tests/unittests/test_script_file_name.py b/tests/unittests/test_script_file_name.py
diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py
diff --git a/tests/utils/testutils.py b/tests/utils/testutils.py

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +import azure.functions as func
++
++
 +def main(req: func.HttpRequest):
 +    pass