feat: SDK bindings changes

.github/ISSUE_TEMPLATE/deferred_bindings_bug_report.yml

@@ -0,0 +1,71 @@
+name: Python Worker Deferred Bindings Feature Request
+description: File a Deferred Bindings bug report
+title: "[Bug] Bug Title Here"
+labels: ["python", "bug", "deferred-bindings"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        This form will help you to fill in a bug report for the Azure Functions Python Worker Deferred Bindings feature.
+
+  - type: textarea
+    id: expected-behavior
+    attributes:
+      label: Expected Behavior
+      description: A clear and concise description of what you expected to happen.
+      placeholder: What should have occurred?
+
+  - type: textarea
+    id: actual-behavior
+    attributes:
+      label: Actual Behavior
+      description: A clear and concise description of what actually happened.
+      placeholder: What went wrong?
+
+  - type: textarea
+    id: reproduction-steps
+    attributes:
+      label: Steps to Reproduce
+      description: Please provide detailed step-by-step instructions on how to reproduce the bug.
+      placeholder: |
+        1. Go to the [specific page or section] in the application.
+        2. Click on [specific button or link].
+        3. Scroll down to [specific location].
+        4. Observe [describe what you see, e.g., an error message or unexpected behavior].
+        5. Include any additional steps or details that may be relevant.
+
+  - type: textarea
+    id: code-snippet
+    attributes:
+      label: Relevant code being tried
+      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+
+  - type: textarea
+    id: requirements
+    attributes:
+      label: requirements.txt file
+      description: Please copy and paste your requirements.txt file. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+
+  - type: dropdown
+    id: environment
+    attributes:
+      label: Where are you facing this problem?
+      default: 0
+      options:
+        - Local - Core Tools
+        - Production Environment (explain below)
+
+  - type: textarea
+    id: additional-info
+    attributes:
+      label: Additional Information
+      description: Add any other information about the problem here.

.github/ISSUE_TEMPLATE/deferred_bindings_feature_request.yml

@@ -0,0 +1,36 @@
+name: Python Worker Deferred Bindings Feature Request
+description: File a Deferred Bindings Feature report
+title: "Request a feature"
+labels: ["python", "feature", "deferred-bindings"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        This form will help you to fill in a feature request for the Azure Functions Python Worker Deferred Bindings feature.
+
+  - type: textarea
+    id: binding-type
+    attributes:
+      label: Binding Type
+      description: Add information about the binding type.
+      placeholder: Is this on an existing binding or new binding?
+
+  - type: textarea
+    id: expected-behavior
+    attributes:
+      label: Expected Behavior
+      description: A clear and concise description of what you expected to happen.
+      placeholder: What should have occurred?
+
+  - type: textarea
+    id: code-snippet
+    attributes:
+      label: Relevant sample code snipped
+      description: Please copy and paste any relevant code snippet of how you want the feature to be used. (This will be automatically formatted into code, so no need for backticks)
+      render: shell
+
+  - type: textarea
+    id: additional-info
+    attributes:
+      label: Additional Information
+      description: Add any other information about the problem here.

.github/Scripts/e2e-tests.sh

@@ -1,3 +1,4 @@
 #!/usr/bin/env bash
 python -m pytest -q -n auto --dist loadfile --reruns 4 --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend/test_worker_process_count_functions.py tests/endtoend/test_threadpool_thread_count_functions.py
-python -m pytest -q -n auto --dist loadfile --reruns 4 --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append --ignore=tests/endtoend/test_worker_process_count_functions.py --ignore=tests/endtoend/test_threadpool_thread_count_functions.py tests/endtoend
+python -m pytest -q -n auto --dist loadfile --reruns 4 --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append --ignore=tests/endtoend/test_worker_process_count_functions.py --ignore=tests/endtoend/test_threadpool_thread_count_functions.py tests/endtoend
+python -m pytest -q -n auto --dist loadfile --reruns 4 --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/extension_tests/deferred_bindings_tests

.github/workflows/ci_consumption_workflow.yml

@@ -32,6 +32,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
+          python -m pip install --upgrade pip
           python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
           python -m pip install -U -e .[dev]
           python setup.py build

.github/workflows/ci_e2e_workflow.yml

@@ -15,9 +15,9 @@ on:
   pull_request:
     branches: [dev, main, release/*]
   schedule:
-    # Monday to Thursday 3 AM CST build
+    # Monday to Friday 3 AM CST build
     # * is a special character in YAML so you have to quote this string
-    - cron: "0 8 * * 1,2,3,4"
+    - cron: "0 8 * * 1,2,3,4,5"
 
 jobs:
   build:
@@ -61,6 +61,9 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
           python -m pip install -U -e .[dev]
+          if [[ "${{ matrix.python-version }}" != "3.7" && "${{ matrix.python-version }}" != "3.8" ]]; then
+              python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-deferred-bindings]
+          fi
 
           # Retry a couple times to avoid certificate issue
           retry 5 python setup.py build

.github/workflows/ci_ut_workflow.yml

@@ -58,7 +58,7 @@ jobs:
 
         python -m pip install --upgrade pip
         python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
-        python -m pip install -U -e .[dev] 
+        python -m pip install -U -e .[dev]
 
         # Retry a couple times to avoid certificate issue
         retry 5 python setup.py build

azure_functions_worker/bindings/__init__.py

@@ -8,7 +8,8 @@
 from .meta import has_implicit_output
 from .meta import is_trigger_binding, load_binding_registry
 from .meta import from_incoming_proto, to_outgoing_proto, \
-    to_outgoing_param_binding
+    to_outgoing_param_binding, check_deferred_bindings_enabled, \
+    get_deferred_raw_bindings
 from .out import Out
 
 
@@ -19,5 +20,6 @@
     'check_input_type_annotation', 'check_output_type_annotation',
     'has_implicit_output',
     'from_incoming_proto', 'to_outgoing_proto', 'TraceContext', 'RetryContext',
-    'to_outgoing_param_binding'
+    'to_outgoing_param_binding', 'check_deferred_bindings_enabled',
+    'get_deferred_raw_bindings'
 )

azure_functions_worker/bindings/datumdef.py

@@ -93,6 +93,8 @@ def from_typed_data(cls, td: protos.TypedData):
             val = td.collection_string
         elif tt == 'collection_sint64':
             val = td.collection_sint64
+        elif tt == 'model_binding_data':
+            val = td.model_binding_data
         elif tt is None:
             return None
         else:

azure_functions_worker/bindings/meta.py

@@ -1,39 +1,83 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
+import os
 import sys
 import typing
 
 from .. import protos
 
 from . import datumdef
 from . import generic
+
 from .shared_memory_data_transfer import SharedMemoryManager
+from ..constants import CUSTOMER_PACKAGES_PATH
+from ..logging import logger
 
 PB_TYPE = 'rpc_data'
 PB_TYPE_DATA = 'data'
 PB_TYPE_RPC_SHARED_MEMORY = 'rpc_shared_memory'
+# Base extension supported Python minor version
+BASE_EXT_SUPPORTED_PY_MINOR_VERSION = 8
+
 BINDING_REGISTRY = None
+DEFERRED_BINDING_REGISTRY = None
+deferred_bindings_cache = {}
 
 
 def load_binding_registry() -> None:
+    """
+    Tries to load azure-functions from the customer's BYO. If it's
+    not found, it loads the builtin. If the BINDING_REGISTRY is None,
+    azure-functions hasn't been loaded in properly.
+
+    Tries to load the base extension only for python 3.8+.
+    """
+
     func = sys.modules.get('azure.functions')
 
-    # If fails to acquire customer's BYO azure-functions, load the builtin
     if func is None:
         import azure.functions as func
 
     global BINDING_REGISTRY
     BINDING_REGISTRY = func.get_binding_registry()
 
-
-def get_binding(bind_name: str) -> object:
+    if BINDING_REGISTRY is None:
+        raise AttributeError('BINDING_REGISTRY is None. azure-functions '
+                             'library not found. Sys Path: %s. '
+                             'Sys Modules: %s. '
+                             'python-packages Path exists: %s.',
+                             sys.path, sys.modules,
+                             os.path.exists(CUSTOMER_PACKAGES_PATH))
+
+    if sys.version_info.minor >= BASE_EXT_SUPPORTED_PY_MINOR_VERSION:
+        try:
+            import azurefunctions.extensions.base as clients
+            global DEFERRED_BINDING_REGISTRY
+            DEFERRED_BINDING_REGISTRY = clients.get_binding_registry()
+        except ImportError:
+            logger.debug('Base extension not found. '
+                         'Python version: 3.%s, Sys path: %s, '
+                         'Sys Module: %s, python-packages Path exists: %s.',
+                         sys.version_info.minor, sys.path,
+                         sys.modules, os.path.exists(CUSTOMER_PACKAGES_PATH))
+
+
+def get_binding(bind_name: str,
+                is_deferred_binding: typing.Optional[bool] = False)\
+        -> object:
+    """
+    First checks if the binding is a non-deferred binding. This is
+    the most common case.
+    Second checks if the binding is a deferred binding.
+    If the binding is neither, it's a generic type.
+    """
     binding = None
-    registry = BINDING_REGISTRY
-    if registry is not None:
-        binding = registry.get(bind_name)
+    if binding is None and not is_deferred_binding:
+        binding = BINDING_REGISTRY.get(bind_name)
+    if binding is None and is_deferred_binding:
+        binding = DEFERRED_BINDING_REGISTRY.get(bind_name)
     if binding is None:
         binding = generic.GenericBinding
-
     return binding
 
 
@@ -42,8 +86,10 @@ def is_trigger_binding(bind_name: str) -> bool:
     return binding.has_trigger_support()
 
 
-def check_input_type_annotation(bind_name: str, pytype: type) -> bool:
-    binding = get_binding(bind_name)
+def check_input_type_annotation(bind_name: str,
+                                pytype: type,
+                                is_deferred_binding: bool) -> bool:
+    binding = get_binding(bind_name, is_deferred_binding)
     return binding.check_input_type_annotation(pytype)
 
 
@@ -71,8 +117,9 @@ def from_incoming_proto(
         pb: protos.ParameterBinding, *,
         pytype: typing.Optional[type],
         trigger_metadata: typing.Optional[typing.Dict[str, protos.TypedData]],
-        shmem_mgr: SharedMemoryManager) -> typing.Any:
-    binding = get_binding(binding)
+        shmem_mgr: SharedMemoryManager,
+        is_deferred_binding: typing.Optional[bool] = False) -> typing.Any:
+    binding = get_binding(binding, is_deferred_binding)
     if trigger_metadata:
         metadata = {
             k: datumdef.Datum.from_typed_data(v)
@@ -93,6 +140,13 @@ def from_incoming_proto(
         raise TypeError(f'Unknown ParameterBindingType: {pb_type}')
 
     try:
+        # if the binding is an sdk type binding
+        if is_deferred_binding:
+            return deferred_bindings_decode(binding=binding,
+                                            pb=pb,
+                                            pytype=pytype,
+                                            datum=datum,
+                                            metadata=metadata)
         return binding.decode(datum, trigger_metadata=metadata)
     except NotImplementedError:
         # Binding does not support the data.
@@ -184,3 +238,61 @@ def to_outgoing_param_binding(binding: str, obj: typing.Any, *,
         return protos.ParameterBinding(
             name=out_name,
             data=rpc_val)
+
+
+def deferred_bindings_decode(binding: typing.Any,
+                             pb: protos.ParameterBinding, *,
+                             pytype: typing.Optional[type],
+                             datum: typing.Any,
+                             metadata: typing.Any):
+    """
+    This cache holds deferred binding types (ie. BlobClient, ContainerClient)
+    That have already been created, so that the worker can reuse the
+    Previously created type without creating a new one.
+
+    If cache is empty or key doesn't exist, deferred_binding_type is None
+    """
+    global deferred_bindings_cache
+
+    if deferred_bindings_cache.get((pb.name,
+                                    pytype,
+                                    datum.value.content), None) is not None:
+        return deferred_bindings_cache.get((pb.name,
+                                            pytype,
+                                            datum.value.content))
+    else:
+        deferred_binding_type = binding.decode(datum,
+                                               trigger_metadata=metadata,
+                                               pytype=pytype)
+        deferred_bindings_cache[(pb.name,
+                                 pytype,
+                                 datum.value.content)] = deferred_binding_type
+        return deferred_binding_type
+
+
+def check_deferred_bindings_enabled(param_anno: type,
+                                    deferred_bindings_enabled: bool) -> (bool,
+                                                                         bool):
+    """
+    Checks if deferred bindings is enabled at fx and single binding level
+
+    The first bool represents if deferred bindings is enabled at a fx level
+    The second represents if the current binding is deferred binding
+    """
+    if (DEFERRED_BINDING_REGISTRY is not None
+            and DEFERRED_BINDING_REGISTRY.check_supported_type(param_anno)):
+        return True, True
+    else:
+        return deferred_bindings_enabled, False
+
+
+def get_deferred_raw_bindings(indexed_function, input_types):
+    """
+    Calls a method from the base extension that generates the raw bindings
+    for a given function. It also returns logs for that function including
+    the defined binding type and if deferred bindings is enabled for that
+    binding.
+    """
+    raw_bindings, bindings_logs = DEFERRED_BINDING_REGISTRY.get_raw_bindings(
+        indexed_function, input_types)
+    return raw_bindings, bindings_logs