ignore

.github/ISSUE_TEMPLATE/sdk_bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+---
+
+## SDK Bindings Bug Report
+This is a bug report for the newly added SDK bindings feature for Python Azure Functions.
+
+- **Package Name**: 
+- **Package Version**: 
+- **Operating System**:
+- **Python Version**: 
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. 
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/sdk_feature_request.md

@@ -0,0 +1,23 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+## SDK Bindings Feature Request
+This is a feature request for the newly added SDK bindings feature for Python Azure Functions.
+
+- **Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+- **Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+- **Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+- **Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/ci_deferred_bindings_workflow.yml

@@ -0,0 +1,123 @@
+# This workflow will install Python dependencies and run end to end tests with single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: CI Deferred Bindings tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      archive_webhost_logging:
+        description: "For debugging purposes, archive test webhost logs"
+        required: false
+        default: "false"
+  push:
+    branches: [dev, main, release/*]
+  pull_request:
+    branches: [dev, main, release/*]
+  schedule:
+    # Monday to Thursday 1 AM PDT build
+    # * is a special character in YAML so you have to quote this string
+    - cron: "0 8 * * 1,2,3,4"
+
+jobs:
+  build:
+    name: "Python Deferred Bindings CI Run"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.9, "3.10", "3.11"]
+    permissions: read-all
+    steps:
+      - name: Checkout code.
+        uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Set up Dotnet 6.x
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: "6.x"
+      - name: Install dependencies and the worker
+        run: |
+          retry() {
+            local -r -i max_attempts="$1"; shift
+            local -r cmd="$@"
+            local -i attempt_num=1
+            until $cmd
+            do
+              if (( attempt_num == max_attempts ))
+              then
+                echo "Attempt $attempt_num failed and there are no more attempts left!"
+                return 1
+              else
+                echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
+                sleep 1
+              fi
+            done
+          }
+
+          python -m pip install --upgrade pip
+          python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
+          python -m pip install -U -e .[dev]
+          python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[deferred-bindings]
+
+          # Retry a couple times to avoid certificate issue
+          retry 5 python setup.py build
+          retry 5 python setup.py webhost --branch-name=dev
+          retry 5 python setup.py extension --test-type=deferred-bindings
+          mkdir logs
+      - name: Running 3.9 Tests
+        if: matrix.python-version == 3.9
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString39 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString39 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString39 }}
+          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString39 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString39 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }}
+          ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
+        run: |
+          python -m pytest -q -n auto --dist loadfile --reruns 4 --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/extension_tests/deferred_bindings_tests
+      - name: Running 3.10 Tests
+        if: matrix.python-version == 3.10
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString310 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString310 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString310 }}
+          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString310 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }}
+          ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
+        run: |
+          python -m pytest -q -n auto --dist loadfile --reruns 4 --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/extension_tests/deferred_bindings_tests
+      - name: Running 3.11 Tests
+        if: matrix.python-version == 3.11
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString311 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString311 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString311 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString311 }}
+          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString311 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString311 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString311 }}
+          ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
+        run: |
+          python -m pytest -q -n auto --dist loadfile --reruns 4 --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/extension_tests/deferred_bindings_tests
+      - name: Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml # optional
+          flags: unittests # optional
+          name: codecov # optional
+          fail_ci_if_error: false # optional (default = false)
+      - name: Publish Logs to Artifact
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Test WebHost Logs ${{ github.run_id }} ${{ matrix.python-version }}
+          path: logs/*.log
+          if-no-files-found: ignore

azure_functions_worker/bindings/datumdef.py

@@ -93,6 +93,8 @@ def from_typed_data(cls, td: protos.TypedData):
             val = td.collection_string
         elif tt == 'collection_sint64':
             val = td.collection_sint64
+        elif tt == 'model_binding_data':
+            val = td.model_binding_data
         elif tt is None:
             return None
         else:

azure_functions_worker/bindings/meta.py

@@ -4,6 +4,7 @@
 import typing
 
 from .. import protos
+from ..constants import BASE_EXT_SUPPORTED_PY_MINOR_VERSION
 
 from . import datumdef
 from . import generic
@@ -13,6 +14,9 @@
 PB_TYPE_DATA = 'data'
 PB_TYPE_RPC_SHARED_MEMORY = 'rpc_shared_memory'
 BINDING_REGISTRY = None
+SDK_BINDING_REGISTRY = None
+DEFERRED_BINDINGS_ENABLED = False
+SDK_CACHE = {}
 
 
 def load_binding_registry() -> None:
@@ -25,12 +29,25 @@ def load_binding_registry() -> None:
     global BINDING_REGISTRY
     BINDING_REGISTRY = func.get_binding_registry()
 
-
-def get_binding(bind_name: str) -> object:
-    binding = None
-    registry = BINDING_REGISTRY
-    if registry is not None:
-        binding = registry.get(bind_name)
+    # The SDKs only support python 3.8+
+    if sys.version_info.minor >= BASE_EXT_SUPPORTED_PY_MINOR_VERSION:
+        # Import the base extension
+        try:
+            import azure.functions.extension.base as clients
+            global SDK_BINDING_REGISTRY
+            SDK_BINDING_REGISTRY = clients.get_binding_registry()
+        except ImportError:
+            # This will throw a ModuleNotFoundError in env reload because
+            # azure.functions.extension isn't loaded in
+            pass
+
+
+def get_binding(bind_name: str, pytype: typing.Optional[type] = None) -> object:
+    binding = get_deferred_binding(bind_name=bind_name, pytype=pytype)
+
+    # Either cx didn't import library or didn't define sdk type
+    if binding is None and BINDING_REGISTRY is not None:
+        binding = BINDING_REGISTRY.get(bind_name)
     if binding is None:
         binding = generic.GenericBinding
 
@@ -43,7 +60,8 @@ def is_trigger_binding(bind_name: str) -> bool:
 
 
 def check_input_type_annotation(bind_name: str, pytype: type) -> bool:
-    binding = get_binding(bind_name)
+    # check that needs to pass for sdk bindings -- pass in pytype
+    binding = get_binding(bind_name, pytype)
     return binding.check_input_type_annotation(pytype)
 
 
@@ -72,7 +90,7 @@ def from_incoming_proto(
         pytype: typing.Optional[type],
         trigger_metadata: typing.Optional[typing.Dict[str, protos.TypedData]],
         shmem_mgr: SharedMemoryManager) -> typing.Any:
-    binding = get_binding(binding)
+    binding = get_binding(binding, pytype)
     if trigger_metadata:
         metadata = {
             k: datumdef.Datum.from_typed_data(v)
@@ -93,6 +111,14 @@ def from_incoming_proto(
         raise TypeError(f'Unknown ParameterBindingType: {pb_type}')
 
     try:
+        # if the binding is an sdk type binding
+        if (SDK_BINDING_REGISTRY is not None
+                and SDK_BINDING_REGISTRY.check_supported_type(pytype)):
+            return deferred_bindings_decode(binding=binding,
+                                            pb=pb,
+                                            pytype=pytype,
+                                            datum=datum,
+                                            metadata=metadata)
         return binding.decode(datum, trigger_metadata=metadata)
     except NotImplementedError:
         # Binding does not support the data.
@@ -184,3 +210,41 @@ def to_outgoing_param_binding(binding: str, obj: typing.Any, *,
         return protos.ParameterBinding(
             name=out_name,
             data=rpc_val)
+
+
+def get_deferred_binding(bind_name: str,
+                         pytype: typing.Optional[type] = None) -> object:
+    binding = None
+
+    # Checks if pytype is a supported sdk type
+    if (SDK_BINDING_REGISTRY is not None
+            and SDK_BINDING_REGISTRY.check_supported_type(pytype)):
+        # Set flag once
+        global DEFERRED_BINDINGS_ENABLED
+        if not DEFERRED_BINDINGS_ENABLED:
+            DEFERRED_BINDINGS_ENABLED = True
+        binding = SDK_BINDING_REGISTRY.get(bind_name)
+
+    # This will return None if not a supported type
+    return binding
+
+
+def deferred_bindings_decode(binding: typing.Any,
+                             pb: protos.ParameterBinding, *,
+                             pytype: typing.Optional[type],
+                             datum: typing.Any,
+                             metadata: typing.Any):
+    global SDK_CACHE
+    # Check is the object is already in the cache
+    # If dict is empty or key doesn't exist, obj is None
+    obj = SDK_CACHE.get((pb.name, pytype, datum.value.content), None)
+
+    # If the object is in the cache, return it
+    if obj is not None:
+        return obj
+    # If the object is not in the cache, create and add it to the cache
+    else:
+        obj = binding.decode(datum, trigger_metadata=metadata,
+                             pytype=pytype)
+        SDK_CACHE[(pb.name, pytype, datum.value.content)] = obj
+        return obj

azure_functions_worker/constants.py

@@ -56,6 +56,9 @@
 # Paths
 CUSTOMER_PACKAGES_PATH = "/home/site/wwwroot/.python_packages/lib/site-packages"
 
+# Base extension supported Python minor version
+BASE_EXT_SUPPORTED_PY_MINOR_VERSION = 8
+
 # Flag to index functions in handle init request
 PYTHON_ENABLE_INIT_INDEXING = "PYTHON_ENABLE_INIT_INDEXING"
 

azure_functions_worker/loader.py

@@ -14,7 +14,7 @@
 
 from google.protobuf.duration_pb2 import Duration
 
-from . import protos, functions
+from . import protos, functions, bindings
 from .bindings.retrycontext import RetryPolicy
 from .utils.common import get_app_setting
 from .constants import MODULE_NOT_FOUND_TS_URL, PYTHON_SCRIPT_FILE_NAME, \
@@ -130,6 +130,9 @@ def process_indexed_function(functions_registry: functions.Registry,
         binding_protos = build_binding_protos(indexed_function)
         retry_protos = build_retry_protos(indexed_function)
 
+        raw_bindings = get_fx_raw_bindings(indexed_function=indexed_function,
+                                           function_info=function_info)
+
         function_metadata = protos.RpcFunctionMetadata(
             name=function_info.name,
             function_id=function_info.function_id,
@@ -140,7 +143,7 @@ def process_indexed_function(functions_registry: functions.Registry,
             is_proxy=False,  # not supported in V4
             language=PYTHON_LANGUAGE_RUNTIME,
             bindings=binding_protos,
-            raw_bindings=indexed_function.get_raw_bindings(),
+            raw_bindings=raw_bindings,
             retry_options=retry_protos,
             properties={METADATA_PROPERTIES_WORKER_INDEXED: "True"})
 
@@ -239,3 +242,17 @@ def index_function_app(function_path: str):
                          f"{script_file_name}.")
 
     return app.get_functions()
+
+
+def get_fx_raw_bindings(indexed_function, function_info):
+    # Check if deferred bindings is enabled
+    if (bindings.meta is not None
+            and bindings.meta.DEFERRED_BINDINGS_ENABLED
+            and bindings.meta.SDK_BINDING_REGISTRY is not None):
+        # Reset the flag
+        bindings.meta.DEFERRED_BINDINGS_ENABLED = False
+        return bindings.meta.SDK_BINDING_REGISTRY.get_raw_bindings(
+            indexed_function, function_info.input_types)
+
+    else:
+        return indexed_function.get_raw_bindings()

azure_functions_worker/logging.py

@@ -13,7 +13,6 @@
 SDK_LOG_PREFIX = "azure.functions"
 SYSTEM_ERROR_LOG_PREFIX = "azure_functions_worker_errors"
 
-
 logger: logging.Logger = logging.getLogger(SYSTEM_LOG_PREFIX)
 error_logger: logging.Logger = (
     logging.getLogger(SYSTEM_ERROR_LOG_PREFIX))