lint + public build

Author: hallvictoria

.flake8

@@ -7,7 +7,7 @@ ignore = W503,E402,E731
 
 exclude = .git, __pycache__, build, dist, .eggs, .github, .local, docs/,
           Samples, azure_functions_worker/protos/,
-          azure_functions_worker/_thirdparty/typing_inspect.py,
+          azure_functions_worker/utils/typing_inspect.py,
           tests/unittests/test_typing_inspect.py,
           tests/unittests/broken_functions/syntax_error/main.py,
           tests/protos/,

CODEOWNERS

@@ -10,4 +10,4 @@
 # For all file changes, github would automatically
 # include the following people in the PRs.
 
-* @vrdmr @gavin-aguiar @YunchuWang @pdthummar @hallvictoria
+* @vrdmr @gavin-aguiar @hallvictoria

azure_functions_worker/__init__.py

@@ -1,6 +1,14 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
-from .handle_event import worker_init_request, functions_metadata_request, function_environment_reload_request, invocation_request, functions_load_request
+from .handle_event import (worker_init_request,
+                           functions_metadata_request,
+                           function_environment_reload_request,
+                           invocation_request,
+                           functions_load_request)
 
-__all__ = ('worker_init_request', 'functions_metadata_request', 'function_environment_reload_request', 'invocation_request', 'functions_load_request')
+__all__ = ('worker_init_request',
+           'functions_metadata_request',
+           'function_environment_reload_request',
+           'invocation_request',
+           'functions_load_request')

azure_functions_worker/bindings/context.py

@@ -50,7 +50,7 @@ def retry_context(self) -> RetryContext:
 
 
 def get_context(invoc_request, name: str,
-                 directory: str) -> Context:
+                directory: str) -> Context:
     """ For more information refer:
     https://aka.ms/azfunc-invocation-context
     """

azure_functions_worker/bindings/datumdef.py

@@ -14,8 +14,6 @@
     to_nullable_timestamp,
 )
 
-from ..logging import logger
-
 try:
     from http.cookies import SimpleCookie
 except ImportError:
@@ -69,8 +67,9 @@ def __repr__(self):
     def from_typed_data(cls, protos):
         try:
             td = protos.TypedData
-        except Exception as ex:
-            # Todo: better catch for Datum.from_typed_data(http.body) -- if the data being sent in is already protos.TypedData
+        except Exception:
+            # Todo: better catch for Datum.from_typed_data(http.body)
+            # if the data being sent in is already protos.TypedData
             td = protos
         tt = td.WhichOneof('data')
         if tt == 'http':
@@ -116,7 +115,7 @@ def from_typed_data(cls, protos):
         return cls(val, tt)
 
 
-def datum_as_proto(datum: Datum,  protos):
+def datum_as_proto(datum: Datum, protos):
     if datum.type == 'string':
         return protos.TypedData(string=datum.value)
     elif datum.type == 'bytes':

azure_functions_worker/functions.py

@@ -136,7 +136,8 @@ def is_context_required(params, bound_params: dict,
     def validate_function_params(params: dict, bound_params: dict,
                                  annotations: dict, func_name: str,
                                  protos):
-        logger.info("Params: %s, BoundParams: %s, Annotations: %s, FuncName: %s", params, bound_params, annotations, func_name)
+        logger.info("Params: %s, BoundParams: %s, Annotations: %s, FuncName: %s",
+                    params, bound_params, annotations, func_name)
         if set(params) - set(bound_params):
             raise FunctionLoadError(
                 func_name,

azure_functions_worker/handle_event.py

@@ -6,7 +6,6 @@
 import os
 import sys
 
-from datetime import datetime
 from typing import List, Optional
 
 from .functions import FunctionInfo, Registry
@@ -22,7 +21,9 @@
 from .otel import otel_manager, initialize_azure_monitor, configure_opentelemetry
 
 from .bindings.context import get_context
-from .bindings.meta import load_binding_registry, is_trigger_binding, from_incoming_proto, to_outgoing_param_binding, to_outgoing_proto
+from .bindings.meta import (load_binding_registry, is_trigger_binding,
+                            from_incoming_proto, to_outgoing_param_binding,
+                            to_outgoing_proto)
 from .bindings.out import Out
 from .utils.constants import (FUNCTION_DATA_CACHE,
                               RAW_HTTP_BODY_BYTES,
@@ -56,7 +57,6 @@
 protos = None
 
 
-# Protos will be the retry / binding / metadata protos object that we populate and return
 async def worker_init_request(request):
     logger.info("Library Worker: received worker_init_request")
     global result, _host, protos, _function_data_cache_enabled
@@ -83,14 +83,14 @@ async def worker_init_request(request):
         if otel_manager.get_azure_monitor_available():
             capabilities[WORKER_OPEN_TELEMETRY_ENABLED] = TRUE
 
-
     # loading bindings registry and saving results to a static
     # dictionary which will be later used in the invocation request
     load_binding_registry()
 
     try:
-        result = asyncio.create_task(load_function_metadata(init_request.function_app_directory,
-                                                            caller_info="worker_init_request"))
+        result = asyncio.create_task(load_function_metadata(
+            init_request.function_app_directory,
+            caller_info="worker_init_request"))
         if get_app_setting(setting=PYTHON_ENABLE_INIT_INDEXING):
             capabilities[HTTP_URI] = \
                 initialize_http_server(_host)
@@ -130,11 +130,10 @@ async def functions_metadata_request(request):
 
     else:
         return protos.FunctionMetadataResponse(
-                    use_default_metadata_indexing=False,
-                    function_metadata_results=metadata_result,
-                    result=protos.StatusResult(
-                        status=protos.StatusResult.Success))
-
+            use_default_metadata_indexing=False,
+            function_metadata_results=metadata_result,
+            result=protos.StatusResult(
+                status=protos.StatusResult.Success))
 
 
 async def functions_load_request(request):
@@ -152,7 +151,6 @@ async def functions_load_request(request):
 async def invocation_request(request):
     logger.info("Library Worker: received worker_invocation_request")
     global protos
-    invocation_time = datetime.now()
     invoc_request = request.request.invocation_request
     invocation_id = invoc_request.invocation_id
     function_id = invoc_request.function_id
@@ -256,22 +254,22 @@ async def invocation_request(request):
         # Actively flush customer print() function to console
         sys.stdout.flush()
         return protos.InvocationResponse(
-                    invocation_id=invocation_id,
-                    return_value=return_value,
-                    result=protos.StatusResult(
-                        status=protos.StatusResult.Success),
-                    output_data=output_data)
+            invocation_id=invocation_id,
+            return_value=return_value,
+            result=protos.StatusResult(
+                status=protos.StatusResult.Success),
+            output_data=output_data)
 
     except Exception as ex:
         if http_v2_enabled:
             http_coordinator.set_http_response(invocation_id, ex)
         global metadata_result
         metadata_result = ex
         return protos.InvocationResponse(
-                    invocation_id=invocation_id,
-                    result=protos.StatusResult(
-                        status=protos.StatusResult.Failure,
-                        exception=serialize_exception(ex)))
+            invocation_id=invocation_id,
+            result=protos.StatusResult(
+                status=protos.StatusResult.Failure,
+                exception=serialize_exception(ex)))
 
 
 async def function_environment_reload_request(request):
@@ -308,9 +306,10 @@ async def function_environment_reload_request(request):
             global _host, result, protos
             _host = request.properties.get("host")
             protos = request.properties.get("protos")
-            result = asyncio.create_task(load_function_metadata(directory,
-                                                                caller_info="environment_reload_request"))
-            if get_app_setting(setting=PYTHON_ENABLE_INIT_INDEXING):  # PYTHON_ENABLE_HTTP_STREAMING
+            result = asyncio.create_task(load_function_metadata(
+                directory,
+                caller_info="environment_reload_request"))
+            if get_app_setting(setting=PYTHON_ENABLE_INIT_INDEXING):
                 capabilities[HTTP_URI] = \
                     initialize_http_server(_host)
                 capabilities[REQUIRES_ROUTE_PARAMETERS] = TRUE
@@ -324,18 +323,18 @@ async def function_environment_reload_request(request):
                 func_env_reload_request.function_app_directory)
 
         return protos.FunctionEnvironmentReloadResponse(
-                capabilities=capabilities,
-                worker_metadata=get_worker_metadata(protos),
-                result=protos.StatusResult(
-                    status=protos.StatusResult.Success))
+            capabilities=capabilities,
+            worker_metadata=get_worker_metadata(protos),
+            result=protos.StatusResult(
+                status=protos.StatusResult.Success))
 
     except Exception as ex:
         global metadata_exception
         metadata_exception = ex
         return protos.FunctionEnvironmentReloadResponse(
-                result=protos.StatusResult(
-                    status=protos.StatusResult.Failure,
-                    exception=serialize_exception(ex)))
+            result=protos.StatusResult(
+                status=protos.StatusResult.Failure,
+                exception=serialize_exception(ex)))
 
 
 async def load_function_metadata(function_app_directory, caller_info):

azure_functions_worker/loader.py

@@ -9,7 +9,7 @@
 import time
 
 from datetime import timedelta
-from typing import Dict, Optional, Union
+from typing import Dict, Optional
 
 
 from .functions import Registry

azure_functions_worker/utils/__init__.py

@@ -1,2 +1,2 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License.
+# Licensed under the MIT License.

azure_functions_worker/utils/current.py

@@ -28,4 +28,3 @@ def run_sync_func(invocation_id, context, func, params):
         return result(params)
     finally:
         context.thread_local_storage.invocation_id = None
-

azure_functions_worker/utils/env_state.py

@@ -1,7 +1,6 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 import os
-import sys
 from typing import Callable, Optional
 
 

azure_functions_worker/utils/helpers.py

@@ -26,4 +26,4 @@ def get_worker_metadata(protos):
                         f"{sys.version_info.minor}",
         worker_version=VERSION,
         worker_bitness=platform.machine(),
-        custom_properties={})
+        custom_properties={})

azure_functions_worker/utils/tracing.py

@@ -53,4 +53,4 @@ def serialize_exception(exc: Exception, protos):
     except Exception:
         stack_trace = ''
 
-    return protos.RpcException(message=message, stack_trace=stack_trace)
+    return protos.RpcException(message=message, stack_trace=stack_trace)

azure_functions_worker/utils/validators.py

@@ -3,8 +3,6 @@
 
 import re
 
-from .constants import PYTHON_THREADPOOL_THREAD_COUNT, PYTHON_THREADPOOL_THREAD_COUNT_MIN
-
 
 class InvalidFileNameError(Exception):
 

eng/ci/public-build.yml

@@ -28,14 +28,6 @@ resources:
 
 variables:
   - template: /eng/templates/utils/variables.yml@self
-  - name: codeql.excludePathPatterns
-    value: deps/,build/
-  - name: codeql.compiled.enabled
-    value: true
-  - name: codeql.runSourceLanguagesInSourceAnalysis
-    value: true
-  - name: codeql.sourceLanguages
-    value: python, powershell
 
 extends:
   template: v1/1ES.Unofficial.PipelineTemplate.yml@1es
@@ -44,19 +36,23 @@ extends:
       name: 1es-pool-azfunc-public
       image: 1es-windows-2022
       os: windows
+
+    sdl:
+      codeql:
+        compiled:
+          enabled: true # still only runs for default branch
+        sourceLanguages: python, powershell
+        excludePathPatterns: deps/,build/
+        runSourceLanguagesInSourceAnalysis: true
+
     settings:
       skipBuildTagsForGitHubPullRequests: ${{ variables['System.PullRequest.IsFork'] }}
+
     stages:
     - stage: Build
       jobs:
       - template: /eng/templates/jobs/build.yml@self
     - stage: RunUnitTests
       dependsOn: Build
       jobs:
-        - template: /eng/templates/jobs/ci-unit-tests.yml@self
-    - stage: RunEmulatorTests
-      dependsOn: Build
-      jobs:
-        - template: /eng/templates/jobs/ci-emulator-tests.yml@self
-          parameters:
-            PoolName: 1es-pool-azfunc-public
+        - template: /eng/templates/jobs/ci-unit-tests.yml@self

eng/templates/jobs/build.yml

@@ -7,16 +7,29 @@ jobs:
       image: 1es-ubuntu-22.04
       os: linux
 
+    variables:
+      # Extract the version number from the branch name
+      pythonVersion: $[substring(variables['Build.SourceBranchName'], 4, 2)]  # Extract last two digits from dev-xxx
+
     steps:
+      - script: |
+          echo "Branch name: $(Build.SourceBranchName)"
+          # Extract the last two digits (minor version) from the branch name
+          version=$(echo $(Build.SourceBranchName) | sed 's/dev-\([0-9]*\)/\1/')
+          minor_version=${version: -2}  # Get last two digits
+          echo "Extracted minor version: $minor_version"
+          echo "##vso[task.setvariable variable=pythonVersion]$minor_version"
+        displayName: 'Extract Python version from branch name'
       - task: UsePythonVersion@0
         inputs:
-          versionSpec: "3.13"
+          versionSpec: '3.$(pythonVersion)'
       - bash: |
           python --version
         displayName: 'Check python version'
       - bash: |
           python -m pip install --upgrade pip
-          python -m pip install .
+          python -m pip install build
+          python -m build
         displayName: 'Build python worker'
       - bash: |
           pip install pip-audit