feat: add exclude serilog project option to az func databricks jobmet…

…rics project template (#687) * feat: add exclude serilog project option to az func databricks jobmetrics project template * pr-fix: update with correct task * Update template.json * Update template.json * Update template.json * pr-fix: more stable databricks project startup * pr-fix: more stable startup for all az func projects * pr-fix: more stable startup wait for all az func projects
arcus-azure · Oct 6, 2022 · f74d1ff · f74d1ff
1 parent ae28413
commit f74d1ff
Show file tree

Hide file tree

Showing 16 changed files with 188 additions and 47 deletions.
diff --git a/docs/preview/features/azurefunctions-databricks-jobmetrics-template.md b/docs/preview/features/azurefunctions-databricks-jobmetrics-template.md
@@ -24,3 +24,9 @@ When installed, the template can be created with shortname: `arcus-az-databricks
 Creates a starter worker project with by default configured:
 
 * Azure Function timer that for each cycle the finished Databricks job runs reports as metrics ([official docs](https://background-jobs.arcus-azure.net/features/databricks/gain-insights))
+* Dockerfile
+
+### Configuration
+
+And additional features available with options:
+* `-es|--exclude-serilog`: Exclude the [Serilog](https://serilog.net/) logging infrastructure in the Azure Functions project which includes default enrichers ([version](https://observability.arcus-azure.net/features/telemetry-enrichment#version-enricher) and [application](https://observability.arcus-azure.net/features/telemetry-enrichment#application-enricher)), and sinking to Application Insights.
diff --git a/src/Arcus.Templates.AzureFunctions.Databricks.JobMetrics/.template.config/ide.host.json b/src/Arcus.Templates.AzureFunctions.Databricks.JobMetrics/.template.config/ide.host.json
@@ -9,6 +9,12 @@
   "learnMoreLink": "https://templates.arcus-azure.net/features/azurefunctions-databricks-jobmetrics-template",
   "icon": "icon.jpg",
   "symbolInfo": [
-
+    {
+      "id": "exclude-serilog",
+      "name": {
+        "text": "Exclude Serilog"
+      },
+      "isVisible": true
+    }
   ]
 }
diff --git a/src/Arcus.Templates.AzureFunctions.Databricks.JobMetrics/.template.config/template.json b/src/Arcus.Templates.AzureFunctions.Databricks.JobMetrics/.template.config/template.json
@@ -54,6 +54,24 @@
         "value": "#endif"
       },
       "replaces": "//[#endif]"
+    },
+    "ErrorDirective": {
+      "type": "generated",
+      "generator": "constant",
+      "parameters": {
+        "value": "#error"
+      },
+      "replaces": "//#error"
+    },
+    "exclude-serilog": {
+      "type": "parameter",
+      "datatype": "bool",
+      "defaultValue": "false",
+      "description": "Exclude the Serilog logging infrastructure in the Azure Functions project"
+    },
+    "Serilog_AppInsights": {
+      "type": "computed",
+      "value": "!(exclude-serilog)"
     }
   },
   "postActions": [
@@ -69,7 +87,7 @@
       "args": {
         "referenceType": "package",
         "reference": "Microsoft.NET.Sdk.Functions",
-        "version": "4.0.1",
+        "version": "4.0.13",
         "projectFileExtensions": ".csproj"
       }
     }

diff --git a/...nctions.Databricks.JobMetrics/Arcus.Templates.AzureFunctions.Databricks.JobMetrics.csproj b/...nctions.Databricks.JobMetrics/Arcus.Templates.AzureFunctions.Databricks.JobMetrics.csproj
@@ -36,7 +36,8 @@
   </ItemGroup>
 
   <PropertyGroup>
-    <DefineConstants>$(DefineConstants);</DefineConstants>
+    <DefineConstants>$(DefineConstants);Serilog_AppInsights</DefineConstants>
+    <Serilog_AppInsights>true</Serilog_AppInsights>
     <DockerFastModeProjectMountDirectory>/home/site/wwwroot</DockerFastModeProjectMountDirectory>
   </PropertyGroup>
 
@@ -59,13 +60,13 @@
     <PackageReference Include="Arcus.BackgroundJobs.Databricks" Version="0.4.0" />
     <PackageReference Include="Arcus.Observability.Telemetry.Core" Version="2.6.0" />
     <PackageReference Include="Arcus.Observability.Telemetry.AzureFunctions" Version="2.6.0" />
-    <PackageReference Include="Arcus.Observability.Telemetry.Serilog.Filters" Version="2.6.0" />
-    <PackageReference Include="Arcus.Observability.Telemetry.Serilog.Sinks.Applicationinsights" Version="2.6.0" />
+    <PackageReference Include="Arcus.Observability.Telemetry.Serilog.Filters" Version="2.6.0" Condition="'$(Serilog_AppInsights)' == 'true'" />
+    <PackageReference Include="Arcus.Observability.Telemetry.Serilog.Sinks.Applicationinsights" Version="2.6.0" Condition="'$(Serilog_AppInsights)' == 'true'" />
     <PackageReference Include="Arcus.Security.AzureFunctions" Version="1.8.1" />
     <PackageReference Include="Arcus.Security.Providers.AzureKeyVault" Version="1.8.1" />
     <PackageReference Include="Microsoft.Azure.WebJobs.Extensions" Version="4.0.1" />
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.17.0" />
-    <PackageReference Include="Serilog.Extensions.Logging" Version="3.1.0" />
-    <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
+    <PackageReference Include="Serilog.Extensions.Logging" Version="3.1.0" Condition="'$(Serilog_AppInsights)' == 'true'" />
+    <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" Condition="'$(Serilog_AppInsights)' == 'true'"  />
   </ItemGroup>
 </Project>
diff --git a/src/Arcus.Templates.AzureFunctions.Databricks.JobMetrics/Startup.cs b/src/Arcus.Templates.AzureFunctions.Databricks.JobMetrics/Startup.cs
@@ -5,9 +5,11 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+#if Serilog_AppInsights
 using Serilog;
 using Serilog.Configuration;
-using Serilog.Events;
+using Serilog.Events; 
+#endif
 
 [assembly: FunctionsStartup(typeof(Startup))]
 
@@ -38,14 +40,17 @@ public override void Configure(IFunctionsHostBuilder builder)
                 //#error Please provide a valid secret provider, for example Azure Key Vault: https://security.arcus-azure.net/features/secret-store/provider/key-vault
                 stores.AddAzureKeyVaultWithManagedIdentity("https://your-keyvault.vault.azure.net/", CacheConfiguration.Default);
             });
+#if Serilog_AppInsights
 
             LoggerConfiguration logConfig = CreateLoggerConfiguration(builder);
             builder.Services.AddLogging(logging =>
             {
                 logging.RemoveMicrosoftApplicationInsightsLoggerProvider()
                        .AddSerilog(logConfig.CreateLogger(), dispose: true);
-            });
+            }); 
+#endif
         }
+#if Serilog_AppInsights
 
         private static LoggerConfiguration CreateLoggerConfiguration(IFunctionsHostBuilder builder)
         {
@@ -57,14 +62,15 @@ private static LoggerConfiguration CreateLoggerConfiguration(IFunctionsHostBuild
                 .Enrich.WithComponentName("Azure Databricks Metrics Scraper")
                 .Enrich.WithVersion()
                 .WriteTo.Console();
-            
+
             var connectionString = appConfig.GetValue<string>("APPLICATIONINSIGHTS_CONNECTION_STRING");
             if (!string.IsNullOrWhiteSpace(connectionString))
             {
                 configuration.WriteTo.AzureApplicationInsightsWithConnectionString(connectionString);
             }
-            
+
             return configuration;
-        }
+        } 
+#endif
     }
 }
diff --git a/src/Arcus.Templates.AzureFunctions.Http/.template.config/template.json b/src/Arcus.Templates.AzureFunctions.Http/.template.config/template.json
@@ -109,7 +109,7 @@
       "type": "parameter",
       "datatype": "bool",
       "defaultValue": "false",
-      "description": "Exclude the Serilog logging infrastructure in the worker project"
+      "description": "Exclude the Serilog logging infrastructure in the Azure Functions project"
     },
     "Serilog_AppInsights": {
       "type": "computed",
@@ -129,9 +129,9 @@
       "args": {
         "referenceType": "package",
         "reference": "Microsoft.NET.Sdk.Functions",
-        "version": "4.1.1",
+        "version": "4.1.3",
         "projectFileExtensions": ".csproj"
       }
     }
   ]
-}
+}
diff --git a/src/Arcus.Templates.AzureFunctions.ServiceBus.Queue/.template.config/template.json b/src/Arcus.Templates.AzureFunctions.ServiceBus.Queue/.template.config/template.json
@@ -67,7 +67,7 @@
       "type": "parameter",
       "datatype": "bool",
       "defaultValue": "false",
-      "description": "Exclude the Serilog logging infrastructure in the worker project"
+      "description": "Exclude the Serilog logging infrastructure in the Azure Functions project"
     },
     "Serilog_AppInsights": {
       "type": "computed",
@@ -87,9 +87,9 @@
       "args": {
         "referenceType": "package",
         "reference": "Microsoft.NET.Sdk.Functions",
-        "version": "4.1.1",
+        "version": "4.1.3",
         "projectFileExtensions": ".csproj"
       }
     }
   ]
-}
+}
diff --git a/src/Arcus.Templates.AzureFunctions.ServiceBus.Topic/.template.config/template.json b/src/Arcus.Templates.AzureFunctions.ServiceBus.Topic/.template.config/template.json
@@ -105,7 +105,7 @@
       "type": "parameter",
       "datatype": "bool",
       "defaultValue": "false",
-      "description": "Exclude the Serilog logging infrastructure in the worker project"
+      "description": "Exclude the Serilog logging infrastructure in the Azure Functions project"
     },
     "Serilog_AppInsights": {
       "type": "computed",
@@ -126,9 +126,9 @@
       "args": {
         "referenceType": "package",
         "reference": "Microsoft.NET.Sdk.Functions",
-        "version": "4.1.1",
+        "version": "4.1.3",
         "projectFileExtensions": ".csproj"
       }
     }
   ]
-}
+}
diff --git a/src/Arcus.Templates.Tests.Integration/AzureFunctions/Admin/AdminEndpointService.cs b/src/Arcus.Templates.Tests.Integration/AzureFunctions/Admin/AdminEndpointService.cs
@@ -26,22 +26,27 @@ public AdminEndpointService(int httpPort, string functionName, ITestOutputHelper
         {
             _httpPort = httpPort;
             _functionName = functionName;
+            Endpoint = new Uri($"http://localhost:{_httpPort}/admin/functions/{_functionName}");
         }
 
+        /// <summary>
+        /// Gets the admin HTTP endpoint to interact with the Azure Function directly.
+        /// </summary>
+        public Uri Endpoint { get; }
+
         /// <summary>
         /// Triggers the running Azure Functions project.
         /// </summary>
         /// <exception cref="HttpRequestException">Thrown when the Azure Functions endpoint cannot be contacted.</exception>
         public async Task TriggerFunctionAsync()
         {
-            string endpoint = $"http://localhost:{_httpPort}/admin/functions/{_functionName}";
             try
             {
-                Logger.WriteLine("POST -> {0}", endpoint);
+                Logger.WriteLine("POST -> {0}", Endpoint);
                 using (var content = new StringContent("{}", Encoding.UTF8, "application/json"))
-                using (HttpResponseMessage response = await HttpClient.PostAsync(endpoint, content))
+                using (HttpResponseMessage response = await HttpClient.PostAsync(Endpoint, content))
                 {
-                    Logger.WriteLine("{0} <- {1}", response.StatusCode, endpoint);
+                    Logger.WriteLine("{0} <- {1}", response.StatusCode, Endpoint);
                 }
             }
             catch (Exception exception)

diff --git a/src/Arcus.Templates.Tests.Integration/AzureFunctions/AzureFunctionsProject.cs b/src/Arcus.Templates.Tests.Integration/AzureFunctions/AzureFunctionsProject.cs
@@ -77,16 +77,8 @@ protected void AddLocalSettings(FunctionsWorker workerType)
             string storageAccountConnectionString = AzureFunctionsConfig.StorageAccountConnectionString;
             string workerRuntime = DetermineWorkerRuntime(workerType);
 
-            var json = JsonNode.Parse(
-                $"{{ \"IsEncrypted\": false, " 
-                + $"\"Values\": {{ \"AzureWebJobsStorage\": \"{storageAccountConnectionString}\", \"FUNCTIONS_WORKER_RUNTIME\": \"{workerRuntime}\" }} }}");
-
-            if (workerType is FunctionsWorker.InProcess)
-            {
-                json["Host"] = JsonNode.Parse($"{{ \"LocalHttpPort\": {RootEndpoint.Port} }}");
-            }
-
-            AddFileInProject("local.settings.json", json.ToString());
+            AddFileInProject("local.settings.json",  
+                $"{{ \"IsEncrypted\": false, \"Values\": {{ \"AzureWebJobsStorage\": \"{storageAccountConnectionString}\", \"FUNCTIONS_WORKER_RUNTIME\": \"{workerRuntime}\" }}, \"Host\": {{ \"LocalHttpPort\": {RootEndpoint.Port} }} }}");
         }
 
         private static string DetermineWorkerRuntime(FunctionsWorker workerType)

diff --git a/...Tests.Integration/AzureFunctions/Databricks/JobMetrics/AzureFunctionsDatabricksProject.cs b/...Tests.Integration/AzureFunctions/Databricks/JobMetrics/AzureFunctionsDatabricksProject.cs
@@ -1,8 +1,11 @@
 using System;
 using System.Diagnostics;
+using System.Threading.Tasks;
+using Arcus.Templates.Tests.Integration.AzureFunctions.Admin;
 using Arcus.Templates.Tests.Integration.AzureFunctions.Databricks.JobMetrics.Configuration;
 using Arcus.Templates.Tests.Integration.Fixture;
 using Arcus.Templates.Tests.Integration.Worker.Fixture;
+using GuardNet;
 using Xunit.Abstractions;
 
 namespace Arcus.Templates.Tests.Integration.AzureFunctions.Databricks.JobMetrics
@@ -29,13 +32,19 @@ private AzureFunctionsDatabricksProject(
                    outputWriter)
         {
             AzureFunctionDatabricksConfig = configuration.GetDatabricksConfig();
+            Admin = new AdminEndpointService(RootEndpoint.Port, FunctionName, outputWriter);
         }
 
         /// <summary>
         /// Gets the Databricks connectivity information from the current application configuration, used by this project.
         /// </summary>
         public AzureFunctionDatabricksConfig AzureFunctionDatabricksConfig { get; }
 
+        /// <summary>
+        /// Gets the service to run administrative actions on the Azure Functions project.
+        /// </summary>
+        public AdminEndpointService Admin { get; }
+
         /// <summary>
         /// Starts a newly created project from the Azure Functions Databricks Job Metrics project template.
         /// </summary>
@@ -44,27 +53,54 @@ private AzureFunctionsDatabricksProject(
         /// <returns>
         ///     A Azure Functions Databricks Job Metrics project with a full set of endpoint services to interact with the Azure Function.
         /// </returns>
-        public static AzureFunctionsDatabricksProject StartNew(TestConfig configuration, ITestOutputHelper outputWriter)
+        /// <exception cref="ArgumentNullException">Thrown when the <paramref name="configuration"/> or <paramref name="outputWriter"/> is <c>null</c>.</exception>
+        public static async Task<AzureFunctionsDatabricksProject> StartNewAsync(TestConfig configuration, ITestOutputHelper outputWriter)
         {
-            AzureFunctionsDatabricksProject project = CreateNew(configuration, outputWriter);
-            project.Start();
+            Guard.NotNull(configuration, nameof(configuration), "Requires a test configuration instance to retrieve integration test configuration values to interact with Azure Databricks");
+            Guard.NotNull(outputWriter, nameof(outputWriter), "Requires a logging instance to write diagnostic information during the creation and startup process");
+
+            AzureFunctionsDatabricksProject project = await StartNewAsync(configuration, new AzureFunctionsDatabricksProjectOptions(), outputWriter);
+            return project;
+        }
+
+        /// <summary>
+        /// Starts a newly created project from the Azure Functions Databricks Job Metrics project template.
+        /// </summary>
+        /// <param name="configuration">The configuration to control the hosting of the to-be-created project.</param>
+        /// <param name="options">The additional user project options to change the project contents and functionality.</param>
+        /// <param name="outputWriter">The output logger to add telemetry information during the creation and startup process.</param>
+        /// <returns>
+        ///     A Azure Functions Databricks Job Metrics project with a full set of endpoint services to interact with the Azure Function.
+        /// </returns>
+        /// <exception cref="ArgumentNullException">
+        ///     Thrown when the <paramref name="configuration"/>, <paramref name="options"/>, or <paramref name="outputWriter"/> is <c>null</c>.
+        /// </exception>
+        public static async Task<AzureFunctionsDatabricksProject> StartNewAsync(TestConfig configuration, AzureFunctionsDatabricksProjectOptions options, ITestOutputHelper outputWriter)
+        {
+            Guard.NotNull(configuration, nameof(configuration), "Requires a test configuration instance to retrieve integration test configuration values to interact with Azure Databricks");
+            Guard.NotNull(options, nameof(options), "Requires a project options to change the project contents and functionality");
+            Guard.NotNull(outputWriter, nameof(outputWriter), "Requires a logging instance to write diagnostic information during the creation and startup process");
+
+            AzureFunctionsDatabricksProject project = CreateNew(configuration, options, outputWriter);
+            await project.StartAsync();
 
             return project;
         }
 
-        private static AzureFunctionsDatabricksProject CreateNew(TestConfig configuration, ITestOutputHelper outputWriter)
+        private static AzureFunctionsDatabricksProject CreateNew(TestConfig configuration, AzureFunctionsDatabricksProjectOptions options, ITestOutputHelper outputWriter)
         {
             var project = new AzureFunctionsDatabricksProject(configuration, outputWriter);
-            project.CreateNewProject(new ProjectOptions());
+            project.CreateNewProject(options);
             project.AddDatabricksSecurityToken(project.AzureFunctionDatabricksConfig.SecurityToken);
             project.AddLocalSettings(FunctionsWorker.InProcess);
 
             return project;
         }
 
-        private void Start()
+        private async Task StartAsync()
         {
             Run(BuildConfiguration.Debug, TargetFramework.Net6_0);
+            await WaitUntilTriggerIsAvailableAsync(Admin.Endpoint);
         }
 
         private void AddDatabricksSecurityToken(string securityToken)

diff --git a/...ntegration/AzureFunctions/Databricks/JobMetrics/AzureFunctionsDatabricksProjectOptions.cs b/...ntegration/AzureFunctions/Databricks/JobMetrics/AzureFunctionsDatabricksProjectOptions.cs
@@ -0,0 +1,26 @@
+using Arcus.Templates.Tests.Integration.Fixture;
+
+namespace Arcus.Templates.Tests.Integration.AzureFunctions.Databricks.JobMetrics
+{
+    /// <summary>
+    /// Represents the additional user project options to change the <see cref="AzureFunctionsDatabricksProject"/> project contents and functionality.
+    /// </summary>
+    public class AzureFunctionsDatabricksProjectOptions : ProjectOptions
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="AzureFunctionsDatabricksProjectOptions" /> class.
+        /// </summary>
+        public AzureFunctionsDatabricksProjectOptions()
+        {
+        }
+
+        /// <summary>
+        /// Adds a project option that excludes all the Serilog logging functionality from the resulting project.
+        /// </summary>
+        public AzureFunctionsDatabricksProjectOptions WithExcludeSerilog()
+        {
+            AddOption("--exclude-serilog");
+            return this;
+        }
+    }
+}