Merge pull request #338 from zcarroll4/extractor-filter-out-subscriptions

Extractor filter out subscriptions

Author: waelkdouh

configuration.extractor.yaml

@@ -3,8 +3,16 @@
 # More information about workspaces can be found here https://learn.microsoft.com/en-us/azure/api-management/workspaces-overview
 
  apiNames:
-  - api1
-  - api2
+  - apiName1
+  - apiName2
+
+ backendNames:
+  - backendName1
+  - backendName2
+
+ diagnosticNames:
+ - diagnosticName1
+ - diagnosticName2
 
  loggerNames:
   - loggerName1
@@ -15,13 +23,13 @@
   - namedValueName2
 
  productNames:
-  - productName2
+  - productName1
   - productName2
 
- backendNames:
-  - backend1
-  - backend2
+ subscriptionNames:
+ - subscriptionName1
+ - subscriptionName2
 
  tagNames:
-  - tag1
-  - tag2
+  - tagName1
+  - tagName2

docs/apiops/7-additionalTopics/apiops-7-1-content.md

@@ -12,6 +12,6 @@ nav_order: 1
 | tools/.github/workflows/ | Github actions for running the extractor and publisher tools |
 | tools/code | Source code for the extractor and publisher tools |
 | tools/utils/create_pull_request.sh | Script to allow the Azure Devops extractor pipeline to create a Git Pull Request. Not required if using Github actions |
-| configuration.extractor.yaml | A sample yaml extractor configuration file to signal to the extractor to extract select apis, backends, products, tags, loggers,and namedvalues. This is an optional parameter and will only come into play if you want different teams to manage different apis, tags, etc.. You typically will have one configuration per team. Note: You can call the file whatever you want as long as you reference the right file within your extractor pipeline. On a side note since the introduction of Workspaces we believe they are the better solution compared to using this file, but we are keeping the support as not everyone will be able to use workspaces. |
+| configuration.extractor.yaml | A sample yaml extractor configuration file to signal to the extractor to extract select apis, backends, products, tags, loggers, diagnostics, subscriptions, and namedvalues. This is an optional parameter and will only come into play if you want different teams to manage different apis, tags, etc.. You typically will have one configuration per team. Note: You can call the file whatever you want as long as you reference the right file within your extractor pipeline. On a side note since the introduction of Workspaces we believe they are the better solution compared to using this file, but we are keeping the support as not everyone will be able to use workspaces. |
 | configuration.[env].yaml | A sample yaml publisher configuration file to override configuration when running the publisher to promote across different environments (e.g. configuration.prod.yaml for prod environment). Although its optional parameter, you are expected to provide a unique file for each environment as usually different environments have different values (e.g. namevalue). For example if you have a QA environment you would provide another file called configuration.qa.yaml which would have qa specific configuration. Note: You can call the file whatever you want as long as you reference the right file within your publisher pipeline. Also note that if you don't pass the target instance name in the pipelines themselves then you have to pass it as part of the configuration file itself |
 | sample-artifacts-folder | Sample output from the extractor tool. The publisher tool expects this structure and can automatically push changes back to Azure |

tools/code/extractor/Extractor.cs

@@ -27,8 +27,9 @@ public record Parameters
         public IEnumerable<string>? ProductNamesToExport { get; init; }
         public IEnumerable<string>? BackendNamesToExport { get; init; }
         public IEnumerable<string>? TagNamesToExport { get; init; }
+        public IEnumerable<string>? SubscriptionNamesToExport { get; init; }
     }
-
+    
     private readonly Parameters parameters;
 
     public Extractor(Parameters parameters)
@@ -76,6 +77,7 @@ await Service.Export(parameters.ServiceDirectory,
                              parameters.ProductNamesToExport,
                              parameters.BackendNamesToExport,
                              parameters.TagNamesToExport,
+                             parameters.SubscriptionNamesToExport,
                              parameters.ListRestResources,
                              parameters.GetRestResource,
                              parameters.DownloadResource,

tools/code/extractor/Program.cs

@@ -189,6 +189,7 @@ private static Extractor.Parameters GetExtractorParameters(IServiceProvider prov
             ProductNamesToExport = GetProductNamesToExport(configuration),
             BackendNamesToExport = GetBackendNamesToExport(configuration),
             TagNamesToExport = GetTagNamesToExport(configuration),
+            SubscriptionNamesToExport = GetSubscriptionNamesToExport(configuration),
             DefaultApiSpecification = GetApiSpecification(configuration),
             ApplicationLifetime = provider.GetRequiredService<IHostApplicationLifetime>(),
             DownloadResource = provider.GetRequiredService<DownloadResource>(),
@@ -242,6 +243,12 @@ private static Extractor.Parameters GetExtractorParameters(IServiceProvider prov
                            ?.Get<IEnumerable<string>>();
     }
 
+    private static IEnumerable<string>? GetSubscriptionNamesToExport(IConfiguration configuration)
+    {
+        return configuration.TryGetSection("subscriptionNames")
+                           ?.Get<IEnumerable<string>>();
+    }
+
     private static DefaultApiSpecification GetApiSpecification(IConfiguration configuration)
     {
         var configurationFormat = configuration.TryGetValue("API_SPECIFICATION_FORMAT")

tools/code/extractor/Service.cs

@@ -19,6 +19,7 @@ public static async ValueTask Export(
         IEnumerable<string>? productNamesToExport,
         IEnumerable<string>? backendNamesToExport,
         IEnumerable<string>? tagNamesToExport,
+        IEnumerable<string>? subscriptionNamesToExport,
         ListRestResources listRestResources,
         GetRestResource getRestResource,
         DownloadResource downloadResource,
@@ -59,6 +60,6 @@ public static async ValueTask Export(
         await Api.ExportAll(serviceDirectory, serviceUri, defaultSpecification, apiNamesToExport, listRestResources, getRestResource, downloadResource, logger, cancellationToken);
 
         logger.LogInformation("Exporting subscriptions...");
-        await Subscription.ExportAll(serviceDirectory, serviceUri, listRestResources, getRestResource, logger, cancellationToken);
+        await Subscription.ExportAll(serviceDirectory, serviceUri, listRestResources, getRestResource, logger, subscriptionNamesToExport, cancellationToken);
     }
 }

tools/code/extractor/Subscription.cs

@@ -1,5 +1,6 @@
 using common;
 using Microsoft.Extensions.Logging;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
@@ -9,9 +10,11 @@ namespace extractor
 {
     internal class Subscription
     {
-        public static async ValueTask ExportAll(ServiceDirectory serviceDirectory, ServiceUri serviceUri, ListRestResources listRestResources, GetRestResource getRestResource, ILogger logger, CancellationToken cancellationToken)
+        public static async ValueTask ExportAll(ServiceDirectory serviceDirectory, ServiceUri serviceUri, ListRestResources listRestResources, GetRestResource getRestResource, ILogger logger, IEnumerable<string>? subscriptionNamesToExport, CancellationToken cancellationToken)
         {
             await List(serviceUri, listRestResources, cancellationToken)
+                    // Filter out diagnostics that should not be exported   
+                    .Where(subscriptionName => ShouldExport(subscriptionName, subscriptionNamesToExport))
                     .ForEachParallel(async subscriptionName => await Export(serviceDirectory, serviceUri, subscriptionName, getRestResource, logger, cancellationToken),
                                      cancellationToken);
         }
@@ -24,6 +27,12 @@ private static IAsyncEnumerable<SubscriptionName> List(ServiceUri serviceUri, Li
                                         .Select(name => new SubscriptionName(name));
         }
 
+        private static bool ShouldExport(SubscriptionName subscriptionName, IEnumerable<string>? subscriptionNamesToExport)
+        {
+            return subscriptionNamesToExport is null
+                   || subscriptionNamesToExport.Any(subscriptionNameToExport => subscriptionNameToExport.Equals(subscriptionName.ToString(), StringComparison.OrdinalIgnoreCase));
+        }
+
         private static async ValueTask Export(ServiceDirectory serviceDirectory, ServiceUri serviceUri, SubscriptionName subscriptionName, GetRestResource getRestResource, ILogger logger, CancellationToken cancellationToken)
         {
             var subscriptionsDirectory = new SubscriptionsDirectory(serviceDirectory);