Fix changes in Spark Pool DynamicExecutor Allocation (#91)

* Fix changes in Spark Pool DynamicExecutor Alloc * update linter * Updated api version for app insights
Azure · Sep 22, 2021 · be01da6 · be01da6
1 parent 311c857
commit be01da6
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 9 deletions.
diff --git a/.github/linters/.arm-ttk.psd1 b/.github/linters/.arm-ttk.psd1
@@ -9,5 +9,6 @@
         'DependsOn Best Practices',
         'Outputs Must Not Contain Secrets',
         'IDs Should Be Derived From ResourceIDs'
+        'Parameters Must Be Referenced'
     )
 }
diff --git a/infra/main.json b/infra/main.json
@@ -5,7 +5,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.4.613.9944",
-      "templateHash": "7118739398889617477"
+      "templateHash": "12179575723073357481"
     }
   },
   "parameters": {
@@ -468,7 +468,7 @@
             "_generator": {
               "name": "bicep",
               "version": "0.4.613.9944",
-              "templateHash": "4001849972006585130"
+              "templateHash": "1677187290007973520"
             }
           },
           "parameters": {
@@ -591,13 +591,15 @@
                 },
                 "autoScale": {
                   "enabled": true,
-                  "maxNodeCount": 10,
-                  "minNodeCount": 3
+                  "minNodeCount": 3,
+                  "maxNodeCount": 10
                 },
                 "customLibraries": [],
                 "defaultSparkLogFolder": "logs/",
                 "dynamicExecutorAllocation": {
-                  "enabled": true
+                  "enabled": true,
+                  "minExecutors": 1,
+                  "maxExecutors": 9
                 },
                 "nodeSize": "Small",
                 "nodeSizeFamily": "MemoryOptimized",
@@ -1565,7 +1567,7 @@
             "_generator": {
               "name": "bicep",
               "version": "0.4.613.9944",
-              "templateHash": "7730118055684773448"
+              "templateHash": "11163868011668257849"
             }
           },
           "parameters": {
@@ -1586,7 +1588,7 @@
           "resources": [
             {
               "type": "Microsoft.Insights/components",
-              "apiVersion": "2020-02-02-preview",
+              "apiVersion": "2020-02-02",
               "name": "[parameters('applicationInsightsName')]",
               "location": "[parameters('location')]",
               "tags": "[parameters('tags')]",

diff --git a/infra/modules/services/applicationinsights.bicep b/infra/modules/services/applicationinsights.bicep
@@ -13,7 +13,7 @@ param logAnalyticsWorkspaceId string
 // Variables
 
 // Resources
-resource applicationInsights 'Microsoft.Insights/components@2020-02-02-preview' = {
+resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = {
   name: applicationInsightsName
   location: location
   tags: tags

diff --git a/infra/modules/services/synapse.bicep b/infra/modules/services/synapse.bicep
@@ -86,14 +86,16 @@ resource synapseBigDataPool001 'Microsoft.Synapse/workspaces/bigDataPools@2021-0
     }
     autoScale: {
       enabled: true
-      maxNodeCount: 10
       minNodeCount: 3
+      maxNodeCount: 10
     }
     // cacheSize: 100  // Uncomment to set a specific cache size
     customLibraries: []
     defaultSparkLogFolder: 'logs/'
     dynamicExecutorAllocation: {
       enabled: true
+      minExecutors: 1
+      maxExecutors: 9
     }
     // isComputeIsolationEnabled: true  // Uncomment to enable compute isolation (only available in selective regions)
     // libraryRequirements: {  // Uncomment to install pip dependencies on the Spark cluster