diff --git a/.github/linters/.arm-ttk.psd1 b/.github/linters/.arm-ttk.psd1 index 2164b4a..447bfa3 100644 --- a/.github/linters/.arm-ttk.psd1 +++ b/.github/linters/.arm-ttk.psd1 @@ -4,10 +4,10 @@ @{ # Test = @( ) Skip = @( - 'Template Should Not Contain Blanks', + 'Template Should Not Contain Blanks' 'DeploymentTemplate Must Not Contain Hardcoded Uri' - 'DependsOn Best Practices', - 'Outputs Must Not Contain Secrets', + 'DependsOn Best Practices' + 'Outputs Must Not Contain Secrets' 'IDs Should Be Derived From ResourceIDs' 'Parameters Must Be Referenced' ) diff --git a/.github/linters/.powershell-psscriptanalyzer.psd1 b/.github/linters/.powershell-psscriptanalyzer.psd1 index 7edeaab..20bcc42 100644 --- a/.github/linters/.powershell-psscriptanalyzer.psd1 +++ b/.github/linters/.powershell-psscriptanalyzer.psd1 @@ -8,7 +8,7 @@ #) #IncludeDefaultRules=${true} ExcludeRules = @( - 'PSUseShouldProcessForStateChangingFunctions', + 'PSUseShouldProcessForStateChangingFunctions' 'PSReviewUnusedParameter' 'PSAvoidGlobalVars' 'PSAvoidUsingPlainTextForPassword' diff --git a/docs/EnterpriseScaleAnalytics-AzureDevOpsDeployment.md b/docs/EnterpriseScaleAnalytics-AzureDevOpsDeployment.md index 8c7ec4b..0e5b4cd 100644 --- a/docs/EnterpriseScaleAnalytics-AzureDevOpsDeployment.md +++ b/docs/EnterpriseScaleAnalytics-AzureDevOpsDeployment.md @@ -88,6 +88,8 @@ To begin, please open the [infra/params.dev.json](/infra/params.dev.json). In th | databricksWorkspaceUrl | Specifies the workspace URL of the Databricks workspace that will be connected to the Machine Learning Workspace. | `adb-{databricks-workspace-id}.azuredatabricks.net` | | databricksAccessToken | Specifies the access token of the Databricks workspace that will be connected to the Machine Learning Workspace. Use a secret for this parameter and overwrite as part of the deployment pipelines. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Purview/accounts/{purview-name}` | | enableRoleAssignments | Specifies whether role assignments should be enabled. | `true` or `false` | +| cognitiveServiceKinds | Specifies the cognitive service kind that will be deployed. | [`FormRecognizer`, `LUIS`] | +| enableSearch | Specifies whether Azure Search should be deployed as part of the template. | `true` or `false` | | subnetId | Specifies the resource ID of the subnet to which all services will connect. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/virtualNetworks/{vnet-name}/subnets/{subnet-name}` | | privateDnsZoneIdKeyVault | Specifies the resource ID of the private DNS zone for KeyVault. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.vaultcore.azure.net` | | privateDnsZoneIdSynapseDev | Specifies the resource ID of the private DNS zone for Synapse Dev. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.dev.azuresynapse.net` | diff --git a/docs/EnterpriseScaleAnalytics-GitHubActionsDeployment.md b/docs/EnterpriseScaleAnalytics-GitHubActionsDeployment.md index e4fd633..ae65618 100644 --- a/docs/EnterpriseScaleAnalytics-GitHubActionsDeployment.md +++ b/docs/EnterpriseScaleAnalytics-GitHubActionsDeployment.md @@ -79,6 +79,8 @@ To begin, please open the [infra/params.dev.json](/infra/params.dev.json). In th | databricksWorkspaceUrl | Specifies the workspace URL of the Databricks workspace that will be connected to the Machine Learning Workspace. | `adb-{databricks-workspace-id}.azuredatabricks.net` | | databricksAccessToken | Specifies the access token of the Databricks workspace that will be connected to the Machine Learning Workspace. Use a secret for this parameter and overwrite as part of the deployment pipelines. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Purview/accounts/{purview-name}` | | enableRoleAssignments | Specifies whether role assignments should be enabled. | `true` or `false` | +| cognitiveServiceKinds | Specifies the cognitive service kind that will be deployed. | [`FormRecognizer`, `LUIS`] | +| enableSearch | Specifies whether Azure Search should be deployed as part of the template. | `true` or `false` | | subnetId | Specifies the resource ID of the subnet to which all services will connect. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/virtualNetworks/{vnet-name}/subnets/{subnet-name}` | | privateDnsZoneIdKeyVault | Specifies the resource ID of the private DNS zone for KeyVault. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.vaultcore.azure.net` | | privateDnsZoneIdSynapseDev | Specifies the resource ID of the private DNS zone for Synapse Dev. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.dev.azuresynapse.net` | diff --git a/docs/reference/portal.dataProduct.json b/docs/reference/portal.dataProduct.json index b4f180a..ef2ee6e 100644 --- a/docs/reference/portal.dataProduct.json +++ b/docs/reference/portal.dataProduct.json @@ -862,6 +862,96 @@ } } }, + { + "name": "cognitiveServiceKinds", + "label": "Cognitive Services Kind", + "type": "Microsoft.Common.DropDown", + "visible": true, + "defaultValue": "None", + "toolTip": "Select the kind of Cognitive Service that should be deployed.", + "multiselect": true, + "selectAll": false, + "filter": true, + "filterPlaceholder": "Filter items ...", + "multiLine": true, + "constraints": { + "allowedValues": [ + { + "label": "Anomaly Detector", + "description": "Select if you want to deploy Anomaly Detector.", + "value": "AnomalyDetector" + }, + { + "label": "Computer Vision", + "description": "Select if you want to deploy Computer Vision.", + "value": "ComputerVision" + }, + { + "label": "Content Moderator", + "description": "Select if you want to deploy Content Moderator.", + "value": "ContentModerator" + }, + { + "label": "Custom Vision", + "description": "Select if you want to deploy Custom Vision.", + "value": "CustomVision.Training" + }, + { + "label": "Face", + "description": "Select if you want to deploy Face.", + "value": "Face" + }, + { + "label": "Form Recognizer", + "description": "Select if you want to deploy Form Recognizer.", + "value": "FormRecognizer" + }, + { + "label": "Immersive Reader", + "description": "Select if you want to deploy Immersive Reader.", + "value": "ImmersiveReader" + }, + { + "label": "LUIS", + "description": "Select if you want to deploy Language Understanding Intelligent Service.", + "value": "LUIS" + }, + { + "label": "Personalizer", + "description": "Select if you want to deploy Personalizer.", + "value": "Personalizer" + }, + { + "label": "Speech Services", + "description": "Select if you want to deploy Speech Services.", + "value": "SpeechServices" + }, + { + "label": "Text Analytics", + "description": "Select if you want to deploy Text Analytics.", + "value": "TextAnalytics" + }, + { + "label": "Text Translator", + "description": "Select if you want to deploy Text Translator.", + "value": "TextTranslation" + } + ], + "required": false + } + }, + { + "name": "enableSearch", + "label": "Enable Azure Search", + "type": "Microsoft.Common.CheckBox", + "visible": true, + "defaultValue": false, + "toolTip": "Enable the deployment of Azure Search.", + "constraints": { + "required": false, + "validationMessage": "Enable the deployment of Azure Search." + } + }, { "name": "infoBoxRoleAssignment", "type": "Microsoft.Common.InfoBox", @@ -1309,6 +1399,8 @@ "databricksWorkspaceUrl": "[if(empty(steps('generalSettings').machineLearningDeploymentSettings.databricksWorkspaceUrl.properties.workspaceUrl), '', steps('generalSettings').machineLearningDeploymentSettings.databricksWorkspaceUrl.properties.workspaceUrl)]", "databricksAccessToken": "[if(empty(steps('generalSettings').machineLearningDeploymentSettings.databricksAccessToken), '', steps('generalSettings').machineLearningDeploymentSettings.databricksAccessToken)]", "enableRoleAssignments": "[steps('generalSettings').generalSettings.enableRoleAssignments]", + "cognitiveServiceKinds": "[if(empty(steps('generalSettings').generalSettings.cognitiveServiceKinds), parse('[]'), steps('generalSettings').generalSettings.cognitiveServiceKinds)]", + "enableSearch": "[steps('generalSettings').generalSettings.enableSearch]", "subnetId": "[if(empty(steps('connectivitySettings').virtualNetwork.subnetId), '', steps('connectivitySettings').virtualNetwork.subnetId)]", "privateDnsZoneIdKeyVault": "[if(empty(steps('connectivitySettings').privateDnsZones.privateDnsZoneIdKeyVault), '', steps('connectivitySettings').privateDnsZones.privateDnsZoneIdKeyVault)]", "privateDnsZoneIdSynapseDev": "[if(empty(steps('connectivitySettings').privateDnsZones.privateDnsZoneIdSynapseDev), '', steps('connectivitySettings').privateDnsZones.privateDnsZoneIdSynapseDev)]", diff --git a/infra/main.bicep b/infra/main.bicep index 3e6f7ff..98dd33f 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -54,6 +54,25 @@ param databricksWorkspaceUrl string = '' param databricksAccessToken string = '' @description('Specifies whether role assignments should be enabled for Synapse (Blob Storage Contributor to default storage account).') param enableRoleAssignments bool = false +@allowed([ + 'AnomalyDetector' + 'ComputerVision' + 'ContentModerator' + 'CustomVision.Training' + 'CustomVision.Prediction' + 'Face' + 'FormRecognizer' + 'ImmersiveReader' + 'LUIS' + 'Personalizer' + 'SpeechServices' + 'TextAnalytics' + 'TextTranslation' +]) +@description('Specifies the cognitive service kind that will be deployed.') +param cognitiveServiceKinds array = [] +@description('Specifies whether Azure Search should be deployed as part of the template.') +param enableSearch bool = false // Network parameters @description('Specifies the resource ID of the subnet to which all services will connect.') @@ -107,7 +126,7 @@ var datalakeFileSystemScopes = [for datalakeFileSystemId in datalakeFileSystemId var keyvault001Name = '${name}-vault001' var synapse001Name = '${name}-synapse001' var datafactory001Name = '${name}-datafactory001' -var cognitiveservice001Name = '${name}-cognitiveservice001' +var cognitiveservicesName = '${name}-cognitiveservice' var search001Name = '${name}-search001' var applicationInsights001Name = '${name}-insights001' var containerRegistry001Name = '${name}-containerregistry001' @@ -172,21 +191,21 @@ module datafactory001 'modules/services/datafactory.bicep' = if (processingServi } } -module cognitiveservice001 'modules/services/cognitiveservices.bicep' = { - name: 'cognitiveservice001' +module cognitiveservices 'modules/services/cognitiveservices.bicep' = [for (cognitiveServiceKind, index) in cognitiveServiceKinds: { + name: 'cognitiveservice${padLeft(index + 1, 3, '0')}' scope: resourceGroup() params: { location: location tags: tagsJoined subnetId: subnetId - cognitiveServiceName: cognitiveservice001Name - cognitiveServiceKind: 'FormRecognizer' + cognitiveServiceName: '${cognitiveservicesName}${padLeft(index + 1, 3, '0')}' + cognitiveServiceKind: cognitiveServiceKind cognitiveServiceSkuName: 'S0' privateDnsZoneIdCognitiveService: privateDnsZoneIdCognitiveService } -} +}] -module search001 'modules/services/search.bicep' = { +module search001 'modules/services/search.bicep' = if(enableSearch) { name: 'search001' scope: resourceGroup() params: { diff --git a/infra/main.json b/infra/main.json index 8861ca2..e464b71 100644 --- a/infra/main.json +++ b/infra/main.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.4.613.9944", - "templateHash": "12179575723073357481" + "templateHash": "15295458181843473147" } }, "parameters": { @@ -137,6 +137,35 @@ "description": "Specifies whether role assignments should be enabled for Synapse (Blob Storage Contributor to default storage account)." } }, + "cognitiveServiceKinds": { + "type": "array", + "defaultValue": [], + "metadata": { + "description": "Specifies the cognitive service kind that will be deployed." + }, + "allowedValues": [ + "AnomalyDetector", + "ComputerVision", + "ContentModerator", + "CustomVision.Training", + "CustomVision.Prediction", + "Face", + "FormRecognizer", + "ImmersiveReader", + "LUIS", + "Personalizer", + "SpeechServices", + "TextAnalytics", + "TextTranslation" + ] + }, + "enableSearch": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Specifies whether Azure Search should be deployed as part of the template." + } + }, "subnetId": { "type": "string", "metadata": { @@ -257,7 +286,7 @@ "keyvault001Name": "[format('{0}-vault001', variables('name'))]", "synapse001Name": "[format('{0}-synapse001', variables('name'))]", "datafactory001Name": "[format('{0}-datafactory001', variables('name'))]", - "cognitiveservice001Name": "[format('{0}-cognitiveservice001', variables('name'))]", + "cognitiveservicesName": "[format('{0}-cognitiveservice', variables('name'))]", "search001Name": "[format('{0}-search001', variables('name'))]", "applicationInsights001Name": "[format('{0}-insights001', variables('name'))]", "containerRegistry001Name": "[format('{0}-containerregistry001', variables('name'))]", @@ -1186,9 +1215,13 @@ ] }, { + "copy": { + "name": "cognitiveservices", + "count": "[length(parameters('cognitiveServiceKinds'))]" + }, "type": "Microsoft.Resources/deployments", "apiVersion": "2019-10-01", - "name": "cognitiveservice001", + "name": "[format('cognitiveservice{0}', padLeft(add(copyIndex(), 1), 3, '0'))]", "properties": { "expressionEvaluationOptions": { "scope": "inner" @@ -1205,10 +1238,10 @@ "value": "[parameters('subnetId')]" }, "cognitiveServiceName": { - "value": "[variables('cognitiveservice001Name')]" + "value": "[format('{0}{1}', variables('cognitiveservicesName'), padLeft(add(copyIndex(), 1), 3, '0'))]" }, "cognitiveServiceKind": { - "value": "FormRecognizer" + "value": "[parameters('cognitiveServiceKinds')[copyIndex()]]" }, "cognitiveServiceSkuName": { "value": "S0" @@ -1224,7 +1257,7 @@ "_generator": { "name": "bicep", "version": "0.4.613.9944", - "templateHash": "1987002146294078897" + "templateHash": "3093428550225992785" } }, "parameters": { @@ -1260,8 +1293,7 @@ "Personalizer", "SpeechServices", "TextAnalytics", - "QnAMaker", - "TranslatorText" + "TextTranslation" ] }, "privateDnsZoneIdCognitiveService": { @@ -1284,7 +1316,7 @@ "type": "SystemAssigned" }, "sku": { - "name": "[parameters('cognitiveServiceSkuName')]" + "name": "[if(or(equals(parameters('cognitiveServiceKind'), 'ComputerVision'), equals(parameters('cognitiveServiceKind'), 'TextTranslation')), 'S1', if(equals(parameters('cognitiveServiceKind'), 'TextAnalytics'), 'S', parameters('cognitiveServiceSkuName')))]" }, "kind": "[parameters('cognitiveServiceKind')]", "properties": { @@ -1292,9 +1324,6 @@ "apiProperties": {}, "customSubDomainName": "[parameters('cognitiveServiceName')]", "disableLocalAuth": true, - "encryption": { - "keySource": "Microsoft.CognitiveServices" - }, "networkAcls": { "defaultAction": "Deny", "ipRules": [], @@ -1356,6 +1385,7 @@ } }, { + "condition": "[parameters('enableSearch')]", "type": "Microsoft.Resources/deployments", "apiVersion": "2019-10-01", "name": "search001", diff --git a/infra/modules/services/cognitiveservices.bicep b/infra/modules/services/cognitiveservices.bicep index 7d59aa2..b0d3506 100644 --- a/infra/modules/services/cognitiveservices.bicep +++ b/infra/modules/services/cognitiveservices.bicep @@ -24,8 +24,7 @@ param cognitiveServiceSkuName string = 'S0' 'Personalizer' 'SpeechServices' 'TextAnalytics' - 'QnAMaker' - 'TranslatorText' + 'TextTranslation' ]) param cognitiveServiceKind string param privateDnsZoneIdCognitiveService string = '' @@ -42,7 +41,7 @@ resource cognitiveService 'Microsoft.CognitiveServices/accounts@2021-04-30' = { type: 'SystemAssigned' } sku: { - name: cognitiveServiceSkuName + name: cognitiveServiceKind == 'ComputerVision' || cognitiveServiceKind == 'TextTranslation' ? 'S1' : cognitiveServiceKind == 'TextAnalytics' ? 'S' : cognitiveServiceSkuName } kind: cognitiveServiceKind properties: { @@ -50,9 +49,6 @@ resource cognitiveService 'Microsoft.CognitiveServices/accounts@2021-04-30' = { apiProperties: {} customSubDomainName: cognitiveServiceName disableLocalAuth: true - encryption: { - keySource: 'Microsoft.CognitiveServices' - } networkAcls: { defaultAction: 'Deny' ipRules: [] diff --git a/infra/params.dev.json b/infra/params.dev.json index 2461b5c..3347066 100644 --- a/infra/params.dev.json +++ b/infra/params.dev.json @@ -53,6 +53,15 @@ "enableRoleAssignments": { "value": false }, + "cognitiveServiceKinds": { + "value": [ + "CustomVision.Training", + "CustomVision.Prediction" + ] + }, + "enableSearch": { + "value": true + }, "subnetId": { "value": "/subscriptions/2150d511-458f-43b9-8691-6819ba2e6c7b/resourceGroups/dlz01-dev-network/providers/Microsoft.Network/virtualNetworks/dlz01-dev-vnet/subnets/DataProduct001Subnet" }, diff --git a/infra/params.prod.json b/infra/params.prod.json index 133bfe2..45aec70 100644 --- a/infra/params.prod.json +++ b/infra/params.prod.json @@ -53,6 +53,15 @@ "enableRoleAssignments": { "value": false }, + "cognitiveServiceKinds": { + "value": [ + "CustomVision.Training", + "CustomVision.Prediction" + ] + }, + "enableSearch": { + "value": true + }, "subnetId": { "value": "/subscriptions/2150d511-458f-43b9-8691-6819ba2e6c7b/resourceGroups/dlz01-prod-network/providers/Microsoft.Network/virtualNetworks/dlz01-prod-vnet/subnets/DataProduct001Subnet" }, diff --git a/infra/params.test.json b/infra/params.test.json index 39d1e7c..d30be65 100644 --- a/infra/params.test.json +++ b/infra/params.test.json @@ -53,6 +53,15 @@ "enableRoleAssignments": { "value": false }, + "cognitiveServiceKinds": { + "value": [ + "CustomVision.Training", + "CustomVision.Prediction" + ] + }, + "enableSearch": { + "value": true + }, "subnetId": { "value": "/subscriptions/2150d511-458f-43b9-8691-6819ba2e6c7b/resourceGroups/dlz01-test-network/providers/Microsoft.Network/virtualNetworks/dlz01-test-vnet/subnets/DataProduct001Subnet" },