Merge pull request #970 from Azure-Samples/e2e/databricks/parking-sen…

…sors-V1

[E2E Databricks]: feat: improving deployment experience and functionality

e2e_samples/parking_sensors/.devcontainer/Dockerfile

@@ -22,6 +22,7 @@ RUN az config set extension.use_dynamic_install=yes_without_prompt
 RUN az config set extension.dynamic_install_allow_preview=true
 RUN az extension add --name application-insights
 RUN az extension add --name azure-devops
+RUN az bicep upgrade
 
 # Install jq & makepasswd for some frequently used utility
 RUN apt-get update \

e2e_samples/parking_sensors/.devcontainer/post_create.sh

@@ -4,8 +4,6 @@
 az config set extension.use_dynamic_install=yes_without_prompt
 az config set extension.dynamic_install_allow_preview=true
 
-# Configure az devops cli
-az devops configure --defaults organization="$AZDO_ORGANIZATION_URL" project="$AZDO_PROJECT"
 
 # Install requirements depending if devcontainer was openned at root or in parking_sensor folder.
 pip install -r /workspace/e2e_samples/parking_sensors/src/ddo_transform/requirements_dev.txt

e2e_samples/parking_sensors/databricks/config/cluster.config.json

@@ -1,8 +1,8 @@
 {
     "cluster_name": "ddo_cluster",
     "autoscale": { "min_workers": 1, "max_workers": 2 },
-    "spark_version": "14.3.x-scala2.12",
-    "autotermination_minutes": 30,
+    "spark_version": "15.4.x-scala2.12",
+    "autotermination_minutes": 10,
     "node_type_id": "Standard_D4as_v5",
     "data_security_mode": "SINGLE_USER",
     "runtime_engine": "PHOTON",

e2e_samples/parking_sensors/deploy.sh

@@ -19,11 +19,10 @@
 set -o errexit
 set -o pipefail
 set -o nounset
-# set -o xtrace # For debugging
 
-. ./scripts/common.sh
-. ./scripts/verify_prerequisites.sh
 . ./scripts/init_environment.sh
+. ./scripts/verify_prerequisites.sh
+
 
 
 project=mdwdops # CONSTANT - this is prefixes to all resources of the Parking Sensor sample
@@ -68,11 +67,11 @@ DEV_DATAFACTORY_NAME=$DEV_DATAFACTORY_NAME \
 
 ####
 
-print_style "DEPLOYMENT SUCCESSFUL
+log "DEPLOYMENT SUCCESSFUL
 Details of the deployment can be found in local .env.* files.\n\n" "success"
 
-print_style "IMPORTANT:
+log "IMPORTANT:
 This script has updated your local Azure Pipeline YAML definitions to point to your Github repo.
 ACTION REQUIRED: Commit and push up these changes to your Github repo before proceeding.\n\n" "warning"
 
-echo "See README > Setup and Deployment for more details and next steps." 
+log "See README > Setup and Deployment for more details and next steps." 

e2e_samples/parking_sensors/infrastructure/main.bicep

@@ -7,6 +7,7 @@ param keyvault_owner_object_id string
 @secure()
 param sql_server_password string
 param enable_monitoring bool
+param keyvault_name string
 param enable_keyvault_soft_delete bool = true
 param enable_keyvault_purge_protection bool = true
 param entra_admin_login string
@@ -61,10 +62,9 @@ module synapse_sql_pool './modules/synapse_sql_pool.bicep' = {
 module keyvault './modules/keyvault.bicep' = {
   name: 'keyvault_deploy_${deployment_id}'
   params: {
-    project: project
+    keyvault_name: keyvault_name
     env: env
     location: location
-    deployment_id: deployment_id
     enable_soft_delete: enable_keyvault_soft_delete
     enable_purge_protection: enable_keyvault_purge_protection
     keyvault_owner_object_id: keyvault_owner_object_id

e2e_samples/parking_sensors/infrastructure/modules/actiongroup.bicep

@@ -1,19 +1,30 @@
+//https://learn.microsoft.com/en-us/azure/templates/microsoft.insights/actiongroups
+// Parameters
+@description('The project name.')
 param project string
+@description('The environment for the deployment.')
 @allowed([
   'dev'
   'stg'
   'prod'
 ])
 param env string
+@description('The unique identifier for this deployment.')
 param deployment_id string
+@description('The email address for the alert action group.')
 param email_id string
-
-resource actiongroup 'Microsoft.Insights/actionGroups@2021-09-01' = {
+// Resource: Action Group
+resource actiongroup 'Microsoft.Insights/actionGroups@2024-10-01-preview' = {
   name: '${project}-emailactiongroup-${env}-${deployment_id}'
   location: 'global'
   tags: {
     DisplayName: 'Action Group'
     Environment: env
+    Project: project
+    DeploymentId: deployment_id
+  }
+  identity: {
+    type: 'SystemAssigned' // Optional: Adjust based on your managed identity requirements
   }
   properties: {
     groupShortName: 'emailgroup'
@@ -25,7 +36,9 @@ resource actiongroup 'Microsoft.Insights/actionGroups@2021-09-01' = {
       }
     ]
     enabled: true
+    // Additional receivers can be added here if needed
   }
 }
-
+// Outputs
+@description('The ID of the created action group.')
 output actiongroup_id string = actiongroup.id

e2e_samples/parking_sensors/infrastructure/modules/alerts.bicep

@@ -1,52 +1,68 @@
+//https://learn.microsoft.com/en-us/azure/templates/microsoft.insights/metricalerts?pivots=deployment-language-bicep
+// Parameters
+@description('The project name.')
 param project string
+@description('The environment for the deployment.')
 @allowed([
   'dev'
   'stg'
   'prod'
 ])
 param env string
+@description('The location of the resource.')
 param location string = resourceGroup().location
+@description('The unique identifier for this deployment.')
 param deployment_id string
+@description('The name of the Data Factory.')
 param datafactory_name string
+@description('The ID of the action group.')
 param action_group_id string
-
+// Resource: Metric Alert
 resource adpipelinefailed 'Microsoft.Insights/metricAlerts@2018-03-01' = {
   name: '${project}-adffailedalert-${env}-${deployment_id}'
   location: 'global'
   tags: {
     DisplayName: 'ADF Pipeline Failed'
     Environment: env
+    Project: project
+    DeploymentId: deployment_id
   }
   properties: {
-    actions: [
-      {
-        actionGroupId: action_group_id        
-      }
+    description: 'ADF pipeline failed'
+    enabled: true
+    severity: 1
+    evaluationFrequency: 'PT1M'
+    windowSize: 'PT5M'
+    scopes: [
+      '${subscription().id}/resourceGroups/${resourceGroup().name}/providers/Microsoft.DataFactory/factories/${datafactory_name}'
     ]
-    autoMitigate: false
+    targetResourceType: 'Microsoft.DataFactory/factories'
+    targetResourceRegion: location
     criteria: {
       'odata.type': 'Microsoft.Azure.Monitor.MultipleResourceMultipleMetricCriteria'
       allOf: [
         {
-            threshold : 1
-            name : 'Metric1'
-            metricNamespace: 'Microsoft.DataFactory/factories'
-            metricName: 'PipelineFailedRuns'
-            operator: 'GreaterThan'
-            timeAggregation: 'Total'
-            criterionType: 'StaticThresholdCriterion'
+          name: 'PipelineFailedRunsCriteria'
+          metricName: 'PipelineFailedRuns'
+          metricNamespace: 'Microsoft.DataFactory/factories'
+          operator: 'GreaterThan'
+          threshold: 1
+          timeAggregation: 'Total'
+          criterionType: 'StaticThresholdCriterion'
         }
-       ]      
+      ]
     }
-    description: 'ADF pipeline failed'
-    enabled: true
-    evaluationFrequency: 'PT1M'
-    scopes: [
-      '${subscription().id}/resourceGroups/${resourceGroup().name}/providers/Microsoft.DataFactory/factories/${datafactory_name}'
+    actions: [
+      {
+        actionGroupId: action_group_id
+        webHookProperties: {
+          exampleProperty: 'exampleValue'
+        }
+      }
     ]
-    severity: 1
-    targetResourceRegion: location
-    targetResourceType: 'Microsoft.DataFactory/factories'
-    windowSize: 'PT5M'
+    autoMitigate: false
   }
 }
+// Outputs
+@description('The ID of the created metric alert.')
+output adpipelinefailed_id string = adpipelinefailed.id

e2e_samples/parking_sensors/infrastructure/modules/appinsights.bicep

@@ -1,25 +1,33 @@
+//https://learn.microsoft.com/en-us/azure/templates/microsoft.insights/components
+// Parameters
+@description('The project name.')
 param project string
+@description('The environment for the deployment.')
 @allowed([
   'dev'
   'stg'
   'prod'
 ])
 param env string
+@description('The location of the resource.')
 param location string = resourceGroup().location
+@description('The unique identifier for this deployment.')
 param deployment_id string
-
-
-resource appinsights 'Microsoft.Insights/components@2020-02-02-preview' = {
+// Resource: Application Insights
+resource appinsights 'Microsoft.Insights/components@2020-02-02' = {
   name: '${project}-appi-${env}-${deployment_id}'
   location: location
   tags: {
     DisplayName: 'Application Insights'
     Environment: env
+    Project: project
+    DeploymentId: deployment_id
   }
   kind: 'other'
   properties: {
     Application_Type: 'other'
   }
 }
-
+// Output: Application Insights Name
+@description('The name of the created Application Insights resource.')
 output appinsights_name string = appinsights.name

e2e_samples/parking_sensors/infrastructure/modules/dashboard.bicep

@@ -1,22 +1,33 @@
+//https://learn.microsoft.com/en-us/azure/templates/microsoft.portal/dashboards
+// Parameters
+@description('The project name.')
 param project string
+@description('The environment for the deployment.')
 @allowed([
   'dev'
   'stg'
   'prod'
 ])
 param env string
+@description('The location of the resource.')
 param location string = resourceGroup().location
+@description('The unique identifier for this deployment.')
 param deployment_id string
+@description('The name of the Data Factory.')
 param datafactory_name string
+@description('The name of the SQL server.')
 param sql_server_name string
+@description('The name of the SQL database.')
 param sql_database_name string
-
-resource dashboard 'Microsoft.Portal/dashboards@2020-09-01-preview' = {
+// Resource: Azure Portal Dashboard
+resource dashboard 'Microsoft.Portal/dashboards@2022-12-01-preview' = {
   name: '${project}-dashboard-${env}-${deployment_id}'
   location: location
   tags: {
     DisplayName: 'Azure Dashboard'
     Environment: env
+    Project: project
+    DeploymentId: deployment_id
   }
   properties: {
     lenses: [

e2e_samples/parking_sensors/infrastructure/modules/data_quality_workbook.bicep

@@ -1,7 +1,14 @@
+//https://learn.microsoft.com/en-us/azure/templates/microsoft.insights/workbooks
+// Parameters
+@description('The display name of the workbook.')
 param workbookDisplayName string = 'DQ Report'
+@description('The category/type of the workbook.')
 param workbookType string = 'workbook'
+@description('The name of the Application Insights resource.')
 param appinsights_name string
+@description('The location of the resource.')
 param location string = resourceGroup().location
+// Variables
 var workbookSourceId = subscriptionResourceId('microsoft.insights/components', '${appinsights_name}')
 var workbookId = guid(workbookSourceId)
 var serializedData = '{"version":"Notebook/1.0","items":[{"type":3,"content":{"version":"KqlItem/1.0","query":"traces\\r\\n| where message==\\"verifychecks\\"\\r\\n| where customDimensions.check_name==\\"Parkingbay Data DQ\\" or customDimensions.check_name==\\"Transformed Data\\" \\r\\n| where severityLevel==\\"1\\" or severityLevel==\\"3\\"\\r\\n| where notempty(customDimensions.pipelinerunid)\\r\\n| project Status = iif(severityLevel==\\"1\\", \\"success\\", \\"failed\\"),CheckName=customDimensions.check_name,RunID = customDimensions.pipelinerunid, Details=customDimensions,Timestamp=timestamp","size":0,"aggregation":3,"timeContext":{"durationMs":604800000},"queryType":0,"resourceType":"microsoft.insights/components","visualization":"table","gridSettings":{"formatters":[{"columnMatch":"Status","formatter":11},{"columnMatch":"status","formatter":11}]}},"name":"query - 0"},{"type":3,"content":{"version":"KqlItem/1.0","query":"traces\\r\\n| where message==\\"verifychecks\\"\\r\\n| where customDimensions.check_name==\\"DQ checks\\"\\r\\n| where severityLevel==\\"1\\" or severityLevel==\\"3\\"\\r\\n| where notempty(customDimensions.pipelinerunid)\\r\\n| project Status = iif(severityLevel==\\"1\\", \\"Success\\", \\"Failed\\"),CheckName=customDimensions.check_name,RunID = customDimensions.pipelinerunid, Details=customDimensions,Timestamp=timestamp\\r\\n| summarize count() by Status \\r\\n| render piechart","size":0,"timeContext":{"durationMs":604800000},"queryType":0,"resourceType":"microsoft.insights/components","visualization":"piechart","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"Status","formatter":1},"leftContent":{"columnMatch":"count_","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}},"graphSettings":{"type":0,"topContent":{"columnMatch":"Status","formatter":1},"centerContent":{"columnMatch":"count_","formatter":1,"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}},"chartSettings":{"seriesLabelSettings":[{"seriesName":"success","label":"","color":"greenDark"},{"seriesName":"failed","color":"red"}]},"mapSettings":{"locInfo":"LatLong","sizeSettings":"count_","sizeAggregation":"Sum","legendMetric":"count_","legendAggregation":"Sum","itemColorSettings":{"type":"heatmap","colorAggregation":"Sum","nodeColorField":"count_","heatmapPalette":"greenRed"}}},"name":"query - 1"}],"fallbackResourceIds":["/subscriptions/XXX-XXX-XXX-XX-XXX/resourceGroups/XXXX/providers/microsoft.insights/components/XXXX"],"$schema":"https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json"}'
@@ -14,8 +21,8 @@ var updatedWorkbookData = {
   ]
 }
 var reserializedData = string(updatedWorkbookData)
-
-resource data_quality_workbook_resource 'microsoft.insights/workbooks@2022-04-01' = {
+// Resource: Data Quality Workbook
+resource data_quality_workbook_resource 'microsoft.insights/workbooks@2023-06-01' = {
   name: workbookId
   location: location
   kind: 'shared'
@@ -25,8 +32,18 @@ resource data_quality_workbook_resource 'microsoft.insights/workbooks@2022-04-01
     version: '1.0'
     sourceId: workbookSourceId
     category: workbookType
+    description: 'Data Quality Report Workbook'
+    tags: [
+      'DataQuality'
+      'Monitoring'
+    ]
+  }
+  tags: {
+    Environment: 'Production'
+    Project: 'DataQualityMonitoring'
   }
   dependsOn: []
 }
-
+// Output: Workbook ID
+@description('The ID of the created workbook resource.')
 output workbookId string = data_quality_workbook_resource.id

e2e_samples/parking_sensors/infrastructure/modules/databricks.bicep

@@ -1,18 +1,26 @@
+//https://learn.microsoft.com/en-us/azure/templates/microsoft.databricks/workspaces
+//https://learn.microsoft.com/en-us/azure/templates/microsoft.authorization/roleassignments
+//Parameters
+@description('The project name.')
 param project string
+@description('The environment for the deployment.')
 @allowed([
   'dev'
   'stg'
   'prod'
 ])
 param env string
+@description('The location of the resource.')
 param location string = resourceGroup().location
+@description('The unique identifier for this deployment.')
 param deployment_id string
+@description('The principal ID of the contributor.')
 param contributor_principal_id string
-
-//https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles
+// Variables
+@description('Role definition ID for Contributor.')
 var contributor = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')
-
-resource databricks 'Microsoft.Databricks/workspaces@2018-04-01' = {
+// Databricks Workspace Resource
+resource databricks 'Microsoft.Databricks/workspaces@2024-09-01-preview' = {
   name: '${project}-dbw-${env}-${deployment_id}'
   location: location
   tags: {
@@ -21,22 +29,27 @@ resource databricks 'Microsoft.Databricks/workspaces@2018-04-01' = {
   }
   sku: {
     name: 'premium'
+    tier: 'Premium'
   }
   properties: {
     managedResourceGroupId: subscriptionResourceId('Microsoft.Resources/resourceGroups', '${project}-${deployment_id}-dbw-${env}-rg')
   }
 }
-
-resource databricks_roleassignment 'Microsoft.Authorization/roleAssignments@2020-08-01-preview' = {
+// Role Assignment Resource
+resource databricks_roleassignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
   name: guid(databricks.id)
   scope: databricks
   properties: {
     roleDefinitionId: contributor
     principalId: contributor_principal_id
     principalType: 'ServicePrincipal'
+    description: 'Contributor access for Databricks workspace.'
   }
 }
-
+// Outputs
+@description('Databricks workspace details.')
 output databricks_output object = databricks
+@description('Databricks workspace ID.')
 output databricks_id string = databricks.id
+@description('Databricks workspace URL.')
 output databricks_workspace_url string = databricks.properties.workspaceUrl