Merge branch 'main' into adrama/update-python-versions-for-llm-rag

Azure · Jul 15, 2024 · 6305088 · 6305088
2 parents 8f8a870 + 6769467
commit 6305088
Show file tree

Hide file tree

Showing 81 changed files with 298 additions and 182 deletions.
diff --git a/.github/workflows/assets-docs.yaml b/.github/workflows/assets-docs.yaml
@@ -57,10 +57,10 @@ jobs:
           token: ${{ secrets.AZUREML_BOT_PAT }}
           path: ${{ env.wiki_dir }}
 
-      - name: Use Python 3.8 or newer
+      - name: Use Python 3.10 or newer
         uses: actions/setup-python@v4
         with:
-          python-version: '>=3.8'
+          python-version: '>=3.10'
 
       - name: Install dependencies
         run: pip install -e $main_dir/$scripts_azureml_assets_dir -r $main_dir/$scripts_docs_dir/requirements.txt

diff --git a/.github/workflows/assets-release.yaml b/.github/workflows/assets-release.yaml
@@ -57,10 +57,10 @@ jobs:
           fetch-depth: 0
           path: ${{ env.release_dir }}
 
-      - name: Use Python 3.8 or newer
+      - name: Use Python 3.10 or newer
         uses: actions/setup-python@v4
         with:
-          python-version: '>=3.8'
+          python-version: '>=3.10'
 
       - name: Install dependencies
         run: pip install -e $main_dir/$scripts_azureml_assets_dir
@@ -223,10 +223,10 @@ jobs:
           token: ${{ secrets.AZUREML_BOT_PAT }}
           path: ${{ env.release_dir }}
 
-      - name: Use Python 3.8 or newer
+      - name: Use Python 3.10 or newer
         uses: actions/setup-python@v4
         with:
-          python-version: '>=3.8'
+          python-version: '>=3.10'
 
       - name: Install dependencies
         run: pip install -e $main_dir/$scripts_azureml_assets_dir

diff --git a/.github/workflows/assets-test.yaml b/.github/workflows/assets-test.yaml
@@ -70,10 +70,10 @@ jobs:
           separator: ','
           sha: ${{ env.merge_commit_hash || github.sha }}
 
-      - name: Use Python 3.8 or newer
+      - name: Use Python 3.10 or newer
         uses: actions/setup-python@v4
         with:
-          python-version: '>=3.8'
+          python-version: '>=3.10'
 
       - name: Install dependencies
         run: pip install -e $scripts_azureml_assets_dir
@@ -119,10 +119,10 @@ jobs:
         with:
           forked-pr: ${{ needs.check-execution-context.outputs.forked_pr }}
 
-      - name: Use Python 3.8 or newer
+      - name: Use Python 3.10 or newer
         uses: actions/setup-python@v4
         with:
-          python-version: '>=3.8'
+          python-version: '>=3.10'
 
       - name: Install dependencies
         run: pip install -e $scripts_azureml_assets_dir

diff --git a/.github/workflows/assets-validation.yaml b/.github/workflows/assets-validation.yaml
@@ -48,10 +48,10 @@ jobs:
           files_separator: ','
           separator: ','
 
-      - name: Use Python 3.8 or newer
+      - name: Use Python 3.10 or newer
         uses: actions/setup-python@v4
         with:
-          python-version: '>=3.8'
+          python-version: '>=3.10'
 
       - name: Log in to Azure
         env:

diff --git a/.github/workflows/batch-score-ci.yaml b/.github/workflows/batch-score-ci.yaml
@@ -41,10 +41,10 @@ jobs:
         uses: Azure/azureml-assets/.github/actions/clone-repo@main
         with:
           forked-pr: ${{ needs.check-execution-context.outputs.forked_pr }}
-      - name: Use Python 3.8
+      - name: Use Python 3.10 or newer
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '>=3.10'
       - name: Log in to Azure and create resources
         uses: ./.github/actions/create-azure-resources
         with:

diff --git a/.github/workflows/check-new-assets.yaml b/.github/workflows/check-new-assets.yaml
@@ -0,0 +1,45 @@
+name: check-new-assets
+
+on:
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  # Required to clone repo
+  contents: read
+
+jobs:
+  check-new-assets:
+    name: Check new assets
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Clone branch
+        uses: actions/checkout@v3
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v35
+
+      - name: Check for new asset
+        run: |
+          echo "continue=false" >> $GITHUB_ENV
+
+          for file in ${{ steps.changed-files.outputs.added_files }}; do
+            if [[ "$file" =~ .*\/spec.yaml ]] || [[ "$file" =~ .*\/asset.yaml ]]; then
+              echo "continue=true" >> $GITHUB_ENV
+              break
+            fi
+          done
+
+      - name: Check PR labels for "safe to publish" label
+        if: fromJSON(env.continue) && !contains(github.event.pull_request.labels.*.name, 'safe to publish')
+        run: |
+          echo "::error::This PR contains a brand new asset and requires the 'safe to publish' label to run this check. Assets must be properly tested before being added Github and the public registry."
+          exit 1
diff --git a/.github/workflows/environments-ci.yaml b/.github/workflows/environments-ci.yaml
@@ -68,10 +68,10 @@ jobs:
           separator: ','
           sha: ${{ env.merge_commit_hash || github.sha }}
 
-      - name: Use Python 3.8 or newer
+      - name: Use Python 3.10 or newer
         uses: actions/setup-python@v4
         with:
-          python-version: '>=3.8'
+          python-version: '>=3.10'
 
       - name: Install dependencies
         run: pip install -e $scripts_azureml_assets_dir

diff --git a/.github/workflows/scripts-syntax.yaml b/.github/workflows/scripts-syntax.yaml
@@ -40,10 +40,10 @@ jobs:
           files: |
             **/*.py
       
-      - name: Use Python 3.8 or newer
+      - name: Use Python 3.10 or newer
         uses: actions/setup-python@v4
         with:
-          python-version: '>=3.8'
+          python-version: '>=3.10'
 
       - name: Install dependencies
         run: pip install -r $scripts_validation_dir/requirements.txt

diff --git a/.github/workflows/scripts-test.yaml b/.github/workflows/scripts-test.yaml
@@ -57,10 +57,10 @@ jobs:
         with:
           forked-pr: ${{ needs.check-execution-context.outputs.forked_pr }}
 
-      - name: Use Python 3.8 or newer
+      - name: Use Python 3.10 or newer
         uses: actions/setup-python@v4
         with:
-          python-version: '>=3.8'
+          python-version: '>=3.10'
 
       - name: Install dependencies
         run: pip install -e $scripts_azureml_assets_dir -r $scripts_test_dir/requirements.txt

diff --git a/.github/workflows/training-model-mgmt-unittests.yaml b/.github/workflows/training-model-mgmt-unittests.yaml
@@ -38,10 +38,10 @@ jobs:
       - name: Clone branch
         uses: actions/checkout@v3
 
-      - name: Use Python 3.8
+      - name: Use Python 3.10 or newer
         uses: actions/setup-python@v4
         with:
-          python-version: '==3.8'
+          python-version: '>=3.10'
 
       - name: Create conda env
         run: |

diff --git a/assets/common/components/batch_deploy_model/spec.yaml b/assets/common/components/batch_deploy_model/spec.yaml
@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: batch_deploy_model
-version: 0.0.4
+version: 0.0.5
 type: command
 
 is_deterministic: True
@@ -9,7 +9,7 @@ display_name: Batch deploy model
 description:
   Batch deploy a model to a workspace. The component works on compute with [MSI](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-create-manage-compute-instance?tabs=python) attached.
 
-environment: azureml://registries/azureml/environments/python-sdk-v2/versions/15
+environment: azureml://registries/azureml/environments/python-sdk-v2/versions/19
 
 code: ../../src
 command: >-

diff --git a/assets/common/components/delete_endpoint/spec.yaml b/assets/common/components/delete_endpoint/spec.yaml
@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: delete_endpoint
-version: 0.0.6
+version: 0.0.7
 type: command
 
 is_deterministic: True
@@ -9,7 +9,7 @@ display_name: Delete Endpoint
 description:
   Deletes an endpoint resource.
 
-environment: azureml://registries/azureml/environments/python-sdk-v2/versions/15
+environment: azureml://registries/azureml/environments/python-sdk-v2/versions/19
 
 code: ../../src
 command: >-

diff --git a/assets/common/components/deploy_model/spec.yaml b/assets/common/components/deploy_model/spec.yaml
@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: deploy_model
-version: 0.0.10
+version: 0.0.11
 type: command
 
 is_deterministic: True
@@ -9,7 +9,7 @@ display_name: Deploy model
 description:
   Deploy a model to a workspace. The component works on compute with [MSI](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-create-manage-compute-instance?tabs=python) attached.
 
-environment: azureml://registries/azureml/environments/python-sdk-v2/versions/15
+environment: azureml://registries/azureml/environments/python-sdk-v2/versions/19
 
 code: ../../src
 command: >-

diff --git a/assets/common/components/mlflow_model_local_validation/spec.yaml b/assets/common/components/mlflow_model_local_validation/spec.yaml
@@ -1,15 +1,15 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 
 name: mlflow_model_local_validation
-version: 0.0.11
+version: 0.0.12
 type: command
 
 is_deterministic: True
 
 display_name: MLFlow model local validation
 description: Validates if a MLFLow model can be loaded on a compute and is usable for inferencing.
 
-environment: azureml://registries/azureml/environments/python-sdk-v2/versions/15
+environment: azureml://registries/azureml/environments/python-sdk-v2/versions/19
 
 code: ../../src/
 command: >-

diff --git a/assets/common/components/register_model/spec.yaml b/assets/common/components/register_model/spec.yaml
@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: register_model
-version: 0.0.14
+version: 0.0.15
 type: command
 
 is_deterministic: True
@@ -9,7 +9,7 @@ display_name: Register model
 description:
   Register a model to a workspace or a registry. The component works on compute with [MSI](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-create-manage-compute-instance?tabs=python) attached.
 
-environment: azureml://registries/azureml/environments/python-sdk-v2/versions/15
+environment: azureml://registries/azureml/environments/python-sdk-v2/versions/19
 
 code: ../../src
 command: >-

diff --git a/assets/common/environments/python-sdk-v2/context/Dockerfile b/assets/common/environments/python-sdk-v2/context/Dockerfile
@@ -1,6 +1,6 @@
 FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:{{latest-image-tag}}
 
-RUN apt-get update -y && apt-get install nscd git cpio less libglib2.0-0 libglib2.0-data util-linux binutils libnghttp2-14 libssh-4 libsqlite3-0 libpam-modules linux-libc-dev libldap-common libldap-2.4-2 libc-bin libc-dev-bin libc6 libc6-dev libcurl3-gnutls libgnutls30 tar openssh-server openssh-client openssl curl -y
+RUN apt-get update -y && apt-get install wget nscd git cpio less libglib2.0-0 libglib2.0-data util-linux binutils libnghttp2-14 libssh-4 libsqlite3-0 libpam-modules linux-libc-dev libldap-common libldap-2.4-2 libc-bin libc-dev-bin libc6 libc6-dev libcurl3-gnutls libgnutls30 tar openssh-server openssh-client openssl curl -y
 
 WORKDIR /
 

diff --git a/assets/large_language_models/components_pipelines/data_ingestion_db_to_acs/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_db_to_acs/spec.yaml
@@ -4,7 +4,7 @@ tags:
   Preview: ""
 name: llm_ingest_db_to_acs
 display_name: LLM - SQL Datastore to ACS Pipeline
-version: 0.0.90
+version: 0.0.91
 description: Single job pipeline to chunk data from AzureML sql data store, and create ACS embeddings index
 settings:
   default_compute: serverless
@@ -154,7 +154,7 @@ jobs:
       output_grounding_context_file: ${{parent.outputs.db_context}}
     environment_variables:
       MANAGED_IDENTITY_ENABLED: ${{parent.inputs.managed_identity_enabled}}
-    component: "azureml:llm_dbcopilot_grounding:0.0.64"
+    component: "azureml:llm_dbcopilot_grounding:0.0.65"
     type: command
   generate_meta_embeddings:
     type: command
@@ -221,7 +221,7 @@ jobs:
     #########################################
   db_sample_loading_generator:
     type: command
-    component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.39"
+    component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.40"
     resources:
       instance_count: ${{parent.inputs.serverless_instance_count}}
       instance_type: ${{parent.inputs.serverless_instance_type}}
@@ -336,5 +336,5 @@ jobs:
         path: ${{parent.inputs.include_views}}
       instruct_template:
         path: ${{parent.inputs.instruct_template}}
-    component: "azureml:llm_dbcopilot_create_promptflow:0.0.64"
+    component: "azureml:llm_dbcopilot_create_promptflow:0.0.65"
     type: command
diff --git a/assets/large_language_models/components_pipelines/data_ingestion_db_to_faiss/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_db_to_faiss/spec.yaml
@@ -4,7 +4,7 @@ tags:
   Preview: ""
 name: llm_ingest_db_to_faiss
 display_name: LLM - SQL Datastore to FAISS Pipeline
-version: 0.0.90
+version: 0.0.91
 description: Single job pipeline to chunk data from AzureML sql data store, and create FAISS embeddings index
 settings:
   default_compute: serverless
@@ -144,7 +144,7 @@ jobs:
       output_grounding_context_file: ${{parent.outputs.db_context}}
     environment_variables:
       MANAGED_IDENTITY_ENABLED: ${{parent.inputs.managed_identity_enabled}}
-    component: "azureml:llm_dbcopilot_grounding:0.0.64"
+    component: "azureml:llm_dbcopilot_grounding:0.0.65"
     type: command
   generate_meta_embeddings:
     type: command
@@ -209,7 +209,7 @@ jobs:
   #########################################
   db_sample_loading_generator:
     type: command
-    component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.39"
+    component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.40"
     resources:
       instance_count: ${{parent.inputs.serverless_instance_count}}
       instance_type: ${{parent.inputs.serverless_instance_type}}
@@ -320,5 +320,5 @@ jobs:
         path: ${{parent.inputs.include_views}}
       instruct_template:
         path: ${{parent.inputs.instruct_template}}
-    component: "azureml:llm_dbcopilot_create_promptflow:0.0.64"
+    component: "azureml:llm_dbcopilot_create_promptflow:0.0.65"
     type: command
diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dbcopilot_acs_e2e/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dbcopilot_acs_e2e/spec.yaml
@@ -2,7 +2,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.js
 type: pipeline
 
 name: llm_ingest_dbcopilot_acs_e2e
-version: 0.0.58
+version: 0.0.59
 display_name: Data Ingestion for DB Data Output to ACS E2E Deployment
 description: Single job pipeline to chunk data from AzureML DB Datastore and create acs embeddings index
 
@@ -136,7 +136,7 @@ jobs:
   #########################################
   db_meta_loading_generator:
     type: command
-    component: "azureml:llm_dbcopilot_grounding:0.0.64"
+    component: "azureml:llm_dbcopilot_grounding:0.0.65"
     resources:
       instance_count: ${{parent.inputs.serverless_instance_count}}
       instance_type: ${{parent.inputs.serverless_instance_type}}
@@ -208,7 +208,7 @@ jobs:
   #########################################
   db_sample_loading_generator:
     type: command
-    component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.39"
+    component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.40"
     resources:
       instance_count: ${{parent.inputs.serverless_instance_count}}
       instance_type: ${{parent.inputs.serverless_instance_type}}
@@ -270,7 +270,7 @@ jobs:
   #########################################
   endpoint_deployment_job:
     type: command
-    component: "azureml:llm_dbcopilot_deploy_endpoint:0.0.39"
+    component: "azureml:llm_dbcopilot_deploy_endpoint:0.0.40"
     resources:
       instance_count: ${{parent.inputs.serverless_instance_count}}
       instance_type: ${{parent.inputs.serverless_instance_type}}