diff --git a/.github/workflows/cli-assets-component-component_additional_includes.yml b/.github/workflows/cli-assets-component-component_additional_includes.yml new file mode 100644 index 0000000000..b9b1ac7d24 --- /dev/null +++ b/.github/workflows/cli-assets-component-component_additional_includes.yml @@ -0,0 +1,67 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-assets-component-component_additional_includes +on: + workflow_dispatch: + schedule: + - cron: "47 9/12 * * *" + pull_request: + branches: + - main + paths: + - cli/assets/component/component_additional_includes.yml + - infra/bootstrapping/** + - .github/workflows/cli-assets-component-component_additional_includes.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrapping/bootstrap.sh + working-directory: infra + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/assets/component" + working-directory: infra/bootstrapping + continue-on-error: false + - name: create asset + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml component create -f assets/component/component_additional_includes.yml + working-directory: cli diff --git a/.github/workflows/cli-assets-component-pipeline.yml b/.github/workflows/cli-assets-component-pipeline.yml index 81f17a1869..8027d9141d 100644 --- a/.github/workflows/cli-assets-component-pipeline.yml +++ b/.github/workflows/cli-assets-component-pipeline.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | @@ -63,23 +63,5 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - log_output=$(az ml component create -f assets/component/pipeline.yml 2>&1) - echo "$log_output" > create-asset.log - cat create-asset.log + az ml component create -f assets/component/pipeline.yml working-directory: cli - - name: Determine Failure Reason - run: | - failure_reason="N/A" - if [ "${{ job.status }}" == "failure" ]; then - if grep -q "ResourceNotReady" create-asset.log; then - failure_reason = "ResourceNotReady" - elif grep -q "quota" create-asset.log; then - failure_reason="QuotaIssue" - elif grep -q "ParentResourceNotFound" create-asset.log; then - failure_reason="ParentResourceNotFound" - else - failure_reason="UncategorizedFailure" - fi - fi - echo "FAILURE_REASON=$failure_reason" >> $GITHUB_ENV - working-directory: cli \ No newline at end of file diff --git a/.github/workflows/cli-assets-component-train.yml b/.github/workflows/cli-assets-component-train.yml index b7b2e2a18e..e978fbcfbf 100644 --- a/.github/workflows/cli-assets-component-train.yml +++ b/.github/workflows/cli-assets-component-train.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-data-azure-sql-import.yml b/.github/workflows/cli-assets-data-azure-sql-import.yml new file mode 100644 index 0000000000..a3b54e1e63 --- /dev/null +++ b/.github/workflows/cli-assets-data-azure-sql-import.yml @@ -0,0 +1,67 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-assets-data-azure-sql-import +on: + workflow_dispatch: + schedule: + - cron: "13 6/12 * * *" + pull_request: + branches: + - main + paths: + - cli/assets/data/azure-sql-import.yml + - infra/bootstrapping/** + - .github/workflows/cli-assets-data-azure-sql-import.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrapping/bootstrap.sh + working-directory: infra + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/assets/data" + working-directory: infra/bootstrapping + continue-on-error: false + - name: create asset + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml data create -f assets/data/azure-sql-import.yml + working-directory: cli diff --git a/.github/workflows/cli-assets-data-cloud-file-https.yml b/.github/workflows/cli-assets-data-cloud-file-https.yml index f081869873..60ad44a3f6 100644 --- a/.github/workflows/cli-assets-data-cloud-file-https.yml +++ b/.github/workflows/cli-assets-data-cloud-file-https.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-data-cloud-file-wasbs.yml b/.github/workflows/cli-assets-data-cloud-file-wasbs.yml index 596d62cc3f..724edca949 100644 --- a/.github/workflows/cli-assets-data-cloud-file-wasbs.yml +++ b/.github/workflows/cli-assets-data-cloud-file-wasbs.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-data-cloud-file.yml b/.github/workflows/cli-assets-data-cloud-file.yml index 164f0dab31..304264b156 100644 --- a/.github/workflows/cli-assets-data-cloud-file.yml +++ b/.github/workflows/cli-assets-data-cloud-file.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-data-cloud-folder-https.yml b/.github/workflows/cli-assets-data-cloud-folder-https.yml index a770e0f1e9..f4585f5ccc 100644 --- a/.github/workflows/cli-assets-data-cloud-folder-https.yml +++ b/.github/workflows/cli-assets-data-cloud-folder-https.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-data-cloud-folder-wasbs.yml b/.github/workflows/cli-assets-data-cloud-folder-wasbs.yml new file mode 100644 index 0000000000..2151426f91 --- /dev/null +++ b/.github/workflows/cli-assets-data-cloud-folder-wasbs.yml @@ -0,0 +1,67 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-assets-data-cloud-folder-wasbs +on: + workflow_dispatch: + schedule: + - cron: "23 9/12 * * *" + pull_request: + branches: + - main + paths: + - cli/assets/data/cloud-folder-wasbs.yml + - infra/bootstrapping/** + - .github/workflows/cli-assets-data-cloud-folder-wasbs.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrapping/bootstrap.sh + working-directory: infra + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/assets/data" + working-directory: infra/bootstrapping + continue-on-error: false + - name: create asset + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml data create -f assets/data/cloud-folder-wasbs.yml + working-directory: cli diff --git a/.github/workflows/cli-assets-data-cloud-folder.yml b/.github/workflows/cli-assets-data-cloud-folder.yml index ac80e3eb98..36eff80bcb 100644 --- a/.github/workflows/cli-assets-data-cloud-folder.yml +++ b/.github/workflows/cli-assets-data-cloud-folder.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-data-cloud-mltable.yml b/.github/workflows/cli-assets-data-cloud-mltable.yml index 686c46f0b0..caaad28893 100644 --- a/.github/workflows/cli-assets-data-cloud-mltable.yml +++ b/.github/workflows/cli-assets-data-cloud-mltable.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-data-data_import_schedule_database_inline.yml b/.github/workflows/cli-assets-data-data_import_schedule_database_inline.yml new file mode 100644 index 0000000000..dc9ff9c004 --- /dev/null +++ b/.github/workflows/cli-assets-data-data_import_schedule_database_inline.yml @@ -0,0 +1,67 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-assets-data-data_import_schedule_database_inline +on: + workflow_dispatch: + schedule: + - cron: "30 7/12 * * *" + pull_request: + branches: + - main + paths: + - cli/assets/data/data_import_schedule_database_inline.yml + - infra/bootstrapping/** + - .github/workflows/cli-assets-data-data_import_schedule_database_inline.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrapping/bootstrap.sh + working-directory: infra + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/assets/data" + working-directory: infra/bootstrapping + continue-on-error: false + - name: create asset + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml data create -f assets/data/data_import_schedule_database_inline.yml + working-directory: cli diff --git a/.github/workflows/cli-assets-data-iris-csv-example.yml b/.github/workflows/cli-assets-data-iris-csv-example.yml index 5386ce8ac2..69dd15bebf 100644 --- a/.github/workflows/cli-assets-data-iris-csv-example.yml +++ b/.github/workflows/cli-assets-data-iris-csv-example.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-data-local-file.yml b/.github/workflows/cli-assets-data-local-file.yml index 98a4f7fa75..fbdf0964e2 100644 --- a/.github/workflows/cli-assets-data-local-file.yml +++ b/.github/workflows/cli-assets-data-local-file.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-data-local-folder-sampledata.yml b/.github/workflows/cli-assets-data-local-folder-sampledata.yml index 2d476d7d16..b4f03b0f15 100644 --- a/.github/workflows/cli-assets-data-local-folder-sampledata.yml +++ b/.github/workflows/cli-assets-data-local-folder-sampledata.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-data-local-folder.yml b/.github/workflows/cli-assets-data-local-folder.yml index a6067f587c..d8d812cf73 100644 --- a/.github/workflows/cli-assets-data-local-folder.yml +++ b/.github/workflows/cli-assets-data-local-folder.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-data-local-mltable.yml b/.github/workflows/cli-assets-data-local-mltable.yml index 651df35ecf..2dc8434d44 100644 --- a/.github/workflows/cli-assets-data-local-mltable.yml +++ b/.github/workflows/cli-assets-data-local-mltable.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-data-public-file-https.yml b/.github/workflows/cli-assets-data-public-file-https.yml index ed659099d2..b54085f710 100644 --- a/.github/workflows/cli-assets-data-public-file-https.yml +++ b/.github/workflows/cli-assets-data-public-file-https.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-data-s3-import.yml b/.github/workflows/cli-assets-data-s3-import.yml new file mode 100644 index 0000000000..053f72bbae --- /dev/null +++ b/.github/workflows/cli-assets-data-s3-import.yml @@ -0,0 +1,67 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-assets-data-s3-import +on: + workflow_dispatch: + schedule: + - cron: "46 3/12 * * *" + pull_request: + branches: + - main + paths: + - cli/assets/data/s3-import.yml + - infra/bootstrapping/** + - .github/workflows/cli-assets-data-s3-import.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrapping/bootstrap.sh + working-directory: infra + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/assets/data" + working-directory: infra/bootstrapping + continue-on-error: false + - name: create asset + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml data create -f assets/data/s3-import.yml + working-directory: cli diff --git a/.github/workflows/cli-assets-data-simple_import_schedule.yml b/.github/workflows/cli-assets-data-simple_import_schedule.yml new file mode 100644 index 0000000000..51c30a487c --- /dev/null +++ b/.github/workflows/cli-assets-data-simple_import_schedule.yml @@ -0,0 +1,67 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-assets-data-simple_import_schedule +on: + workflow_dispatch: + schedule: + - cron: "59 3/12 * * *" + pull_request: + branches: + - main + paths: + - cli/assets/data/simple_import_schedule.yml + - infra/bootstrapping/** + - .github/workflows/cli-assets-data-simple_import_schedule.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrapping/bootstrap.sh + working-directory: infra + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/assets/data" + working-directory: infra/bootstrapping + continue-on-error: false + - name: create asset + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml data create -f assets/data/simple_import_schedule.yml + working-directory: cli diff --git a/.github/workflows/cli-assets-data-snowflake-import-managed.yml b/.github/workflows/cli-assets-data-snowflake-import-managed.yml new file mode 100644 index 0000000000..a34abcd06a --- /dev/null +++ b/.github/workflows/cli-assets-data-snowflake-import-managed.yml @@ -0,0 +1,67 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-assets-data-snowflake-import-managed +on: + workflow_dispatch: + schedule: + - cron: "9 7/12 * * *" + pull_request: + branches: + - main + paths: + - cli/assets/data/snowflake-import-managed.yml + - infra/bootstrapping/** + - .github/workflows/cli-assets-data-snowflake-import-managed.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrapping/bootstrap.sh + working-directory: infra + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/assets/data" + working-directory: infra/bootstrapping + continue-on-error: false + - name: create asset + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml data create -f assets/data/snowflake-import-managed.yml + working-directory: cli diff --git a/.github/workflows/cli-assets-data-snowflake-import.yml b/.github/workflows/cli-assets-data-snowflake-import.yml new file mode 100644 index 0000000000..fa53b2db4c --- /dev/null +++ b/.github/workflows/cli-assets-data-snowflake-import.yml @@ -0,0 +1,67 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-assets-data-snowflake-import +on: + workflow_dispatch: + schedule: + - cron: "54 11/12 * * *" + pull_request: + branches: + - main + paths: + - cli/assets/data/snowflake-import.yml + - infra/bootstrapping/** + - .github/workflows/cli-assets-data-snowflake-import.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrapping/bootstrap.sh + working-directory: infra + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/assets/data" + working-directory: infra/bootstrapping + continue-on-error: false + - name: create asset + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml data create -f assets/data/snowflake-import.yml + working-directory: cli diff --git a/.github/workflows/cli-assets-environment-docker-context.yml b/.github/workflows/cli-assets-environment-docker-context.yml index 93a824aa45..ae9863e2cd 100644 --- a/.github/workflows/cli-assets-environment-docker-context.yml +++ b/.github/workflows/cli-assets-environment-docker-context.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-environment-docker-image.yml b/.github/workflows/cli-assets-environment-docker-image.yml index 65ff96a3b0..7ccc750e59 100644 --- a/.github/workflows/cli-assets-environment-docker-image.yml +++ b/.github/workflows/cli-assets-environment-docker-image.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-model-local-file.yml b/.github/workflows/cli-assets-model-local-file.yml index f1d75540c2..e1a67d8eb3 100644 --- a/.github/workflows/cli-assets-model-local-file.yml +++ b/.github/workflows/cli-assets-model-local-file.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-assets-model-local-mlflow.yml b/.github/workflows/cli-assets-model-local-mlflow.yml index 9cc57bb041..cb4d063061 100644 --- a/.github/workflows/cli-assets-model-local-mlflow.yml +++ b/.github/workflows/cli-assets-model-local-mlflow.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-basic-installation-basic-installation.yml b/.github/workflows/cli-basic-installation-basic-installation.yml index 341a509346..3a43da9a4f 100644 --- a/.github/workflows/cli-basic-installation-basic-installation.yml +++ b/.github/workflows/cli-basic-installation-basic-installation.yml @@ -7,7 +7,7 @@ name: cli-basic-installation-basic-installation on: workflow_dispatch: schedule: - - cron: "*/5 * * * *" + - cron: "53 3/12 * * *" pull_request: branches: - main @@ -27,11 +27,6 @@ jobs: steps: - name: check out repo uses: actions/checkout@v2 - - name: Install latest Azure CLI - run: | - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - - name: Check Azure CLI version - run: az --version - name: azure login uses: azure/login@v1 with: @@ -68,5 +63,5 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml workspace list --subscription $SUBSCRIPTION_ID --resource-group $RESOURCE_GROUP_NAME + az ml basic-installation create -f basic-installation/basic-installation.yml working-directory: cli diff --git a/.github/workflows/cli-endpoints-batch-deploy-models-custom-outputs-parquet-endpoint.yml b/.github/workflows/cli-endpoints-batch-deploy-models-custom-outputs-parquet-endpoint.yml index 54cfecf2bb..91f5e53553 100644 --- a/.github/workflows/cli-endpoints-batch-deploy-models-custom-outputs-parquet-endpoint.yml +++ b/.github/workflows/cli-endpoints-batch-deploy-models-custom-outputs-parquet-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n tomoutputsparquetendpoint8741 -y + az ml batch-endpoint delete -n tomoutputsparquetendpoint8192 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,18 +72,18 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/batch/deploy-models/custom-outputs-parquet/endpoint.yml - az ml batch-endpoint create -n tomoutputsparquetendpoint8741 -f endpoints/batch/deploy-models/custom-outputs-parquet/endpoint.yml + az ml batch-endpoint create -n tomoutputsparquetendpoint8192 -f endpoints/batch/deploy-models/custom-outputs-parquet/endpoint.yml working-directory: cli - name: create deployment run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/batch/deploy-models/custom-outputs-parquet/deployment.yml - az ml batch-deployment create -e tomoutputsparquetendpoint8741 -f endpoints/batch/deploy-models/custom-outputs-parquet/deployment.yml + az ml batch-deployment create -e tomoutputsparquetendpoint8192 -f endpoints/batch/deploy-models/custom-outputs-parquet/deployment.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n tomoutputsparquetendpoint8741 -y - working-directory: cli \ No newline at end of file + az ml batch-endpoint delete -n tomoutputsparquetendpoint8192 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-batch-deploy-models-heart-classifier-mlflow-endpoint.yml b/.github/workflows/cli-endpoints-batch-deploy-models-heart-classifier-mlflow-endpoint.yml index dfadc9c364..b16383ddbc 100644 --- a/.github/workflows/cli-endpoints-batch-deploy-models-heart-classifier-mlflow-endpoint.yml +++ b/.github/workflows/cli-endpoints-batch-deploy-models-heart-classifier-mlflow-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n tclassifiermlflowendpoint3191 -y + az ml batch-endpoint delete -n tclassifiermlflowendpoint9437 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,11 +72,11 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/batch/deploy-models/heart-classifier-mlflow/endpoint.yml - az ml batch-endpoint create -n tclassifiermlflowendpoint3191 -f endpoints/batch/deploy-models/heart-classifier-mlflow/endpoint.yml + az ml batch-endpoint create -n tclassifiermlflowendpoint9437 -f endpoints/batch/deploy-models/heart-classifier-mlflow/endpoint.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n tclassifiermlflowendpoint3191 -y - working-directory: cli \ No newline at end of file + az ml batch-endpoint delete -n tclassifiermlflowendpoint9437 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-batch-deploy-models-huggingface-text-summarization-endpoint.yml b/.github/workflows/cli-endpoints-batch-deploy-models-huggingface-text-summarization-endpoint.yml index ffa4503415..d0d836eafc 100644 --- a/.github/workflows/cli-endpoints-batch-deploy-models-huggingface-text-summarization-endpoint.yml +++ b/.github/workflows/cli-endpoints-batch-deploy-models-huggingface-text-summarization-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n textsummarizationendpoint2742 -y + az ml batch-endpoint delete -n textsummarizationendpoint6593 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,18 +72,18 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/batch/deploy-models/huggingface-text-summarization/endpoint.yml - az ml batch-endpoint create -n textsummarizationendpoint2742 -f endpoints/batch/deploy-models/huggingface-text-summarization/endpoint.yml + az ml batch-endpoint create -n textsummarizationendpoint6593 -f endpoints/batch/deploy-models/huggingface-text-summarization/endpoint.yml working-directory: cli - name: create deployment run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/batch/deploy-models/huggingface-text-summarization/deployment.yml - az ml batch-deployment create -e textsummarizationendpoint2742 -f endpoints/batch/deploy-models/huggingface-text-summarization/deployment.yml + az ml batch-deployment create -e textsummarizationendpoint6593 -f endpoints/batch/deploy-models/huggingface-text-summarization/deployment.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n textsummarizationendpoint2742 -y - working-directory: cli \ No newline at end of file + az ml batch-endpoint delete -n textsummarizationendpoint6593 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-batch-deploy-models-imagenet-classifier-endpoint.yml b/.github/workflows/cli-endpoints-batch-deploy-models-imagenet-classifier-endpoint.yml index eb39efbed4..f8ca5ab794 100644 --- a/.github/workflows/cli-endpoints-batch-deploy-models-imagenet-classifier-endpoint.yml +++ b/.github/workflows/cli-endpoints-batch-deploy-models-imagenet-classifier-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n imagenetclassifierendpoint3948 -y + az ml batch-endpoint delete -n imagenetclassifierendpoint9289 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,11 +72,11 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/batch/deploy-models/imagenet-classifier/endpoint.yml - az ml batch-endpoint create -n imagenetclassifierendpoint3948 -f endpoints/batch/deploy-models/imagenet-classifier/endpoint.yml + az ml batch-endpoint create -n imagenetclassifierendpoint9289 -f endpoints/batch/deploy-models/imagenet-classifier/endpoint.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n imagenetclassifierendpoint3948 -y - working-directory: cli \ No newline at end of file + az ml batch-endpoint delete -n imagenetclassifierendpoint9289 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-batch-deploy-models-mnist-classifier-endpoint.yml b/.github/workflows/cli-endpoints-batch-deploy-models-mnist-classifier-endpoint.yml index 5f12392ae4..e5e331a3f3 100644 --- a/.github/workflows/cli-endpoints-batch-deploy-models-mnist-classifier-endpoint.yml +++ b/.github/workflows/cli-endpoints-batch-deploy-models-mnist-classifier-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n lsmnistclassifierendpoint9980 -y + az ml batch-endpoint delete -n lsmnistclassifierendpoint4781 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,11 +72,11 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/batch/deploy-models/mnist-classifier/endpoint.yml - az ml batch-endpoint create -n lsmnistclassifierendpoint9980 -f endpoints/batch/deploy-models/mnist-classifier/endpoint.yml + az ml batch-endpoint create -n lsmnistclassifierendpoint4781 -f endpoints/batch/deploy-models/mnist-classifier/endpoint.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n lsmnistclassifierendpoint9980 -y - working-directory: cli \ No newline at end of file + az ml batch-endpoint delete -n lsmnistclassifierendpoint4781 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-batch-deploy-models-openai-embeddings-endpoint.yml b/.github/workflows/cli-endpoints-batch-deploy-models-openai-embeddings-endpoint.yml new file mode 100644 index 0000000000..3b6592988f --- /dev/null +++ b/.github/workflows/cli-endpoints-batch-deploy-models-openai-embeddings-endpoint.yml @@ -0,0 +1,89 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-endpoints-batch-deploy-models-openai-embeddings-endpoint +on: + workflow_dispatch: + schedule: + - cron: "18 11/12 * * *" + pull_request: + branches: + - main + paths: + - cli/endpoints/batch/deploy-models/openai-embeddings/** + - cli/endpoints/batch/** + - infra/bootstrapping/** + - .github/workflows/cli-endpoints-batch-deploy-models-openai-embeddings-endpoint.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/endpoints/batch/deploy-models/openai-embeddings" + working-directory: infra/bootstrapping + continue-on-error: false + - name: delete endpoint if existing + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml batch-endpoint delete -n sopenaiembeddingsendpoint8006 -y + working-directory: cli + continue-on-error: true + - name: create endpoint + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + cat endpoints/batch/deploy-models/openai-embeddings/endpoint.yml + az ml batch-endpoint create -n sopenaiembeddingsendpoint8006 -f endpoints/batch/deploy-models/openai-embeddings/endpoint.yml + working-directory: cli + - name: create deployment + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + cat endpoints/batch/deploy-models/openai-embeddings/deployment.yml + az ml batch-deployment create -e sopenaiembeddingsendpoint8006 -f endpoints/batch/deploy-models/openai-embeddings/deployment.yml + working-directory: cli + - name: cleanup endpoint + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml batch-endpoint delete -n sopenaiembeddingsendpoint8006 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-endpoint.yml b/.github/workflows/cli-endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-endpoint.yml index 6e25288ed1..221ef2acf1 100644 --- a/.github/workflows/cli-endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-endpoint.yml +++ b/.github/workflows/cli-endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n withpreprocessingendpoint6601 -y + az ml batch-endpoint delete -n withpreprocessingendpoint2674 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,18 +72,18 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/endpoint.yml - az ml batch-endpoint create -n withpreprocessingendpoint6601 -f endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/endpoint.yml + az ml batch-endpoint create -n withpreprocessingendpoint2674 -f endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/endpoint.yml working-directory: cli - name: create deployment run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/deployment.yml - az ml batch-deployment create -e withpreprocessingendpoint6601 -f endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/deployment.yml + az ml batch-deployment create -e withpreprocessingendpoint2674 -f endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/deployment.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n withpreprocessingendpoint6601 -y - working-directory: cli \ No newline at end of file + az ml batch-endpoint delete -n withpreprocessingendpoint2674 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-batch-deploy-pipelines-hello-batch-endpoint.yml b/.github/workflows/cli-endpoints-batch-deploy-pipelines-hello-batch-endpoint.yml index 2aa5ca8e11..c4002d1420 100644 --- a/.github/workflows/cli-endpoints-batch-deploy-pipelines-hello-batch-endpoint.yml +++ b/.github/workflows/cli-endpoints-batch-deploy-pipelines-hello-batch-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n pelineshellobatchendpoint9006 -y + az ml batch-endpoint delete -n pelineshellobatchendpoint3279 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,18 +72,18 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/batch/deploy-pipelines/hello-batch/endpoint.yml - az ml batch-endpoint create -n pelineshellobatchendpoint9006 -f endpoints/batch/deploy-pipelines/hello-batch/endpoint.yml + az ml batch-endpoint create -n pelineshellobatchendpoint3279 -f endpoints/batch/deploy-pipelines/hello-batch/endpoint.yml working-directory: cli - name: create deployment run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/batch/deploy-pipelines/hello-batch/deployment.yml - az ml batch-deployment create -e pelineshellobatchendpoint9006 -f endpoints/batch/deploy-pipelines/hello-batch/deployment.yml + az ml batch-deployment create -e pelineshellobatchendpoint3279 -f endpoints/batch/deploy-pipelines/hello-batch/deployment.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n pelineshellobatchendpoint9006 -y - working-directory: cli \ No newline at end of file + az ml batch-endpoint delete -n pelineshellobatchendpoint3279 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-batch-deploy-pipelines-training-with-components-endpoint.yml b/.github/workflows/cli-endpoints-batch-deploy-pipelines-training-with-components-endpoint.yml index a34a2315ba..a4d546b5aa 100644 --- a/.github/workflows/cli-endpoints-batch-deploy-pipelines-training-with-components-endpoint.yml +++ b/.github/workflows/cli-endpoints-batch-deploy-pipelines-training-with-components-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n ingwithcomponentsendpoint9219 -y + az ml batch-endpoint delete -n ingwithcomponentsendpoint8051 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,11 +72,11 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/batch/deploy-pipelines/training-with-components/endpoint.yml - az ml batch-endpoint create -n ingwithcomponentsendpoint9219 -f endpoints/batch/deploy-pipelines/training-with-components/endpoint.yml + az ml batch-endpoint create -n ingwithcomponentsendpoint8051 -f endpoints/batch/deploy-pipelines/training-with-components/endpoint.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml batch-endpoint delete -n ingwithcomponentsendpoint9219 -y - working-directory: cli \ No newline at end of file + az ml batch-endpoint delete -n ingwithcomponentsendpoint8051 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-online-custom-container-minimal-multimodel-minimal-multimodel-endpoint.yml b/.github/workflows/cli-endpoints-online-custom-container-minimal-multimodel-minimal-multimodel-endpoint.yml index 5e60867033..1ab56f7173 100644 --- a/.github/workflows/cli-endpoints-online-custom-container-minimal-multimodel-minimal-multimodel-endpoint.yml +++ b/.github/workflows/cli-endpoints-online-custom-container-minimal-multimodel-minimal-multimodel-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n minimalmultimodelendpoint9199 -y + az ml online-endpoint delete -n minimalmultimodelendpoint9553 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,11 +72,11 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/custom-container/minimal/multimodel/minimal-multimodel-endpoint.yml - az ml online-endpoint create -n minimalmultimodelendpoint9199 -f endpoints/online/custom-container/minimal/multimodel/minimal-multimodel-endpoint.yml + az ml online-endpoint create -n minimalmultimodelendpoint9553 -f endpoints/online/custom-container/minimal/multimodel/minimal-multimodel-endpoint.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n minimalmultimodelendpoint9199 -y - working-directory: cli \ No newline at end of file + az ml online-endpoint delete -n minimalmultimodelendpoint9553 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-online-custom-container-minimal-single-model-conda-in-dockerfile-minimal-single-model-conda-in-dockerfile-endpoint.yml b/.github/workflows/cli-endpoints-online-custom-container-minimal-single-model-conda-in-dockerfile-minimal-single-model-conda-in-dockerfile-endpoint.yml index 4cd65d571a..b97667ac34 100644 --- a/.github/workflows/cli-endpoints-online-custom-container-minimal-single-model-conda-in-dockerfile-minimal-single-model-conda-in-dockerfile-endpoint.yml +++ b/.github/workflows/cli-endpoints-online-custom-container-minimal-single-model-conda-in-dockerfile-minimal-single-model-conda-in-dockerfile-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n condaindockerfileendpoint3179 -y + az ml online-endpoint delete -n condaindockerfileendpoint2968 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,11 +72,11 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/custom-container/minimal/single-model/conda-in-dockerfile/minimal-single-model-conda-in-dockerfile-endpoint.yml - az ml online-endpoint create -n condaindockerfileendpoint3179 -f endpoints/online/custom-container/minimal/single-model/conda-in-dockerfile/minimal-single-model-conda-in-dockerfile-endpoint.yml + az ml online-endpoint create -n condaindockerfileendpoint2968 -f endpoints/online/custom-container/minimal/single-model/conda-in-dockerfile/minimal-single-model-conda-in-dockerfile-endpoint.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n condaindockerfileendpoint3179 -y - working-directory: cli \ No newline at end of file + az ml online-endpoint delete -n condaindockerfileendpoint2968 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-online-custom-container-minimal-single-model-minimal-single-model-endpoint.yml b/.github/workflows/cli-endpoints-online-custom-container-minimal-single-model-minimal-single-model-endpoint.yml index 9676a7e011..6c7973affd 100644 --- a/.github/workflows/cli-endpoints-online-custom-container-minimal-single-model-minimal-single-model-endpoint.yml +++ b/.github/workflows/cli-endpoints-online-custom-container-minimal-single-model-minimal-single-model-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n inimalsinglemodelendpoint2770 -y + az ml online-endpoint delete -n inimalsinglemodelendpoint2829 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,11 +72,11 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/custom-container/minimal/single-model/minimal-single-model-endpoint.yml - az ml online-endpoint create -n inimalsinglemodelendpoint2770 -f endpoints/online/custom-container/minimal/single-model/minimal-single-model-endpoint.yml + az ml online-endpoint create -n inimalsinglemodelendpoint2829 -f endpoints/online/custom-container/minimal/single-model/minimal-single-model-endpoint.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n inimalsinglemodelendpoint2770 -y - working-directory: cli \ No newline at end of file + az ml online-endpoint delete -n inimalsinglemodelendpoint2829 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-online-custom-container-mlflow-multideployment-scikit-mlflow-endpoint.yml b/.github/workflows/cli-endpoints-online-custom-container-mlflow-multideployment-scikit-mlflow-endpoint.yml index c122b5014d..af9586eea2 100644 --- a/.github/workflows/cli-endpoints-online-custom-container-mlflow-multideployment-scikit-mlflow-endpoint.yml +++ b/.github/workflows/cli-endpoints-online-custom-container-mlflow-multideployment-scikit-mlflow-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n ymentscikitmlflowendpoint4848 -y + az ml online-endpoint delete -n ymentscikitmlflowendpoint6855 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,11 +72,11 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/custom-container/mlflow/multideployment-scikit/mlflow-endpoint.yml - az ml online-endpoint create -n ymentscikitmlflowendpoint4848 -f endpoints/online/custom-container/mlflow/multideployment-scikit/mlflow-endpoint.yml + az ml online-endpoint create -n ymentscikitmlflowendpoint6855 -f endpoints/online/custom-container/mlflow/multideployment-scikit/mlflow-endpoint.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n ymentscikitmlflowendpoint4848 -y - working-directory: cli \ No newline at end of file + az ml online-endpoint delete -n ymentscikitmlflowendpoint6855 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-online-custom-container-r-multimodel-plumber-r-endpoint.yml b/.github/workflows/cli-endpoints-online-custom-container-r-multimodel-plumber-r-endpoint.yml index 7c66c5ce60..da78d4093f 100644 --- a/.github/workflows/cli-endpoints-online-custom-container-r-multimodel-plumber-r-endpoint.yml +++ b/.github/workflows/cli-endpoints-online-custom-container-r-multimodel-plumber-r-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n ultimodelplumberrendpoint9536 -y + az ml online-endpoint delete -n ultimodelplumberrendpoint5281 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,11 +72,11 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/custom-container/r/multimodel-plumber/r-endpoint.yml - az ml online-endpoint create -n ultimodelplumberrendpoint9536 -f endpoints/online/custom-container/r/multimodel-plumber/r-endpoint.yml + az ml online-endpoint create -n ultimodelplumberrendpoint5281 -f endpoints/online/custom-container/r/multimodel-plumber/r-endpoint.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n ultimodelplumberrendpoint9536 -y - working-directory: cli \ No newline at end of file + az ml online-endpoint delete -n ultimodelplumberrendpoint5281 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-online-custom-container-torchserve-densenet-torchserve-endpoint.yml b/.github/workflows/cli-endpoints-online-custom-container-torchserve-densenet-torchserve-endpoint.yml index fc789051f9..8875b2ed29 100644 --- a/.github/workflows/cli-endpoints-online-custom-container-torchserve-densenet-torchserve-endpoint.yml +++ b/.github/workflows/cli-endpoints-online-custom-container-torchserve-densenet-torchserve-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n densenettorchserveendpoint7556 -y + az ml online-endpoint delete -n densenettorchserveendpoint2372 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,11 +72,11 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/custom-container/torchserve/densenet/torchserve-endpoint.yml - az ml online-endpoint create -n densenettorchserveendpoint7556 -f endpoints/online/custom-container/torchserve/densenet/torchserve-endpoint.yml + az ml online-endpoint create -n densenettorchserveendpoint2372 -f endpoints/online/custom-container/torchserve/densenet/torchserve-endpoint.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n densenettorchserveendpoint7556 -y - working-directory: cli \ No newline at end of file + az ml online-endpoint delete -n densenettorchserveendpoint2372 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-online-custom-container-triton-single-model-triton-cc-endpoint.yml b/.github/workflows/cli-endpoints-online-custom-container-triton-single-model-triton-cc-endpoint.yml index 8e4dd5df4b..a796f71e9d 100644 --- a/.github/workflows/cli-endpoints-online-custom-container-triton-single-model-triton-cc-endpoint.yml +++ b/.github/workflows/cli-endpoints-online-custom-container-triton-single-model-triton-cc-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n glemodeltritonccendpoint4238 -y + az ml online-endpoint delete -n glemodeltritonccendpoint4787 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,11 +72,11 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/custom-container/triton/single-model/triton-cc-endpoint.yml - az ml online-endpoint create -n glemodeltritonccendpoint4238 -f endpoints/online/custom-container/triton/single-model/triton-cc-endpoint.yml + az ml online-endpoint create -n glemodeltritonccendpoint4787 -f endpoints/online/custom-container/triton/single-model/triton-cc-endpoint.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n glemodeltritonccendpoint4238 -y - working-directory: cli \ No newline at end of file + az ml online-endpoint delete -n glemodeltritonccendpoint4787 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-online-deploy-with-packages-custom-model-endpoint.yml b/.github/workflows/cli-endpoints-online-deploy-with-packages-custom-model-endpoint.yml new file mode 100644 index 0000000000..9b21089617 --- /dev/null +++ b/.github/workflows/cli-endpoints-online-deploy-with-packages-custom-model-endpoint.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-endpoints-online-deploy-with-packages-custom-model-endpoint +on: + workflow_dispatch: + schedule: + - cron: "18 11/12 * * *" + pull_request: + branches: + - main + paths: + - cli/endpoints/online/deploy-with-packages/custom-model/** + - cli/endpoints/online/** + - infra/bootstrapping/** + - .github/workflows/cli-endpoints-online-deploy-with-packages-custom-model-endpoint.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/endpoints/online/deploy-with-packages/custom-model" + working-directory: infra/bootstrapping + continue-on-error: false + - name: delete endpoint if existing + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml online-endpoint delete -n ckagescustommodelendpoint4359 -y + working-directory: cli + continue-on-error: true + - name: create endpoint + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + cat endpoints/online/deploy-with-packages/custom-model/endpoint.yml + az ml online-endpoint create -n ckagescustommodelendpoint4359 -f endpoints/online/deploy-with-packages/custom-model/endpoint.yml + working-directory: cli + - name: create deployment + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + cat endpoints/online/deploy-with-packages/custom-model/deployment.yml + az ml online-deployment create -e ckagescustommodelendpoint4359 -f endpoints/online/deploy-with-packages/custom-model/deployment.yml + working-directory: cli + - name: create deployment + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + cat endpoints/online/deploy-with-packages/custom-model/model-deployment.yml + az ml online-deployment create -e ckagescustommodelendpoint4359 -f endpoints/online/deploy-with-packages/custom-model/model-deployment.yml + working-directory: cli + - name: cleanup endpoint + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml online-endpoint delete -n ckagescustommodelendpoint4359 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-online-deploy-with-packages-mlflow-model-endpoint.yml b/.github/workflows/cli-endpoints-online-deploy-with-packages-mlflow-model-endpoint.yml new file mode 100644 index 0000000000..6e2d88cba6 --- /dev/null +++ b/.github/workflows/cli-endpoints-online-deploy-with-packages-mlflow-model-endpoint.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-endpoints-online-deploy-with-packages-mlflow-model-endpoint +on: + workflow_dispatch: + schedule: + - cron: "18 11/12 * * *" + pull_request: + branches: + - main + paths: + - cli/endpoints/online/deploy-with-packages/mlflow-model/** + - cli/endpoints/online/** + - infra/bootstrapping/** + - .github/workflows/cli-endpoints-online-deploy-with-packages-mlflow-model-endpoint.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/endpoints/online/deploy-with-packages/mlflow-model" + working-directory: infra/bootstrapping + continue-on-error: false + - name: delete endpoint if existing + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml online-endpoint delete -n ckagesmlflowmodelendpoint1524 -y + working-directory: cli + continue-on-error: true + - name: create endpoint + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + cat endpoints/online/deploy-with-packages/mlflow-model/endpoint.yml + az ml online-endpoint create -n ckagesmlflowmodelendpoint1524 -f endpoints/online/deploy-with-packages/mlflow-model/endpoint.yml + working-directory: cli + - name: create deployment + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + cat endpoints/online/deploy-with-packages/mlflow-model/deployment.yml + az ml online-deployment create -e ckagesmlflowmodelendpoint1524 -f endpoints/online/deploy-with-packages/mlflow-model/deployment.yml + working-directory: cli + - name: create deployment + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + cat endpoints/online/deploy-with-packages/mlflow-model/model-deployment.yml + az ml online-deployment create -e ckagesmlflowmodelendpoint1524 -f endpoints/online/deploy-with-packages/mlflow-model/model-deployment.yml + working-directory: cli + - name: cleanup endpoint + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + az ml online-endpoint delete -n ckagesmlflowmodelendpoint1524 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-online-kubernetes-kubernetes-endpoint.yml b/.github/workflows/cli-endpoints-online-kubernetes-kubernetes-endpoint.yml index aefd24ccd9..e93166a202 100644 --- a/.github/workflows/cli-endpoints-online-kubernetes-kubernetes-endpoint.yml +++ b/.github/workflows/cli-endpoints-online-kubernetes-kubernetes-endpoint.yml @@ -60,18 +60,11 @@ jobs: python check-readme.py "${{ github.workspace }}/cli/endpoints/online/kubernetes" working-directory: infra/bootstrapping continue-on-error: false - - name: ensure k8s compute - run: | - source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; - source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" ensure_k8s_compute; - working-directory: cli - continue-on-error: true - name: delete endpoint if existing run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n berneteskubernetesendpoint4233 -y + az ml online-endpoint delete -n berneteskubernetesendpoint8085 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -79,18 +72,18 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/kubernetes/kubernetes-endpoint.yml - az ml online-endpoint create -n berneteskubernetesendpoint4233 -f endpoints/online/kubernetes/kubernetes-endpoint.yml + az ml online-endpoint create -n berneteskubernetesendpoint8085 -f endpoints/online/kubernetes/kubernetes-endpoint.yml working-directory: cli - name: create deployment run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/kubernetes/kubernetes-blue-deployment.yml - az ml online-deployment create -e berneteskubernetesendpoint4233 -f endpoints/online/kubernetes/kubernetes-blue-deployment.yml + az ml online-deployment create -e berneteskubernetesendpoint8085 -f endpoints/online/kubernetes/kubernetes-blue-deployment.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n berneteskubernetesendpoint4233 -y - working-directory: cli \ No newline at end of file + az ml online-endpoint delete -n berneteskubernetesendpoint8085 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-online-managed-sample-endpoint.yml b/.github/workflows/cli-endpoints-online-managed-sample-endpoint.yml index f3bd319014..006620cabd 100644 --- a/.github/workflows/cli-endpoints-online-managed-sample-endpoint.yml +++ b/.github/workflows/cli-endpoints-online-managed-sample-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n linemanagedsampleendpoint6303 -y + az ml online-endpoint delete -n linemanagedsampleendpoint9077 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,25 +72,25 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/managed/sample/endpoint.yml - az ml online-endpoint create -n linemanagedsampleendpoint6303 -f endpoints/online/managed/sample/endpoint.yml + az ml online-endpoint create -n linemanagedsampleendpoint9077 -f endpoints/online/managed/sample/endpoint.yml working-directory: cli - name: create deployment run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/managed/sample/blue-deployment.yml - az ml online-deployment create -e linemanagedsampleendpoint6303 -f endpoints/online/managed/sample/blue-deployment.yml + az ml online-deployment create -e linemanagedsampleendpoint9077 -f endpoints/online/managed/sample/blue-deployment.yml working-directory: cli - name: create deployment run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/managed/sample/green-deployment.yml - az ml online-deployment create -e linemanagedsampleendpoint6303 -f endpoints/online/managed/sample/green-deployment.yml + az ml online-deployment create -e linemanagedsampleendpoint9077 -f endpoints/online/managed/sample/green-deployment.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n linemanagedsampleendpoint6303 -y - working-directory: cli \ No newline at end of file + az ml online-endpoint delete -n linemanagedsampleendpoint9077 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-online-managed-vnet-mlflow-endpoint.yml b/.github/workflows/cli-endpoints-online-managed-vnet-mlflow-endpoint.yml index 6f64bc30e0..b95d242ed4 100644 --- a/.github/workflows/cli-endpoints-online-managed-vnet-mlflow-endpoint.yml +++ b/.github/workflows/cli-endpoints-online-managed-vnet-mlflow-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n managedvnetmlflowendpoint6555 -y + az ml online-endpoint delete -n managedvnetmlflowendpoint6161 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,11 +72,11 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/managed/vnet/mlflow/endpoint.yml - az ml online-endpoint create -n managedvnetmlflowendpoint6555 -f endpoints/online/managed/vnet/mlflow/endpoint.yml + az ml online-endpoint create -n managedvnetmlflowendpoint6161 -f endpoints/online/managed/vnet/mlflow/endpoint.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n managedvnetmlflowendpoint6555 -y - working-directory: cli \ No newline at end of file + az ml online-endpoint delete -n managedvnetmlflowendpoint6161 -y + working-directory: cli diff --git a/.github/workflows/cli-endpoints-online-managed-vnet-sample-endpoint.yml b/.github/workflows/cli-endpoints-online-managed-vnet-sample-endpoint.yml index cc23446e9f..cfe8c793d1 100644 --- a/.github/workflows/cli-endpoints-online-managed-vnet-sample-endpoint.yml +++ b/.github/workflows/cli-endpoints-online-managed-vnet-sample-endpoint.yml @@ -64,7 +64,7 @@ jobs: run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n managedvnetsampleendpoint8383 -y + az ml online-endpoint delete -n managedvnetsampleendpoint9519 -y working-directory: cli continue-on-error: true - name: create endpoint @@ -72,11 +72,11 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; cat endpoints/online/managed/vnet/sample/endpoint.yml - az ml online-endpoint create -n managedvnetsampleendpoint8383 -f endpoints/online/managed/vnet/sample/endpoint.yml + az ml online-endpoint create -n managedvnetsampleendpoint9519 -f endpoints/online/managed/vnet/sample/endpoint.yml working-directory: cli - name: cleanup endpoint run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - az ml online-endpoint delete -n managedvnetsampleendpoint8383 -y - working-directory: cli \ No newline at end of file + az ml online-endpoint delete -n managedvnetsampleendpoint9519 -y + working-directory: cli diff --git a/.github/workflows/cli-jobs-automl-standalone-jobs-cli-automl-classification-task-bankmarketing-cli-automl-classification-task-bankmarketing-serverless.yml b/.github/workflows/cli-jobs-automl-standalone-jobs-cli-automl-classification-task-bankmarketing-cli-automl-classification-task-bankmarketing-serverless.yml new file mode 100644 index 0000000000..adae1236ed --- /dev/null +++ b/.github/workflows/cli-jobs-automl-standalone-jobs-cli-automl-classification-task-bankmarketing-cli-automl-classification-task-bankmarketing-serverless.yml @@ -0,0 +1,68 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-automl-standalone-jobs-cli-automl-classification-task-bankmarketing-cli-automl-classification-task-bankmarketing-serverless +on: + workflow_dispatch: + schedule: + - cron: "46 6/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-automl-standalone-jobs-cli-automl-classification-task-bankmarketing-cli-automl-classification-task-bankmarketing-serverless.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../../run-job.sh cli-automl-classification-task-bankmarketing-serverless.yml + working-directory: cli/jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-basics-hello-pipeline-abc-serverless.yml b/.github/workflows/cli-jobs-basics-hello-pipeline-abc-serverless.yml new file mode 100644 index 0000000000..15099db41d --- /dev/null +++ b/.github/workflows/cli-jobs-basics-hello-pipeline-abc-serverless.yml @@ -0,0 +1,68 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-basics-hello-pipeline-abc-serverless +on: + workflow_dispatch: + schedule: + - cron: "8 7/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/basics/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-basics-hello-pipeline-abc-serverless.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../run-job.sh hello-pipeline-abc-serverless.yml + working-directory: cli/jobs/basics + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/basics" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-basics-hello-pipeline-customize-output-file-serverless.yml b/.github/workflows/cli-jobs-basics-hello-pipeline-customize-output-file-serverless.yml new file mode 100644 index 0000000000..f7888c6e30 --- /dev/null +++ b/.github/workflows/cli-jobs-basics-hello-pipeline-customize-output-file-serverless.yml @@ -0,0 +1,68 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-basics-hello-pipeline-customize-output-file-serverless +on: + workflow_dispatch: + schedule: + - cron: "56 10/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/basics/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-basics-hello-pipeline-customize-output-file-serverless.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../run-job.sh hello-pipeline-customize-output-file-serverless.yml + working-directory: cli/jobs/basics + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/basics" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-basics-hello-pipeline-customize-output-folder-serverless.yml b/.github/workflows/cli-jobs-basics-hello-pipeline-customize-output-folder-serverless.yml new file mode 100644 index 0000000000..cf1aadad5d --- /dev/null +++ b/.github/workflows/cli-jobs-basics-hello-pipeline-customize-output-folder-serverless.yml @@ -0,0 +1,68 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-basics-hello-pipeline-customize-output-folder-serverless +on: + workflow_dispatch: + schedule: + - cron: "13 7/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/basics/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-basics-hello-pipeline-customize-output-folder-serverless.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../run-job.sh hello-pipeline-customize-output-folder-serverless.yml + working-directory: cli/jobs/basics + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/basics" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-basics-hello-pipeline-default-artifacts-serverless.yml b/.github/workflows/cli-jobs-basics-hello-pipeline-default-artifacts-serverless.yml new file mode 100644 index 0000000000..1d16f859a5 --- /dev/null +++ b/.github/workflows/cli-jobs-basics-hello-pipeline-default-artifacts-serverless.yml @@ -0,0 +1,68 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-basics-hello-pipeline-default-artifacts-serverless +on: + workflow_dispatch: + schedule: + - cron: "38 3/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/basics/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-basics-hello-pipeline-default-artifacts-serverless.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../run-job.sh hello-pipeline-default-artifacts-serverless.yml + working-directory: cli/jobs/basics + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/basics" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-basics-hello-pipeline-io-serverless.yml b/.github/workflows/cli-jobs-basics-hello-pipeline-io-serverless.yml new file mode 100644 index 0000000000..b23a3b1a3c --- /dev/null +++ b/.github/workflows/cli-jobs-basics-hello-pipeline-io-serverless.yml @@ -0,0 +1,68 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-basics-hello-pipeline-io-serverless +on: + workflow_dispatch: + schedule: + - cron: "44 2/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/basics/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-basics-hello-pipeline-io-serverless.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../run-job.sh hello-pipeline-io-serverless.yml + working-directory: cli/jobs/basics + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/basics" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-basics-hello-pipeline-serverless.yml b/.github/workflows/cli-jobs-basics-hello-pipeline-serverless.yml new file mode 100644 index 0000000000..101aead0ad --- /dev/null +++ b/.github/workflows/cli-jobs-basics-hello-pipeline-serverless.yml @@ -0,0 +1,68 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-basics-hello-pipeline-serverless +on: + workflow_dispatch: + schedule: + - cron: "40 0/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/basics/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-basics-hello-pipeline-serverless.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../run-job.sh hello-pipeline-serverless.yml + working-directory: cli/jobs/basics + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/basics" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-basics-hello-pipeline-settings-serverless.yml b/.github/workflows/cli-jobs-basics-hello-pipeline-settings-serverless.yml new file mode 100644 index 0000000000..38147b4345 --- /dev/null +++ b/.github/workflows/cli-jobs-basics-hello-pipeline-settings-serverless.yml @@ -0,0 +1,68 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-basics-hello-pipeline-settings-serverless +on: + workflow_dispatch: + schedule: + - cron: "46 10/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/basics/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-basics-hello-pipeline-settings-serverless.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../run-job.sh hello-pipeline-settings-serverless.yml + working-directory: cli/jobs/basics + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/basics" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-nebulaml-cifar10_deepspeed-job.yml b/.github/workflows/cli-jobs-nebulaml-cifar10_deepspeed-job.yml new file mode 100644 index 0000000000..7cb819adb3 --- /dev/null +++ b/.github/workflows/cli-jobs-nebulaml-cifar10_deepspeed-job.yml @@ -0,0 +1,68 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-nebulaml-cifar10_deepspeed-job +on: + workflow_dispatch: + schedule: + - cron: "54 3/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/nebulaml/cifar10_deepspeed/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-nebulaml-cifar10_deepspeed-job.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../../run-job.sh job.yml + working-directory: cli/jobs/nebulaml/cifar10_deepspeed + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/nebulaml/cifar10_deepspeed" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-pipelines-add-column-and-word-count-using-spark-pipeline.yml b/.github/workflows/cli-jobs-pipelines-add-column-and-word-count-using-spark-pipeline.yml new file mode 100644 index 0000000000..5f460538e6 --- /dev/null +++ b/.github/workflows/cli-jobs-pipelines-add-column-and-word-count-using-spark-pipeline.yml @@ -0,0 +1,69 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-pipelines-add-column-and-word-count-using-spark-pipeline +on: + workflow_dispatch: + schedule: + - cron: "4 6/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/pipelines/add-column-and-word-count-using-spark/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-pipelines-add-column-and-word-count-using-spark-pipeline.yml + - cli/run-pipeline-jobs.sh + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../../run-job.sh pipeline.yml + working-directory: cli/jobs/pipelines/add-column-and-word-count-using-spark + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/pipelines/add-column-and-word-count-using-spark" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-hts-hts_evaluation_pipeline.yml b/.github/workflows/cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-hts-hts_evaluation_pipeline.yml new file mode 100644 index 0000000000..a34a4d59e5 --- /dev/null +++ b/.github/workflows/cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-hts-hts_evaluation_pipeline.yml @@ -0,0 +1,69 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-hts-hts_evaluation_pipeline +on: + workflow_dispatch: + schedule: + - cron: "55 9/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/pipelines/automl/cli-aml-demand-forecast-pipeline/cli-aml-demand-forecast-hts/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-hts-hts_evaluation_pipeline.yml + - cli/run-pipeline-jobs.sh + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../../../../run-job.sh hts_evaluation_pipeline.yml + working-directory: cli/jobs/pipelines/automl/cli-aml-demand-forecast-pipeline/cli-aml-demand-forecast-hts + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/pipelines/automl/cli-aml-demand-forecast-pipeline/cli-aml-demand-forecast-hts" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-mm-many_models_evaluation_pipeline.yml b/.github/workflows/cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-mm-many_models_evaluation_pipeline.yml new file mode 100644 index 0000000000..c3d13142df --- /dev/null +++ b/.github/workflows/cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-mm-many_models_evaluation_pipeline.yml @@ -0,0 +1,69 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-mm-many_models_evaluation_pipeline +on: + workflow_dispatch: + schedule: + - cron: "23 6/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/pipelines/automl/cli-aml-demand-forecast-pipeline/cli-aml-demand-forecast-mm/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-mm-many_models_evaluation_pipeline.yml + - cli/run-pipeline-jobs.sh + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../../../../run-job.sh many_models_evaluation_pipeline.yml + working-directory: cli/jobs/pipelines/automl/cli-aml-demand-forecast-pipeline/cli-aml-demand-forecast-mm + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/pipelines/automl/cli-aml-demand-forecast-pipeline/cli-aml-demand-forecast-mm" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-pipelines-automl-demand-forecast-pipeline-pipeline.yml b/.github/workflows/cli-jobs-pipelines-automl-demand-forecast-pipeline-pipeline.yml new file mode 100644 index 0000000000..1115d9bb42 --- /dev/null +++ b/.github/workflows/cli-jobs-pipelines-automl-demand-forecast-pipeline-pipeline.yml @@ -0,0 +1,69 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-pipelines-automl-demand-forecast-pipeline-pipeline +on: + workflow_dispatch: + schedule: + - cron: "4 6/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/pipelines/automl/demand-forecast-pipeline/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-pipelines-automl-demand-forecast-pipeline-pipeline.yml + - cli/run-pipeline-jobs.sh + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../../../run-job.sh pipeline.yml + working-directory: cli/jobs/pipelines/automl/demand-forecast-pipeline + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/pipelines/automl/demand-forecast-pipeline" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline-serverless-registry.yml b/.github/workflows/cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline-serverless-registry.yml new file mode 100644 index 0000000000..ad7f1eab97 --- /dev/null +++ b/.github/workflows/cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline-serverless-registry.yml @@ -0,0 +1,69 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline-serverless-registry +on: + workflow_dispatch: + schedule: + - cron: "50 1/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/pipelines-with-components/basics/1b_e2e_registered_components/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline-serverless-registry.yml + - cli/run-pipeline-jobs.sh + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/pipelines-with-components/basics/1b_e2e_registered_components" + working-directory: infra/bootstrapping + continue-on-error: false + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../../../run-pipeline-job-with-registry-components.sh pipeline-serverless 1b_e2e_registered_components + working-directory: cli/jobs/pipelines-with-components/basics/1b_e2e_registered_components diff --git a/.github/workflows/cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline-serverless.yml b/.github/workflows/cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline-serverless.yml new file mode 100644 index 0000000000..7a518a8b0d --- /dev/null +++ b/.github/workflows/cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline-serverless.yml @@ -0,0 +1,69 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline-serverless +on: + workflow_dispatch: + schedule: + - cron: "50 1/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/pipelines-with-components/basics/1b_e2e_registered_components/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline-serverless.yml + - cli/run-pipeline-jobs.sh + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../../../run-job.sh pipeline-serverless.yml + working-directory: cli/jobs/pipelines-with-components/basics/1b_e2e_registered_components + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/pipelines-with-components/basics/1b_e2e_registered_components" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-data_pipeline-data_pipeline.yml b/.github/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-data_pipeline-data_pipeline.yml new file mode 100644 index 0000000000..2c276e577d --- /dev/null +++ b/.github/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-data_pipeline-data_pipeline.yml @@ -0,0 +1,69 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-data_pipeline-data_pipeline +on: + workflow_dispatch: + schedule: + - cron: "44 8/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/pipelines-with-components/pipeline_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component/data_pipeline/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-data_pipeline-data_pipeline.yml + - cli/run-pipeline-jobs.sh + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../../../../run-job.sh data_pipeline.yml + working-directory: cli/jobs/pipelines-with-components/pipeline_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component/data_pipeline + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/pipelines-with-components/pipeline_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component/data_pipeline" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-train_pipeline-train_pipeline.yml b/.github/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-train_pipeline-train_pipeline.yml new file mode 100644 index 0000000000..f80bfcc4d6 --- /dev/null +++ b/.github/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-train_pipeline-train_pipeline.yml @@ -0,0 +1,69 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-train_pipeline-train_pipeline +on: + workflow_dispatch: + schedule: + - cron: "37 3/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/pipelines-with-components/pipeline_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component/train_pipeline/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-train_pipeline-train_pipeline.yml + - cli/run-pipeline-jobs.sh + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../../../../run-job.sh train_pipeline.yml + working-directory: cli/jobs/pipelines-with-components/pipeline_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component/train_pipeline + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/pipelines-with-components/pipeline_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component/train_pipeline" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-components-train_pipeline_component.yml b/.github/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-components-train_pipeline_component.yml new file mode 100644 index 0000000000..1c8663765e --- /dev/null +++ b/.github/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-components-train_pipeline_component.yml @@ -0,0 +1,69 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-components-train_pipeline_component +on: + workflow_dispatch: + schedule: + - cron: "53 3/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/pipelines-with-components/pipeline_with_pipeline_component/pipeline_with_train_eval_pipeline_component/components/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-components-train_pipeline_component.yml + - cli/run-pipeline-jobs.sh + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../../../../run-job.sh train_pipeline_component.yml + working-directory: cli/jobs/pipelines-with-components/pipeline_with_pipeline_component/pipeline_with_train_eval_pipeline_component/components + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/pipelines-with-components/pipeline_with_pipeline_component/pipeline_with_train_eval_pipeline_component/components" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-pipelines-with-components-shakespear_sample_and_word_count_using_spark-pipeline.yml b/.github/workflows/cli-jobs-pipelines-with-components-shakespear_sample_and_word_count_using_spark-pipeline.yml new file mode 100644 index 0000000000..0c87c26a16 --- /dev/null +++ b/.github/workflows/cli-jobs-pipelines-with-components-shakespear_sample_and_word_count_using_spark-pipeline.yml @@ -0,0 +1,69 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-pipelines-with-components-shakespear_sample_and_word_count_using_spark-pipeline +on: + workflow_dispatch: + schedule: + - cron: "4 6/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/pipelines-with-components/shakespear_sample_and_word_count_using_spark/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-pipelines-with-components-shakespear_sample_and_word_count_using_spark-pipeline.yml + - cli/run-pipeline-jobs.sh + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../../run-job.sh pipeline.yml + working-directory: cli/jobs/pipelines-with-components/shakespear_sample_and_word_count_using_spark + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/pipelines-with-components/shakespear_sample_and_word_count_using_spark" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-single-step-spark-nyctaxi-job.yml b/.github/workflows/cli-jobs-single-step-spark-nyctaxi-job.yml new file mode 100644 index 0000000000..25ddfa19f6 --- /dev/null +++ b/.github/workflows/cli-jobs-single-step-spark-nyctaxi-job.yml @@ -0,0 +1,68 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-jobs-single-step-spark-nyctaxi-job +on: + workflow_dispatch: + schedule: + - cron: "54 3/12 * * *" + pull_request: + branches: + - main + paths: + - cli/jobs/single-step/spark/nyctaxi/** + - infra/bootstrapping/** + - .github/workflows/cli-jobs-single-step-spark-nyctaxi-job.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run job + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash -x ../../../../run-job.sh job.yml + working-directory: cli/jobs/single-step/spark/nyctaxi + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/jobs/single-step/spark/nyctaxi" + working-directory: infra/bootstrapping + continue-on-error: false diff --git a/.github/workflows/cli-jobs-spark-attached-spark-pipeline-default-identity.yml b/.github/workflows/cli-jobs-spark-attached-spark-pipeline-default-identity.yml index adc95c2c72..7f247e36a1 100644 --- a/.github/workflows/cli-jobs-spark-attached-spark-pipeline-default-identity.yml +++ b/.github/workflows/cli-jobs-spark-attached-spark-pipeline-default-identity.yml @@ -65,7 +65,7 @@ jobs: working-directory: cli continue-on-error: true run: | - bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-system-identity.yml jobs/spark/attached-spark-pipeline-default-identity.yml + bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark.yml jobs/spark/attached-spark-pipeline-default-identity.yml - name: run job run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; diff --git a/.github/workflows/cli-jobs-spark-attached-spark-standalone-default-identity.yml b/.github/workflows/cli-jobs-spark-attached-spark-standalone-default-identity.yml index e8ee20d0df..62fe22d955 100644 --- a/.github/workflows/cli-jobs-spark-attached-spark-standalone-default-identity.yml +++ b/.github/workflows/cli-jobs-spark-attached-spark-standalone-default-identity.yml @@ -65,7 +65,7 @@ jobs: working-directory: cli continue-on-error: true run: | - bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-system-identity.yml jobs/spark/attached-spark-standalone-default-identity.yml + bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark.yml jobs/spark/attached-spark-standalone-default-identity.yml - name: run job run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; diff --git a/.github/workflows/cli-jobs-spark-serverless-spark-pipeline-default-identity.yml b/.github/workflows/cli-jobs-spark-serverless-spark-pipeline-default-identity.yml index cc7f8b41e2..dde0e7084d 100644 --- a/.github/workflows/cli-jobs-spark-serverless-spark-pipeline-default-identity.yml +++ b/.github/workflows/cli-jobs-spark-serverless-spark-pipeline-default-identity.yml @@ -60,6 +60,7 @@ jobs: run: | bash -x upload-data-to-blob.sh jobs/spark/ working-directory: cli + continue-on-error: true - name: run job run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; diff --git a/.github/workflows/cli-jobs-spark-serverless-spark-pipeline-user-identity.yml b/.github/workflows/cli-jobs-spark-serverless-spark-pipeline-user-identity.yml index d9784bc9c6..5d34205bb3 100644 --- a/.github/workflows/cli-jobs-spark-serverless-spark-pipeline-user-identity.yml +++ b/.github/workflows/cli-jobs-spark-serverless-spark-pipeline-user-identity.yml @@ -60,6 +60,7 @@ jobs: run: | bash -x upload-data-to-blob.sh jobs/spark/ working-directory: cli + continue-on-error: true - name: run job run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; diff --git a/.github/workflows/cli-jobs-spark-serverless-spark-standalone-default-identity.yml b/.github/workflows/cli-jobs-spark-serverless-spark-standalone-default-identity.yml index 7a447010a8..a5cd445ac2 100644 --- a/.github/workflows/cli-jobs-spark-serverless-spark-standalone-default-identity.yml +++ b/.github/workflows/cli-jobs-spark-serverless-spark-standalone-default-identity.yml @@ -60,6 +60,7 @@ jobs: run: | bash -x upload-data-to-blob.sh jobs/spark/ working-directory: cli + continue-on-error: true - name: run job run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; diff --git a/.github/workflows/cli-jobs-spark-serverless-spark-standalone-user-identity.yml b/.github/workflows/cli-jobs-spark-serverless-spark-standalone-user-identity.yml index e059d330b4..742f77341f 100644 --- a/.github/workflows/cli-jobs-spark-serverless-spark-standalone-user-identity.yml +++ b/.github/workflows/cli-jobs-spark-serverless-spark-standalone-user-identity.yml @@ -60,6 +60,7 @@ jobs: run: | bash -x upload-data-to-blob.sh jobs/spark/ working-directory: cli + continue-on-error: true - name: run job run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; diff --git a/.github/workflows/cli-resources-compute-cluster-basic.yml b/.github/workflows/cli-resources-compute-cluster-basic.yml index 551ceb8aa7..8b50bb6bd2 100644 --- a/.github/workflows/cli-resources-compute-cluster-basic.yml +++ b/.github/workflows/cli-resources-compute-cluster-basic.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-resources-compute-cluster-location.yml b/.github/workflows/cli-resources-compute-cluster-location.yml index de8de83c9d..9a7d182ffc 100644 --- a/.github/workflows/cli-resources-compute-cluster-location.yml +++ b/.github/workflows/cli-resources-compute-cluster-location.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-resources-compute-cluster-low-priority.yml b/.github/workflows/cli-resources-compute-cluster-low-priority.yml index 53b4274d1e..ed676591dd 100644 --- a/.github/workflows/cli-resources-compute-cluster-low-priority.yml +++ b/.github/workflows/cli-resources-compute-cluster-low-priority.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-resources-compute-cluster-minimal.yml b/.github/workflows/cli-resources-compute-cluster-minimal.yml index 8016f698fb..dcc72a9f9a 100644 --- a/.github/workflows/cli-resources-compute-cluster-minimal.yml +++ b/.github/workflows/cli-resources-compute-cluster-minimal.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-resources-compute-cluster-ssh-password.yml b/.github/workflows/cli-resources-compute-cluster-ssh-password.yml index 9d1cd60ee5..2c844542b5 100644 --- a/.github/workflows/cli-resources-compute-cluster-ssh-password.yml +++ b/.github/workflows/cli-resources-compute-cluster-ssh-password.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-resources-compute-cluster-system-identity.yml b/.github/workflows/cli-resources-compute-cluster-system-identity.yml index 1f33d0d0d9..10de6e9029 100644 --- a/.github/workflows/cli-resources-compute-cluster-system-identity.yml +++ b/.github/workflows/cli-resources-compute-cluster-system-identity.yml @@ -35,8 +35,8 @@ jobs: subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} - name: bootstrap resources run: | - bash bootstrap.sh - working-directory: infra/bootstrapping + bash bootstrapping/bootstrap.sh + working-directory: infra continue-on-error: false - name: setup-cli run: | diff --git a/.github/workflows/cli-scripts-batch-score.yml b/.github/workflows/cli-scripts-batch-score.yml index 169014a1bf..6b65c7b7c7 100644 --- a/.github/workflows/cli-scripts-batch-score.yml +++ b/.github/workflows/cli-scripts-batch-score.yml @@ -27,8 +27,6 @@ jobs: steps: - name: check out repo uses: actions/checkout@v2 - with: - lfs: true - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/cli-scripts-deploy-managed-online-endpoint-workspacevnet.yml b/.github/workflows/cli-scripts-deploy-managed-online-endpoint-workspacevnet.yml new file mode 100644 index 0000000000..c29c95c5b8 --- /dev/null +++ b/.github/workflows/cli-scripts-deploy-managed-online-endpoint-workspacevnet.yml @@ -0,0 +1,67 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-scripts-deploy-managed-online-endpoint-workspacevnet +on: + workflow_dispatch: + schedule: + - cron: "2 9/12 * * *" + pull_request: + branches: + - main + paths: + - cli/deploy-managed-online-endpoint-workspacevnet.sh + - infra/bootstrapping/** + - .github/workflows/cli-scripts-deploy-managed-online-endpoint-workspacevnet.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/" + working-directory: infra/bootstrapping + continue-on-error: false + - name: test script script + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + set -e; bash -x deploy-managed-online-endpoint-workspacevnet.sh + working-directory: cli diff --git a/.github/workflows/cli-scripts-deploy-moe-vnet-legacy.yml b/.github/workflows/cli-scripts-deploy-moe-vnet-legacy.yml new file mode 100644 index 0000000000..efb62d15a7 --- /dev/null +++ b/.github/workflows/cli-scripts-deploy-moe-vnet-legacy.yml @@ -0,0 +1,67 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-scripts-deploy-moe-vnet-legacy +on: + workflow_dispatch: + schedule: + - cron: "7 0/12 * * *" + pull_request: + branches: + - main + paths: + - cli/deploy-moe-vnet-legacy.sh + - infra/bootstrapping/** + - .github/workflows/cli-scripts-deploy-moe-vnet-legacy.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/" + working-directory: infra/bootstrapping + continue-on-error: false + - name: test script script + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + set -e; bash -x deploy-moe-vnet-legacy.sh + working-directory: cli diff --git a/.github/workflows/cli-scripts-deploy-moe-vnet-mlflow-legacy.yml b/.github/workflows/cli-scripts-deploy-moe-vnet-mlflow-legacy.yml new file mode 100644 index 0000000000..fb12d5d527 --- /dev/null +++ b/.github/workflows/cli-scripts-deploy-moe-vnet-mlflow-legacy.yml @@ -0,0 +1,67 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-scripts-deploy-moe-vnet-mlflow-legacy +on: + workflow_dispatch: + schedule: + - cron: "12 7/12 * * *" + pull_request: + branches: + - main + paths: + - cli/deploy-moe-vnet-mlflow-legacy.sh + - infra/bootstrapping/** + - .github/workflows/cli-scripts-deploy-moe-vnet-mlflow-legacy.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/" + working-directory: infra/bootstrapping + continue-on-error: false + - name: test script script + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + set -e; bash -x deploy-moe-vnet-mlflow-legacy.sh + working-directory: cli diff --git a/.github/workflows/cli-scripts-upload-data-to-blob.yml b/.github/workflows/cli-scripts-upload-data-to-blob.yml new file mode 100644 index 0000000000..d4915b0357 --- /dev/null +++ b/.github/workflows/cli-scripts-upload-data-to-blob.yml @@ -0,0 +1,67 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: cli-scripts-upload-data-to-blob +on: + workflow_dispatch: + schedule: + - cron: "49 3/12 * * *" + pull_request: + branches: + - main + paths: + - cli/upload-data-to-blob.sh + - infra/bootstrapping/** + - .github/workflows/cli-scripts-upload-data-to-blob.yml + - cli/setup.sh +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/cli/" + working-directory: infra/bootstrapping + continue-on-error: false + - name: test script script + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + set -e; bash -x upload-data-to-blob.sh + working-directory: cli diff --git a/.github/workflows/sdk-assets-assets-in-registry-share-data-using-registry.yml b/.github/workflows/sdk-assets-assets-in-registry-share-data-using-registry.yml index fed21f71b2..8f66026396 100644 --- a/.github/workflows/sdk-assets-assets-in-registry-share-data-using-registry.yml +++ b/.github/workflows/sdk-assets-assets-in-registry-share-data-using-registry.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-assets-assets-in-registry-share-models-components-environments.yml b/.github/workflows/sdk-assets-assets-in-registry-share-models-components-environments.yml index 291900f03c..8c477ca157 100644 --- a/.github/workflows/sdk-assets-assets-in-registry-share-models-components-environments.yml +++ b/.github/workflows/sdk-assets-assets-in-registry-share-models-components-environments.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-assets-component-component.yml b/.github/workflows/sdk-assets-component-component.yml index d0b30fdce1..a8f7493d41 100644 --- a/.github/workflows/sdk-assets-component-component.yml +++ b/.github/workflows/sdk-assets-component-component.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-assets-data-data.yml b/.github/workflows/sdk-assets-data-data.yml index 306b21c08a..374952489c 100644 --- a/.github/workflows/sdk-assets-data-data.yml +++ b/.github/workflows/sdk-assets-data-data.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-assets-data-versioning.yml b/.github/workflows/sdk-assets-data-versioning.yml index 05cb87bd8a..83162ff6a5 100644 --- a/.github/workflows/sdk-assets-data-versioning.yml +++ b/.github/workflows/sdk-assets-data-versioning.yml @@ -34,7 +34,7 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-assets-data-working_with_mltable.yml b/.github/workflows/sdk-assets-data-working_with_mltable.yml index f887ed94ff..fd53b05937 100644 --- a/.github/workflows/sdk-assets-data-working_with_mltable.yml +++ b/.github/workflows/sdk-assets-data-working_with_mltable.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-assets-environment-environment-with-private-packages-environment-with-private-package-conda-spec.yml b/.github/workflows/sdk-assets-environment-environment-with-private-packages-environment-with-private-package-conda-spec.yml new file mode 100644 index 0000000000..66c61f6e91 --- /dev/null +++ b/.github/workflows/sdk-assets-environment-environment-with-private-packages-environment-with-private-package-conda-spec.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-assets-environment-environment-with-private-packages-environment-with-private-package-conda-spec +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "1 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/assets/environment/environment-with-private-packages/** + - .github/workflows/sdk-assets-environment-environment-with-private-packages-environment-with-private-package-conda-spec.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/assets/environment/environment-with-private-packages" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run assets/environment/environment-with-private-packages/environment-with-private-package-conda-spec.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "environment-with-private-package-conda-spec.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python environment-with-private-package-conda-spec.ipynb environment-with-private-package-conda-spec.output.ipynb + working-directory: sdk/python/assets/environment/environment-with-private-packages + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: environment-with-private-package-conda-spec + path: sdk/python/assets/environment/environment-with-private-packages diff --git a/.github/workflows/sdk-assets-environment-environment-with-private-packages-environment-with-private-package-docker-image.yml b/.github/workflows/sdk-assets-environment-environment-with-private-packages-environment-with-private-package-docker-image.yml new file mode 100644 index 0000000000..0603dae77e --- /dev/null +++ b/.github/workflows/sdk-assets-environment-environment-with-private-packages-environment-with-private-package-docker-image.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-assets-environment-environment-with-private-packages-environment-with-private-package-docker-image +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "11 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/assets/environment/environment-with-private-packages/** + - .github/workflows/sdk-assets-environment-environment-with-private-packages-environment-with-private-package-docker-image.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/assets/environment/environment-with-private-packages" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run assets/environment/environment-with-private-packages/environment-with-private-package-docker-image.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "environment-with-private-package-docker-image.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python environment-with-private-package-docker-image.ipynb environment-with-private-package-docker-image.output.ipynb + working-directory: sdk/python/assets/environment/environment-with-private-packages + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: environment-with-private-package-docker-image + path: sdk/python/assets/environment/environment-with-private-packages diff --git a/.github/workflows/sdk-assets-environment-environment.yml b/.github/workflows/sdk-assets-environment-environment.yml index cab2b51cfc..b9e0132e96 100644 --- a/.github/workflows/sdk-assets-environment-environment.yml +++ b/.github/workflows/sdk-assets-environment-environment.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-assets-model-model.yml b/.github/workflows/sdk-assets-model-model.yml index 631d50b7cc..2beec43d40 100644 --- a/.github/workflows/sdk-assets-model-model.yml +++ b/.github/workflows/sdk-assets-model-model.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-basic-installation-basic-installation.yml b/.github/workflows/sdk-basic-installation-basic-installation.yml index b91a482b31..b21e2aa5aa 100644 --- a/.github/workflows/sdk-basic-installation-basic-installation.yml +++ b/.github/workflows/sdk-basic-installation-basic-installation.yml @@ -9,7 +9,7 @@ name: sdk-basic-installation-basic-installation on: workflow_dispatch: schedule: - - cron: "*/5 * * * *" + - cron: "53 3/12 * * *" pull_request: branches: - main @@ -34,7 +34,7 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-endpoints-batch-deploy-models-custom-outputs-parquet-custom-output-batch.yml b/.github/workflows/sdk-endpoints-batch-deploy-models-custom-outputs-parquet-custom-output-batch.yml index 724b377b6c..b3bb2cb271 100644 --- a/.github/workflows/sdk-endpoints-batch-deploy-models-custom-outputs-parquet-custom-output-batch.yml +++ b/.github/workflows/sdk-endpoints-batch-deploy-models-custom-outputs-parquet-custom-output-batch.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-batch-deploy-models-heart-classifier-mlflow-mlflow-for-batch-tabular.yml b/.github/workflows/sdk-endpoints-batch-deploy-models-heart-classifier-mlflow-mlflow-for-batch-tabular.yml index 17a78a2646..da0609f1ae 100644 --- a/.github/workflows/sdk-endpoints-batch-deploy-models-heart-classifier-mlflow-mlflow-for-batch-tabular.yml +++ b/.github/workflows/sdk-endpoints-batch-deploy-models-heart-classifier-mlflow-mlflow-for-batch-tabular.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-endpoints-batch-deploy-models-huggingface-text-summarization-text-summarization-batch.yml b/.github/workflows/sdk-endpoints-batch-deploy-models-huggingface-text-summarization-text-summarization-batch.yml index 033f886711..332477f87e 100644 --- a/.github/workflows/sdk-endpoints-batch-deploy-models-huggingface-text-summarization-text-summarization-batch.yml +++ b/.github/workflows/sdk-endpoints-batch-deploy-models-huggingface-text-summarization-text-summarization-batch.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-endpoints-batch-deploy-models-imagenet-classifier-imagenet-classifier-batch.yml b/.github/workflows/sdk-endpoints-batch-deploy-models-imagenet-classifier-imagenet-classifier-batch.yml index 4c5f56258c..b32e1611cc 100644 --- a/.github/workflows/sdk-endpoints-batch-deploy-models-imagenet-classifier-imagenet-classifier-batch.yml +++ b/.github/workflows/sdk-endpoints-batch-deploy-models-imagenet-classifier-imagenet-classifier-batch.yml @@ -31,14 +31,12 @@ jobs: steps: - name: check out repo uses: actions/checkout@v2 - with: - lfs: true - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-batch-deploy-models-imagenet-classifier-imagenet-classifier-mlflow.yml b/.github/workflows/sdk-endpoints-batch-deploy-models-imagenet-classifier-imagenet-classifier-mlflow.yml index 17179937bc..53f388c289 100644 --- a/.github/workflows/sdk-endpoints-batch-deploy-models-imagenet-classifier-imagenet-classifier-mlflow.yml +++ b/.github/workflows/sdk-endpoints-batch-deploy-models-imagenet-classifier-imagenet-classifier-mlflow.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -32,14 +31,12 @@ jobs: steps: - name: check out repo uses: actions/checkout@v2 - with: - lfs: true - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-endpoints-batch-deploy-models-mnist-classifier-mnist-batch.yml b/.github/workflows/sdk-endpoints-batch-deploy-models-mnist-classifier-mnist-batch.yml index b563db4c9a..fb0492b803 100644 --- a/.github/workflows/sdk-endpoints-batch-deploy-models-mnist-classifier-mnist-batch.yml +++ b/.github/workflows/sdk-endpoints-batch-deploy-models-mnist-classifier-mnist-batch.yml @@ -31,14 +31,12 @@ jobs: steps: - name: check out repo uses: actions/checkout@v2 - with: - lfs: true - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-batch-deploy-models-openai-embeddings-deploy-and-test.yml b/.github/workflows/sdk-endpoints-batch-deploy-models-openai-embeddings-deploy-and-test.yml new file mode 100644 index 0000000000..0a970817ef --- /dev/null +++ b/.github/workflows/sdk-endpoints-batch-deploy-models-openai-embeddings-deploy-and-test.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-endpoints-batch-deploy-models-openai-embeddings-deploy-and-test +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "10 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/endpoints/batch/deploy-models/openai-embeddings/** + - .github/workflows/sdk-endpoints-batch-deploy-models-openai-embeddings-deploy-and-test.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/endpoints/batch/deploy-models/openai-embeddings" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run endpoints/batch/deploy-models/openai-embeddings/deploy-and-test.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "deploy-and-test.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python deploy-and-test.ipynb deploy-and-test.output.ipynb --log-output + working-directory: sdk/python/endpoints/batch/deploy-models/openai-embeddings + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: deploy-and-test + path: sdk/python/endpoints/batch/deploy-models/openai-embeddings diff --git a/.github/workflows/sdk-endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-sdk-deploy-and-test.yml b/.github/workflows/sdk-endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-sdk-deploy-and-test.yml index 344dda97d8..b9e6d60274 100644 --- a/.github/workflows/sdk-endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-sdk-deploy-and-test.yml +++ b/.github/workflows/sdk-endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-sdk-deploy-and-test.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-batch-deploy-pipelines-hello-batch-sdk-deploy-and-test.yml b/.github/workflows/sdk-endpoints-batch-deploy-pipelines-hello-batch-sdk-deploy-and-test.yml index fe488b8b1a..c25632a29e 100644 --- a/.github/workflows/sdk-endpoints-batch-deploy-pipelines-hello-batch-sdk-deploy-and-test.yml +++ b/.github/workflows/sdk-endpoints-batch-deploy-pipelines-hello-batch-sdk-deploy-and-test.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-batch-deploy-pipelines-training-with-components-sdk-deploy-and-test.yml b/.github/workflows/sdk-endpoints-batch-deploy-pipelines-training-with-components-sdk-deploy-and-test.yml index 0809b7baf8..b8dbde54c1 100644 --- a/.github/workflows/sdk-endpoints-batch-deploy-pipelines-training-with-components-sdk-deploy-and-test.yml +++ b/.github/workflows/sdk-endpoints-batch-deploy-pipelines-training-with-components-sdk-deploy-and-test.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-custom-container-online-endpoints-custom-container-multimodel.yml b/.github/workflows/sdk-endpoints-online-custom-container-online-endpoints-custom-container-multimodel.yml index ae2d19d9ed..b1274a7b0d 100644 --- a/.github/workflows/sdk-endpoints-online-custom-container-online-endpoints-custom-container-multimodel.yml +++ b/.github/workflows/sdk-endpoints-online-custom-container-online-endpoints-custom-container-multimodel.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-custom-container-online-endpoints-custom-container.yml b/.github/workflows/sdk-endpoints-online-custom-container-online-endpoints-custom-container.yml index 12cf1015c7..ab6c593600 100644 --- a/.github/workflows/sdk-endpoints-online-custom-container-online-endpoints-custom-container.yml +++ b/.github/workflows/sdk-endpoints-online-custom-container-online-endpoints-custom-container.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-custom-container-triton-online-endpoints-triton-cc.yml b/.github/workflows/sdk-endpoints-online-custom-container-triton-online-endpoints-triton-cc.yml index a07313df7a..20c2270b1b 100644 --- a/.github/workflows/sdk-endpoints-online-custom-container-triton-online-endpoints-triton-cc.yml +++ b/.github/workflows/sdk-endpoints-online-custom-container-triton-online-endpoints-triton-cc.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-deploy-with-packages-custom-model-sdk-deploy-and-test.yml b/.github/workflows/sdk-endpoints-online-deploy-with-packages-custom-model-sdk-deploy-and-test.yml new file mode 100644 index 0000000000..7a5ce5fb54 --- /dev/null +++ b/.github/workflows/sdk-endpoints-online-deploy-with-packages-custom-model-sdk-deploy-and-test.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-endpoints-online-deploy-with-packages-custom-model-sdk-deploy-and-test +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "29 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/endpoints/online/deploy-with-packages/custom-model/** + - .github/workflows/sdk-endpoints-online-deploy-with-packages-custom-model-sdk-deploy-and-test.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/endpoints/online/deploy-with-packages/custom-model" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run endpoints/online/deploy-with-packages/custom-model/sdk-deploy-and-test.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "sdk-deploy-and-test.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python sdk-deploy-and-test.ipynb sdk-deploy-and-test.output.ipynb + working-directory: sdk/python/endpoints/online/deploy-with-packages/custom-model + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: sdk-deploy-and-test + path: sdk/python/endpoints/online/deploy-with-packages/custom-model diff --git a/.github/workflows/sdk-endpoints-online-deploy-with-packages-mlflow-model-sdk-deploy-and-test.yml b/.github/workflows/sdk-endpoints-online-deploy-with-packages-mlflow-model-sdk-deploy-and-test.yml new file mode 100644 index 0000000000..67dc7518cf --- /dev/null +++ b/.github/workflows/sdk-endpoints-online-deploy-with-packages-mlflow-model-sdk-deploy-and-test.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-endpoints-online-deploy-with-packages-mlflow-model-sdk-deploy-and-test +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "29 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/endpoints/online/deploy-with-packages/mlflow-model/** + - .github/workflows/sdk-endpoints-online-deploy-with-packages-mlflow-model-sdk-deploy-and-test.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/endpoints/online/deploy-with-packages/mlflow-model" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run endpoints/online/deploy-with-packages/mlflow-model/sdk-deploy-and-test.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "sdk-deploy-and-test.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python sdk-deploy-and-test.ipynb sdk-deploy-and-test.output.ipynb + working-directory: sdk/python/endpoints/online/deploy-with-packages/mlflow-model + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: sdk-deploy-and-test + path: sdk/python/endpoints/online/deploy-with-packages/mlflow-model diff --git a/.github/workflows/sdk-endpoints-online-deploy-with-packages-registry-model-sdk-deploy-and-test.yml b/.github/workflows/sdk-endpoints-online-deploy-with-packages-registry-model-sdk-deploy-and-test.yml new file mode 100644 index 0000000000..1ccf97700f --- /dev/null +++ b/.github/workflows/sdk-endpoints-online-deploy-with-packages-registry-model-sdk-deploy-and-test.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-endpoints-online-deploy-with-packages-registry-model-sdk-deploy-and-test +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "29 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/endpoints/online/deploy-with-packages/registry-model/** + - .github/workflows/sdk-endpoints-online-deploy-with-packages-registry-model-sdk-deploy-and-test.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/endpoints/online/deploy-with-packages/registry-model" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run endpoints/online/deploy-with-packages/registry-model/sdk-deploy-and-test.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "sdk-deploy-and-test.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python sdk-deploy-and-test.ipynb sdk-deploy-and-test.output.ipynb + working-directory: sdk/python/endpoints/online/deploy-with-packages/registry-model + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: sdk-deploy-and-test + path: sdk/python/endpoints/online/deploy-with-packages/registry-model diff --git a/.github/workflows/sdk-endpoints-online-kubernetes-kubernetes-online-endpoints-safe-rollout.yml b/.github/workflows/sdk-endpoints-online-kubernetes-kubernetes-online-endpoints-safe-rollout.yml index 50eb7e9d1f..ec64867d83 100644 --- a/.github/workflows/sdk-endpoints-online-kubernetes-kubernetes-online-endpoints-safe-rollout.yml +++ b/.github/workflows/sdk-endpoints-online-kubernetes-kubernetes-online-endpoints-safe-rollout.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-kubernetes-kubernetes-online-endpoints-simple-deployment.yml b/.github/workflows/sdk-endpoints-online-kubernetes-kubernetes-online-endpoints-simple-deployment.yml index 3326e26577..cd2aeeee36 100644 --- a/.github/workflows/sdk-endpoints-online-kubernetes-kubernetes-online-endpoints-simple-deployment.yml +++ b/.github/workflows/sdk-endpoints-online-kubernetes-kubernetes-online-endpoints-simple-deployment.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-managed-managed-identities-online-endpoints-managed-identity-sai.yml b/.github/workflows/sdk-endpoints-online-managed-managed-identities-online-endpoints-managed-identity-sai.yml index 27a6d1295b..e30f0ba1c9 100644 --- a/.github/workflows/sdk-endpoints-online-managed-managed-identities-online-endpoints-managed-identity-sai.yml +++ b/.github/workflows/sdk-endpoints-online-managed-managed-identities-online-endpoints-managed-identity-sai.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-managed-managed-identities-online-endpoints-managed-identity-uai.yml b/.github/workflows/sdk-endpoints-online-managed-managed-identities-online-endpoints-managed-identity-uai.yml index 6c5152a719..d2541b5a65 100644 --- a/.github/workflows/sdk-endpoints-online-managed-managed-identities-online-endpoints-managed-identity-uai.yml +++ b/.github/workflows/sdk-endpoints-online-managed-managed-identities-online-endpoints-managed-identity-uai.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-binary-payloads.yml b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-binary-payloads.yml index 1806972a42..a3bd2f92e2 100644 --- a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-binary-payloads.yml +++ b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-binary-payloads.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-inference-schema.yml b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-inference-schema.yml index 73f3a02da5..584041eadf 100644 --- a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-inference-schema.yml +++ b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-inference-schema.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-keyvault.yml b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-keyvault.yml index 3fe5f7e624..4aa5abefe1 100644 --- a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-keyvault.yml +++ b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-keyvault.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-multimodel.yml b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-multimodel.yml index 56eb7e62b6..bd30962e90 100644 --- a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-multimodel.yml +++ b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-multimodel.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-openapi.yml b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-openapi.yml index a35a5b3ab0..fa8ecd89fa 100644 --- a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-openapi.yml +++ b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-openapi.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-safe-rollout.yml b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-safe-rollout.yml index 1643cd653f..ae4e764cc3 100644 --- a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-safe-rollout.yml +++ b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-safe-rollout.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-simple-deployment.yml b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-simple-deployment.yml index 806f7899b2..c2af35ebad 100644 --- a/.github/workflows/sdk-endpoints-online-managed-online-endpoints-simple-deployment.yml +++ b/.github/workflows/sdk-endpoints-online-managed-online-endpoints-simple-deployment.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model-with-script.yml b/.github/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model-with-script.yml index c37b5a0ec9..1cbbf4a7f9 100644 --- a/.github/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model-with-script.yml +++ b/.github/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model-with-script.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: @@ -84,7 +84,7 @@ jobs: bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "online-endpoints-deploy-mlflow-model-with-script.ipynb"; [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; - papermill -k python online-endpoints-deploy-mlflow-model-with-script.ipynb online-endpoints-deploy-mlflow-model-with-script.output.ipynb --log-output + papermill -k python online-endpoints-deploy-mlflow-model-with-script.ipynb online-endpoints-deploy-mlflow-model-with-script.output.ipynb working-directory: sdk/python/endpoints/online/mlflow - name: upload notebook's working folder as an artifact if: ${{ always() }} diff --git a/.github/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model.yml b/.github/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model.yml index 1a26a9a3a4..88dd3ac092 100644 --- a/.github/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model.yml +++ b/.github/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: @@ -84,7 +84,7 @@ jobs: bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "online-endpoints-deploy-mlflow-model.ipynb"; [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; - papermill -k python online-endpoints-deploy-mlflow-model.ipynb online-endpoints-deploy-mlflow-model.output.ipynb --log-output + papermill -k python online-endpoints-deploy-mlflow-model.ipynb online-endpoints-deploy-mlflow-model.output.ipynb working-directory: sdk/python/endpoints/online/mlflow - name: upload notebook's working folder as an artifact if: ${{ always() }} diff --git a/.github/workflows/sdk-endpoints-online-triton-single-model-online-endpoints-triton.yml b/.github/workflows/sdk-endpoints-online-triton-single-model-online-endpoints-triton.yml index 7a598a7129..3977c5b6b3 100644 --- a/.github/workflows/sdk-endpoints-online-triton-single-model-online-endpoints-triton.yml +++ b/.github/workflows/sdk-endpoints-online-triton-single-model-online-endpoints-triton.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: @@ -84,7 +84,7 @@ jobs: bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "online-endpoints-triton.ipynb"; [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; - papermill -k python online-endpoints-triton.ipynb online-endpoints-triton.output.ipynb --log-output + papermill -k python online-endpoints-triton.ipynb online-endpoints-triton.output.ipynb working-directory: sdk/python/endpoints/online/triton/single-model - name: upload notebook's working folder as an artifact if: ${{ always() }} diff --git a/.github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_cli_samples.yml b/.github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_cli_samples.yml new file mode 100644 index 0000000000..4bee6f9bea --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_cli_samples.yml @@ -0,0 +1,99 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-automation-test-test_featurestore_cli_samples +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "56 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/automation-test/** + - .github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_cli_samples.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/automation-test" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources + run: | + bash -x automation-test/setup-resources-cli.sh automation-test/test_featurestore_cli_samples.ipynb + working-directory: sdk/python/featurestore_sample + continue-on-error: true + - name: run featurestore_sample/automation-test/test_featurestore_cli_samples.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "test_featurestore_cli_samples.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python test_featurestore_cli_samples.ipynb test_featurestore_cli_samples.output.ipynb + working-directory: sdk/python/featurestore_sample/automation-test + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: test_featurestore_cli_samples + path: sdk/python/featurestore_sample/automation-test diff --git a/.github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_sdk_samples.yml b/.github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_sdk_samples.yml new file mode 100644 index 0000000000..95421d0b63 --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_sdk_samples.yml @@ -0,0 +1,99 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-automation-test-test_featurestore_sdk_samples +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "22 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/automation-test/** + - .github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_sdk_samples.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/automation-test" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources + run: | + bash -x automation-test/setup-resources.sh automation-test/test_featurestore_sdk_samples.ipynb + working-directory: sdk/python/featurestore_sample + continue-on-error: true + - name: run featurestore_sample/automation-test/test_featurestore_sdk_samples.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "test_featurestore_sdk_samples.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python test_featurestore_sdk_samples.ipynb test_featurestore_sdk_samples.output.ipynb + working-directory: sdk/python/featurestore_sample/automation-test + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: test_featurestore_sdk_samples + path: sdk/python/featurestore_sample/automation-test diff --git a/.github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_vnet_samples.yml b/.github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_vnet_samples.yml new file mode 100644 index 0000000000..c7f4562b1c --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_vnet_samples.yml @@ -0,0 +1,99 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-automation-test-test_featurestore_vnet_samples +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "3 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/automation-test/** + - .github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_vnet_samples.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/automation-test" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources + run: | + bash -x automation-test/setup-resources-vnet.sh automation-test/test_featurestore_vnet_samples.ipynb + working-directory: sdk/python/featurestore_sample + continue-on-error: true + - name: run featurestore_sample/automation-test/test_featurestore_vnet_samples.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "test_featurestore_vnet_samples.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python test_featurestore_vnet_samples.ipynb test_featurestore_vnet_samples.output.ipynb + working-directory: sdk/python/featurestore_sample/automation-test + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: test_featurestore_vnet_samples + path: sdk/python/featurestore_sample/automation-test diff --git a/.github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-1.Develop-feature-set-and-register.yml b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-1.Develop-feature-set-and-register.yml new file mode 100644 index 0000000000..e940ecdcbe --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-1.Develop-feature-set-and-register.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-notebooks-sdk_and_cli-1.Develop-feature-set-and-register +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "37 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/notebooks/sdk_and_cli/** + - .github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-1.Develop-feature-set-and-register.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/notebooks/sdk_and_cli" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources - name: run featurestore_sample/notebooks/sdk_and_cli/1.Develop-feature-set-and-register.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "1.Develop-feature-set-and-register.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 1.Develop-feature-set-and-register.ipynb 1.Develop-feature-set-and-register.output.ipynb + working-directory: sdk/python/featurestore_sample/notebooks/sdk_and_cli + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 1.Develop-feature-set-and-register + path: sdk/python/featurestore_sample/notebooks/sdk_and_cli diff --git a/.github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-2.Experiment-train-models-using-features.yml b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-2.Experiment-train-models-using-features.yml new file mode 100644 index 0000000000..30215e6335 --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-2.Experiment-train-models-using-features.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-notebooks-sdk_and_cli-2.Experiment-train-models-using-features +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "57 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/notebooks/sdk_and_cli/** + - .github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-2.Experiment-train-models-using-features.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/notebooks/sdk_and_cli" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources - name: run featurestore_sample/notebooks/sdk_and_cli/2.Experiment-train-models-using-features.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "2.Experiment-train-models-using-features.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 2.Experiment-train-models-using-features.ipynb 2.Experiment-train-models-using-features.output.ipynb + working-directory: sdk/python/featurestore_sample/notebooks/sdk_and_cli + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 2.Experiment-train-models-using-features + path: sdk/python/featurestore_sample/notebooks/sdk_and_cli diff --git a/.github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-3.Enable-recurrent-materialization-run-batch-inference.yml b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-3.Enable-recurrent-materialization-run-batch-inference.yml new file mode 100644 index 0000000000..4efb58f404 --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-3.Enable-recurrent-materialization-run-batch-inference.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-notebooks-sdk_and_cli-3.Enable-recurrent-materialization-run-batch-inference +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "48 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/notebooks/sdk_and_cli/** + - .github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-3.Enable-recurrent-materialization-run-batch-inference.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/notebooks/sdk_and_cli" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources - name: run featurestore_sample/notebooks/sdk_and_cli/3.Enable-recurrent-materialization-run-batch-inference.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "3.Enable-recurrent-materialization-run-batch-inference.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 3.Enable-recurrent-materialization-run-batch-inference.ipynb 3.Enable-recurrent-materialization-run-batch-inference.output.ipynb + working-directory: sdk/python/featurestore_sample/notebooks/sdk_and_cli + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 3.Enable-recurrent-materialization-run-batch-inference + path: sdk/python/featurestore_sample/notebooks/sdk_and_cli diff --git a/.github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-network_isolation-Network-isolation-feature-store.yml b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-network_isolation-Network-isolation-feature-store.yml new file mode 100644 index 0000000000..5c61b95438 --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-network_isolation-Network-isolation-feature-store.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-notebooks-sdk_and_cli-network_isolation-Network-isolation-feature-store +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "37 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/notebooks/sdk_and_cli/network_isolation/** + - .github/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-network_isolation-Network-isolation-feature-store.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/notebooks/sdk_and_cli/network_isolation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources - name: run featurestore_sample/notebooks/sdk_and_cli/network_isolation/Network-isolation-feature-store.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "Network-isolation-feature-store.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python Network-isolation-feature-store.ipynb Network-isolation-feature-store.output.ipynb + working-directory: sdk/python/featurestore_sample/notebooks/sdk_and_cli/network_isolation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: Network-isolation-feature-store + path: sdk/python/featurestore_sample/notebooks/sdk_and_cli/network_isolation diff --git a/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-1.Develop-feature-set-and-register.yml b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-1.Develop-feature-set-and-register.yml new file mode 100644 index 0000000000..ea621ae7c2 --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-1.Develop-feature-set-and-register.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-notebooks-sdk_only-1.Develop-feature-set-and-register +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "37 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/notebooks/sdk_only/** + - .github/workflows/sdk-featurestore_sample-notebooks-sdk_only-1.Develop-feature-set-and-register.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/notebooks/sdk_only" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources - name: run featurestore_sample/notebooks/sdk_only/1.Develop-feature-set-and-register.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "1.Develop-feature-set-and-register.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 1.Develop-feature-set-and-register.ipynb 1.Develop-feature-set-and-register.output.ipynb + working-directory: sdk/python/featurestore_sample/notebooks/sdk_only + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 1.Develop-feature-set-and-register + path: sdk/python/featurestore_sample/notebooks/sdk_only diff --git a/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-2.Experiment-train-models-using-features.yml b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-2.Experiment-train-models-using-features.yml new file mode 100644 index 0000000000..5ad87e7974 --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-2.Experiment-train-models-using-features.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-notebooks-sdk_only-2.Experiment-train-models-using-features +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "57 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/notebooks/sdk_only/** + - .github/workflows/sdk-featurestore_sample-notebooks-sdk_only-2.Experiment-train-models-using-features.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/notebooks/sdk_only" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources - name: run featurestore_sample/notebooks/sdk_only/2.Experiment-train-models-using-features.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "2.Experiment-train-models-using-features.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 2.Experiment-train-models-using-features.ipynb 2.Experiment-train-models-using-features.output.ipynb + working-directory: sdk/python/featurestore_sample/notebooks/sdk_only + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 2.Experiment-train-models-using-features + path: sdk/python/featurestore_sample/notebooks/sdk_only diff --git a/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-3.Enable-recurrent-materialization-run-batch-inference.yml b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-3.Enable-recurrent-materialization-run-batch-inference.yml new file mode 100644 index 0000000000..0225faaaea --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-3.Enable-recurrent-materialization-run-batch-inference.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-notebooks-sdk_only-3.Enable-recurrent-materialization-run-batch-inference +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "48 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/notebooks/sdk_only/** + - .github/workflows/sdk-featurestore_sample-notebooks-sdk_only-3.Enable-recurrent-materialization-run-batch-inference.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/notebooks/sdk_only" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources - name: run featurestore_sample/notebooks/sdk_only/3.Enable-recurrent-materialization-run-batch-inference.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "3.Enable-recurrent-materialization-run-batch-inference.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 3.Enable-recurrent-materialization-run-batch-inference.ipynb 3.Enable-recurrent-materialization-run-batch-inference.output.ipynb + working-directory: sdk/python/featurestore_sample/notebooks/sdk_only + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 3.Enable-recurrent-materialization-run-batch-inference + path: sdk/python/featurestore_sample/notebooks/sdk_only diff --git a/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-4.Enable-online-store-run-inference.yml b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-4.Enable-online-store-run-inference.yml new file mode 100644 index 0000000000..c5d42ec4d9 --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-4.Enable-online-store-run-inference.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-notebooks-sdk_only-4.Enable-online-store-run-inference +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "30 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/notebooks/sdk_only/** + - .github/workflows/sdk-featurestore_sample-notebooks-sdk_only-4.Enable-online-store-run-inference.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/notebooks/sdk_only" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources - name: run featurestore_sample/notebooks/sdk_only/4.Enable-online-store-run-inference.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "4.Enable-online-store-run-inference.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 4.Enable-online-store-run-inference.ipynb 4.Enable-online-store-run-inference.output.ipynb + working-directory: sdk/python/featurestore_sample/notebooks/sdk_only + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 4.Enable-online-store-run-inference + path: sdk/python/featurestore_sample/notebooks/sdk_only diff --git a/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-5.Develop-feature-set-custom-source.yml b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-5.Develop-feature-set-custom-source.yml new file mode 100644 index 0000000000..3cf208e759 --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-5.Develop-feature-set-custom-source.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-notebooks-sdk_only-5.Develop-feature-set-custom-source +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "3 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/notebooks/sdk_only/** + - .github/workflows/sdk-featurestore_sample-notebooks-sdk_only-5.Develop-feature-set-custom-source.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/notebooks/sdk_only" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources - name: run featurestore_sample/notebooks/sdk_only/5.Develop-feature-set-custom-source.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "5.Develop-feature-set-custom-source.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 5.Develop-feature-set-custom-source.ipynb 5.Develop-feature-set-custom-source.output.ipynb + working-directory: sdk/python/featurestore_sample/notebooks/sdk_only + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 5.Develop-feature-set-custom-source + path: sdk/python/featurestore_sample/notebooks/sdk_only diff --git a/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-7.Develop-feature-set-domain-specific-language-dsl.yml b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-7.Develop-feature-set-domain-specific-language-dsl.yml new file mode 100644 index 0000000000..c27a66da12 --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-7.Develop-feature-set-domain-specific-language-dsl.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-notebooks-sdk_only-7.Develop-feature-set-domain-specific-language-dsl +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "20 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/notebooks/sdk_only/** + - .github/workflows/sdk-featurestore_sample-notebooks-sdk_only-7.Develop-feature-set-domain-specific-language-dsl.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/notebooks/sdk_only" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources - name: run featurestore_sample/notebooks/sdk_only/7.Develop-feature-set-domain-specific-language-dsl.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "7.Develop-feature-set-domain-specific-language-dsl.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 7.Develop-feature-set-domain-specific-language-dsl.ipynb 7.Develop-feature-set-domain-specific-language-dsl.output.ipynb + working-directory: sdk/python/featurestore_sample/notebooks/sdk_only + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 7.Develop-feature-set-domain-specific-language-dsl + path: sdk/python/featurestore_sample/notebooks/sdk_only diff --git a/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-ADB Example - Develop a feature set and register with managed feature store.yml b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-ADB Example - Develop a feature set and register with managed feature store.yml new file mode 100644 index 0000000000..ad940230ea --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-ADB Example - Develop a feature set and register with managed feature store.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-notebooks-sdk_only-ADB Example - Develop a feature set and register with managed feature store +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "1 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/notebooks/sdk_only/** + - .github/workflows/sdk-featurestore_sample-notebooks-sdk_only-ADB Example - Develop a feature set and register with managed feature store.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/notebooks/sdk_only" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources - name: run featurestore_sample/notebooks/sdk_only/ADB Example - Develop a feature set and register with managed feature store.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "ADB Example - Develop a feature set and register with managed feature store.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python ADB Example - Develop a feature set and register with managed feature store.ipynb ADB Example - Develop a feature set and register with managed feature store.output.ipynb + working-directory: sdk/python/featurestore_sample/notebooks/sdk_only + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: ADB Example - Develop a feature set and register with managed feature store + path: sdk/python/featurestore_sample/notebooks/sdk_only diff --git a/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-Synapse example - Develop feature set and register.yml b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-Synapse example - Develop feature set and register.yml new file mode 100644 index 0000000000..78059931fd --- /dev/null +++ b/.github/workflows/sdk-featurestore_sample-notebooks-sdk_only-Synapse example - Develop feature set and register.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-featurestore_sample-notebooks-sdk_only-Synapse example - Develop feature set and register +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "26 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/featurestore_sample/notebooks/sdk_only/** + - .github/workflows/sdk-featurestore_sample-notebooks-sdk_only-Synapse example - Develop feature set and register.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + - sdk/python/featurestore_sample/** +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample/notebooks/sdk_only" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: setup feature-store resources - name: run featurestore_sample/notebooks/sdk_only/Synapse example - Develop feature set and register.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "Synapse example - Develop feature set and register.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python Synapse example - Develop feature set and register.ipynb Synapse example - Develop feature set and register.output.ipynb + working-directory: sdk/python/featurestore_sample/notebooks/sdk_only + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: Synapse example - Develop feature set and register + path: sdk/python/featurestore_sample/notebooks/sdk_only diff --git a/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-ai21_azure_client.yml b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-ai21_azure_client.yml new file mode 100644 index 0000000000..8179e76de4 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-ai21_azure_client.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-ai21-labs-jamba-1-5-ai21_azure_client +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "13 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/ai21-labs/jamba-1-5/** + - .github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-ai21_azure_client.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/ai21-labs/jamba-1-5" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/ai21-labs/jamba-1-5/ai21_azure_client.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "ai21_azure_client.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python ai21_azure_client.ipynb ai21_azure_client.output.ipynb + working-directory: sdk/python/foundation-models/ai21-labs/jamba-1-5 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: ai21_azure_client + path: sdk/python/foundation-models/ai21-labs/jamba-1-5 diff --git a/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-langchain.yml b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-langchain.yml new file mode 100644 index 0000000000..b808541adf --- /dev/null +++ b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-ai21-labs-jamba-1-5-langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "41 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/ai21-labs/jamba-1-5/** + - .github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/ai21-labs/jamba-1-5" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/ai21-labs/jamba-1-5/langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python langchain.ipynb langchain.output.ipynb + working-directory: sdk/python/foundation-models/ai21-labs/jamba-1-5 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: langchain + path: sdk/python/foundation-models/ai21-labs/jamba-1-5 diff --git a/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-litellm.yml b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-litellm.yml new file mode 100644 index 0000000000..c2de9f8992 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-litellm.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-ai21-labs-jamba-1-5-litellm +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "13 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/ai21-labs/jamba-1-5/** + - .github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-litellm.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/ai21-labs/jamba-1-5" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/ai21-labs/jamba-1-5/litellm.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "litellm.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python litellm.ipynb litellm.output.ipynb + working-directory: sdk/python/foundation-models/ai21-labs/jamba-1-5 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: litellm + path: sdk/python/foundation-models/ai21-labs/jamba-1-5 diff --git a/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-openaisdk.yml b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-openaisdk.yml new file mode 100644 index 0000000000..a38ec66e71 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-openaisdk.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-ai21-labs-jamba-1-5-openaisdk +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/ai21-labs/jamba-1-5/** + - .github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-openaisdk.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/ai21-labs/jamba-1-5" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/ai21-labs/jamba-1-5/openaisdk.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openaisdk.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python openaisdk.ipynb openaisdk.output.ipynb + working-directory: sdk/python/foundation-models/ai21-labs/jamba-1-5 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: openaisdk + path: sdk/python/foundation-models/ai21-labs/jamba-1-5 diff --git a/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-webrequests.yml b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-webrequests.yml new file mode 100644 index 0000000000..7f5f963743 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-webrequests.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-ai21-labs-jamba-1-5-webrequests +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "5 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/ai21-labs/jamba-1-5/** + - .github/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-webrequests.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/ai21-labs/jamba-1-5" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/ai21-labs/jamba-1-5/webrequests.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "webrequests.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python webrequests.ipynb webrequests.output.ipynb + working-directory: sdk/python/foundation-models/ai21-labs/jamba-1-5 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: webrequests + path: sdk/python/foundation-models/ai21-labs/jamba-1-5 diff --git a/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-ai21_client.yml b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-ai21_client.yml new file mode 100644 index 0000000000..ec48571806 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-ai21_client.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-ai21-labs-jamba-instruct-ai21_client +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "13 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/ai21-labs/jamba-instruct/** + - .github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-ai21_client.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/ai21-labs/jamba-instruct" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/ai21-labs/jamba-instruct/ai21_client.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "ai21_client.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python ai21_client.ipynb ai21_client.output.ipynb + working-directory: sdk/python/foundation-models/ai21-labs/jamba-instruct + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: ai21_client + path: sdk/python/foundation-models/ai21-labs/jamba-instruct diff --git a/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-langchain.yml b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-langchain.yml new file mode 100644 index 0000000000..39d9968c2e --- /dev/null +++ b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-ai21-labs-jamba-instruct-langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "41 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/ai21-labs/jamba-instruct/** + - .github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/ai21-labs/jamba-instruct" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/ai21-labs/jamba-instruct/langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python langchain.ipynb langchain.output.ipynb + working-directory: sdk/python/foundation-models/ai21-labs/jamba-instruct + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: langchain + path: sdk/python/foundation-models/ai21-labs/jamba-instruct diff --git a/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-litellm.yml b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-litellm.yml new file mode 100644 index 0000000000..04447499fa --- /dev/null +++ b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-litellm.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-ai21-labs-jamba-instruct-litellm +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "13 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/ai21-labs/jamba-instruct/** + - .github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-litellm.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/ai21-labs/jamba-instruct" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/ai21-labs/jamba-instruct/litellm.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "litellm.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python litellm.ipynb litellm.output.ipynb + working-directory: sdk/python/foundation-models/ai21-labs/jamba-instruct + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: litellm + path: sdk/python/foundation-models/ai21-labs/jamba-instruct diff --git a/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-openaisdk.yml b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-openaisdk.yml new file mode 100644 index 0000000000..d0a2c523b0 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-openaisdk.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-ai21-labs-jamba-instruct-openaisdk +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/ai21-labs/jamba-instruct/** + - .github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-openaisdk.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/ai21-labs/jamba-instruct" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/ai21-labs/jamba-instruct/openaisdk.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openaisdk.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python openaisdk.ipynb openaisdk.output.ipynb + working-directory: sdk/python/foundation-models/ai21-labs/jamba-instruct + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: openaisdk + path: sdk/python/foundation-models/ai21-labs/jamba-instruct diff --git a/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-webrequests.yml b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-webrequests.yml new file mode 100644 index 0000000000..fdd075e7f1 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-webrequests.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-ai21-labs-jamba-instruct-webrequests +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "5 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/ai21-labs/jamba-instruct/** + - .github/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-webrequests.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/ai21-labs/jamba-instruct" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/ai21-labs/jamba-instruct/webrequests.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "webrequests.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python webrequests.ipynb webrequests.output.ipynb + working-directory: sdk/python/foundation-models/ai21-labs/jamba-instruct + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: webrequests + path: sdk/python/foundation-models/ai21-labs/jamba-instruct diff --git a/.github/workflows/sdk-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml b/.github/workflows/sdk-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml index 7e9f779a84..95419be8d4 100644 --- a/.github/workflows/sdk-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml +++ b/.github/workflows/sdk-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-foundation-models-benchmarking-evaluating_claude_models.yml b/.github/workflows/sdk-foundation-models-benchmarking-evaluating_claude_models.yml new file mode 100644 index 0000000000..1aa5a2abb1 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-benchmarking-evaluating_claude_models.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-benchmarking-evaluating_claude_models +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "36 9/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/benchmarking/** + - .github/workflows/sdk-foundation-models-benchmarking-evaluating_claude_models.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/benchmarking" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/benchmarking/evaluating_claude_models.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "evaluating_claude_models.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python evaluating_claude_models.ipynb evaluating_claude_models.output.ipynb + working-directory: sdk/python/foundation-models/benchmarking + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: evaluating_claude_models + path: sdk/python/foundation-models/benchmarking diff --git a/.github/workflows/sdk-foundation-models-benchmarking-evaluating_oai_models.yml b/.github/workflows/sdk-foundation-models-benchmarking-evaluating_oai_models.yml new file mode 100644 index 0000000000..1785135f12 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-benchmarking-evaluating_oai_models.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-benchmarking-evaluating_oai_models +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "27 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/benchmarking/** + - .github/workflows/sdk-foundation-models-benchmarking-evaluating_oai_models.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/benchmarking" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/benchmarking/evaluating_oai_models.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "evaluating_oai_models.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python evaluating_oai_models.ipynb evaluating_oai_models.output.ipynb + working-directory: sdk/python/foundation-models/benchmarking + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: evaluating_oai_models + path: sdk/python/foundation-models/benchmarking diff --git a/.github/workflows/sdk-foundation-models-cohere-cohere-aisearch-langchain-rag.yml b/.github/workflows/sdk-foundation-models-cohere-cohere-aisearch-langchain-rag.yml new file mode 100644 index 0000000000..762033154e --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-cohere-aisearch-langchain-rag.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-cohere-aisearch-langchain-rag +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "27 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-cohere-aisearch-langchain-rag.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/cohere-aisearch-langchain-rag.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "cohere-aisearch-langchain-rag.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python cohere-aisearch-langchain-rag.ipynb cohere-aisearch-langchain-rag.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: cohere-aisearch-langchain-rag + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-cohere-cohere-aisearch-rag.yml b/.github/workflows/sdk-foundation-models-cohere-cohere-aisearch-rag.yml new file mode 100644 index 0000000000..b53cef6814 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-cohere-aisearch-rag.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-cohere-aisearch-rag +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "8 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-cohere-aisearch-rag.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/cohere-aisearch-rag.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "cohere-aisearch-rag.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python cohere-aisearch-rag.ipynb cohere-aisearch-rag.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: cohere-aisearch-rag + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-cohere-cohere-cmdR.yml b/.github/workflows/sdk-foundation-models-cohere-cohere-cmdR.yml new file mode 100644 index 0000000000..e70726b7c7 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-cohere-cmdR.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-cohere-cmdR +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "1 6/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-cohere-cmdR.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/cohere-cmdR.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "cohere-cmdR.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python cohere-cmdR.ipynb cohere-cmdR.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: cohere-cmdR + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-cohere-cohere-embed.yml b/.github/workflows/sdk-foundation-models-cohere-cohere-embed.yml new file mode 100644 index 0000000000..707be4232c --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-cohere-embed.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-cohere-embed +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "29 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-cohere-embed.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/cohere-embed.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "cohere-embed.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python cohere-embed.ipynb cohere-embed.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: cohere-embed + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-cohere-cohere_faiss_langchain_embed.yml b/.github/workflows/sdk-foundation-models-cohere-cohere_faiss_langchain_embed.yml new file mode 100644 index 0000000000..7dbeb03666 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-cohere_faiss_langchain_embed.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-cohere_faiss_langchain_embed +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "49 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-cohere_faiss_langchain_embed.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/cohere_faiss_langchain_embed.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "cohere_faiss_langchain_embed.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python cohere_faiss_langchain_embed.ipynb cohere_faiss_langchain_embed.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: cohere_faiss_langchain_embed + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-cohere-command_faiss_langchain.yml b/.github/workflows/sdk-foundation-models-cohere-command_faiss_langchain.yml new file mode 100644 index 0000000000..41bcee3ea7 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-command_faiss_langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-command_faiss_langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "22 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-command_faiss_langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/command_faiss_langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "command_faiss_langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python command_faiss_langchain.ipynb command_faiss_langchain.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: command_faiss_langchain + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-cohere-command_tools-langchain.yml b/.github/workflows/sdk-foundation-models-cohere-command_tools-langchain.yml new file mode 100644 index 0000000000..024ef247f6 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-command_tools-langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-command_tools-langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "42 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-command_tools-langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/command_tools-langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "command_tools-langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python command_tools-langchain.ipynb command_tools-langchain.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: command_tools-langchain + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-cohere-langchain.yml b/.github/workflows/sdk-foundation-models-cohere-langchain.yml new file mode 100644 index 0000000000..ee07294a69 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "41 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python langchain.ipynb langchain.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: langchain + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-cohere-litellm.yml b/.github/workflows/sdk-foundation-models-cohere-litellm.yml new file mode 100644 index 0000000000..5af713fa63 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-litellm.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-litellm +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "13 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-litellm.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/litellm.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "litellm.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python litellm.ipynb litellm.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: litellm + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-cohere-openaisdk.yml b/.github/workflows/sdk-foundation-models-cohere-openaisdk.yml new file mode 100644 index 0000000000..edd199ad6e --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-openaisdk.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-openaisdk +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-openaisdk.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/openaisdk.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openaisdk.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python openaisdk.ipynb openaisdk.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: openaisdk + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-cohere-rerank-3-langchain.yml b/.github/workflows/sdk-foundation-models-cohere-rerank-3-langchain.yml new file mode 100644 index 0000000000..c6de6d6b4c --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-rerank-3-langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-rerank-3-langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "25 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-rerank-3-langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/rerank-3-langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "rerank-3-langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python rerank-3-langchain.ipynb rerank-3-langchain.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: rerank-3-langchain + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-cohere-rerank-cohere-client.yml b/.github/workflows/sdk-foundation-models-cohere-rerank-cohere-client.yml new file mode 100644 index 0000000000..dfedf05dc8 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-rerank-cohere-client.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-rerank-cohere-client +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "29 9/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-rerank-cohere-client.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/rerank-cohere-client.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "rerank-cohere-client.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python rerank-cohere-client.ipynb rerank-cohere-client.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: rerank-cohere-client + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-cohere-rerank-webrequests.yml b/.github/workflows/sdk-foundation-models-cohere-rerank-webrequests.yml new file mode 100644 index 0000000000..89df8c8e77 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-rerank-webrequests.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-rerank-webrequests +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "2 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-rerank-webrequests.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/rerank-webrequests.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "rerank-webrequests.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python rerank-webrequests.ipynb rerank-webrequests.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: rerank-webrequests + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-cohere-webrequests.yml b/.github/workflows/sdk-foundation-models-cohere-webrequests.yml new file mode 100644 index 0000000000..154d4dd5fb --- /dev/null +++ b/.github/workflows/sdk-foundation-models-cohere-webrequests.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-cohere-webrequests +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "5 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/cohere/** + - .github/workflows/sdk-foundation-models-cohere-webrequests.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/cohere" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/cohere/webrequests.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "webrequests.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python webrequests.ipynb webrequests.output.ipynb + working-directory: sdk/python/foundation-models/cohere + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: webrequests + path: sdk/python/foundation-models/cohere diff --git a/.github/workflows/sdk-foundation-models-healthcare-ai-cxrreportgen-cxr-deploy.yml b/.github/workflows/sdk-foundation-models-healthcare-ai-cxrreportgen-cxr-deploy.yml new file mode 100644 index 0000000000..9289abee0d --- /dev/null +++ b/.github/workflows/sdk-foundation-models-healthcare-ai-cxrreportgen-cxr-deploy.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-healthcare-ai-cxrreportgen-cxr-deploy +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "56 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/healthcare-ai/cxrreportgen/** + - .github/workflows/sdk-foundation-models-healthcare-ai-cxrreportgen-cxr-deploy.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/healthcare-ai/cxrreportgen" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/healthcare-ai/cxrreportgen/cxr-deploy.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "cxr-deploy.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python cxr-deploy.ipynb cxr-deploy.output.ipynb + working-directory: sdk/python/foundation-models/healthcare-ai/cxrreportgen + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: cxr-deploy + path: sdk/python/foundation-models/healthcare-ai/cxrreportgen diff --git a/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-adapter-training.yml b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-adapter-training.yml new file mode 100644 index 0000000000..04e633071a --- /dev/null +++ b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-adapter-training.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-healthcare-ai-medimageinsight-adapter-training +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "31 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/healthcare-ai/medimageinsight/** + - .github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-adapter-training.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/healthcare-ai/medimageinsight" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/healthcare-ai/medimageinsight/adapter-training.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "adapter-training.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python adapter-training.ipynb adapter-training.output.ipynb + working-directory: sdk/python/foundation-models/healthcare-ai/medimageinsight + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: adapter-training + path: sdk/python/foundation-models/healthcare-ai/medimageinsight diff --git a/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-advanced-call-example.yml b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-advanced-call-example.yml new file mode 100644 index 0000000000..e013ea1e14 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-advanced-call-example.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-healthcare-ai-medimageinsight-advanced-call-example +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "26 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/healthcare-ai/medimageinsight/** + - .github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-advanced-call-example.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/healthcare-ai/medimageinsight" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/healthcare-ai/medimageinsight/advanced-call-example.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "advanced-call-example.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python advanced-call-example.ipynb advanced-call-example.output.ipynb + working-directory: sdk/python/foundation-models/healthcare-ai/medimageinsight + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: advanced-call-example + path: sdk/python/foundation-models/healthcare-ai/medimageinsight diff --git a/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-exam-parameter-demo-exam-parameter-detection.yml b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-exam-parameter-demo-exam-parameter-detection.yml new file mode 100644 index 0000000000..a95f36a8ee --- /dev/null +++ b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-exam-parameter-demo-exam-parameter-detection.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-healthcare-ai-medimageinsight-exam-parameter-demo-exam-parameter-detection +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "18 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/healthcare-ai/medimageinsight/exam-parameter-demo/** + - .github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-exam-parameter-demo-exam-parameter-detection.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/healthcare-ai/medimageinsight/exam-parameter-demo" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/healthcare-ai/medimageinsight/exam-parameter-demo/exam-parameter-detection.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "exam-parameter-detection.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python exam-parameter-detection.ipynb exam-parameter-detection.output.ipynb + working-directory: sdk/python/foundation-models/healthcare-ai/medimageinsight/exam-parameter-demo + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: exam-parameter-detection + path: sdk/python/foundation-models/healthcare-ai/medimageinsight/exam-parameter-demo diff --git a/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-mi2-deploy.yml b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-mi2-deploy.yml new file mode 100644 index 0000000000..86ea260325 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-mi2-deploy.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-healthcare-ai-medimageinsight-mi2-deploy +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "58 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/healthcare-ai/medimageinsight/** + - .github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-mi2-deploy.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/healthcare-ai/medimageinsight" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/healthcare-ai/medimageinsight/mi2-deploy.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "mi2-deploy.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python mi2-deploy.ipynb mi2-deploy.output.ipynb + working-directory: sdk/python/foundation-models/healthcare-ai/medimageinsight + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: mi2-deploy + path: sdk/python/foundation-models/healthcare-ai/medimageinsight diff --git a/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-zero-shot-classification.yml b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-zero-shot-classification.yml new file mode 100644 index 0000000000..ebf98cf30e --- /dev/null +++ b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-zero-shot-classification.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-healthcare-ai-medimageinsight-zero-shot-classification +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "59 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/healthcare-ai/medimageinsight/** + - .github/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-zero-shot-classification.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/healthcare-ai/medimageinsight" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/healthcare-ai/medimageinsight/zero-shot-classification.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "zero-shot-classification.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python zero-shot-classification.ipynb zero-shot-classification.output.ipynb + working-directory: sdk/python/foundation-models/healthcare-ai/medimageinsight + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: zero-shot-classification + path: sdk/python/foundation-models/healthcare-ai/medimageinsight diff --git a/.github/workflows/sdk-foundation-models-healthcare-ai-medimageparse-mip-deploy.yml b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageparse-mip-deploy.yml new file mode 100644 index 0000000000..540e00dc56 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageparse-mip-deploy.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-healthcare-ai-medimageparse-mip-deploy +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "37 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/healthcare-ai/medimageparse/** + - .github/workflows/sdk-foundation-models-healthcare-ai-medimageparse-mip-deploy.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/healthcare-ai/medimageparse" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/healthcare-ai/medimageparse/mip-deploy.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "mip-deploy.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python mip-deploy.ipynb mip-deploy.output.ipynb + working-directory: sdk/python/foundation-models/healthcare-ai/medimageparse + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: mip-deploy + path: sdk/python/foundation-models/healthcare-ai/medimageparse diff --git a/.github/workflows/sdk-foundation-models-healthcare-ai-medimageparse-sematic-segmentation-demo-medimageparse_segmentation_demo.yml b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageparse-sematic-segmentation-demo-medimageparse_segmentation_demo.yml new file mode 100644 index 0000000000..733727f8d1 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-healthcare-ai-medimageparse-sematic-segmentation-demo-medimageparse_segmentation_demo.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-healthcare-ai-medimageparse-sematic-segmentation-demo-medimageparse_segmentation_demo +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "25 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/healthcare-ai/medimageparse/sematic-segmentation-demo/** + - .github/workflows/sdk-foundation-models-healthcare-ai-medimageparse-sematic-segmentation-demo-medimageparse_segmentation_demo.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/healthcare-ai/medimageparse/sematic-segmentation-demo" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/healthcare-ai/medimageparse/sematic-segmentation-demo/medimageparse_segmentation_demo.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "medimageparse_segmentation_demo.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python medimageparse_segmentation_demo.ipynb medimageparse_segmentation_demo.output.ipynb + working-directory: sdk/python/foundation-models/healthcare-ai/medimageparse/sematic-segmentation-demo + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: medimageparse_segmentation_demo + path: sdk/python/foundation-models/healthcare-ai/medimageparse/sematic-segmentation-demo diff --git a/.github/workflows/sdk-foundation-models-huggingface-inference-text-generation-streaming-text-generation-streaming-online-endpoint.yml b/.github/workflows/sdk-foundation-models-huggingface-inference-text-generation-streaming-text-generation-streaming-online-endpoint.yml new file mode 100644 index 0000000000..08dc2ec7d6 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-huggingface-inference-text-generation-streaming-text-generation-streaming-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-huggingface-inference-text-generation-streaming-text-generation-streaming-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "37 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/huggingface/inference/text-generation-streaming/** + - .github/workflows/sdk-foundation-models-huggingface-inference-text-generation-streaming-text-generation-streaming-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/huggingface/inference/text-generation-streaming" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/huggingface/inference/text-generation-streaming/text-generation-streaming-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text-generation-streaming-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text-generation-streaming-online-endpoint.ipynb text-generation-streaming-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/huggingface/inference/text-generation-streaming + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text-generation-streaming-online-endpoint + path: sdk/python/foundation-models/huggingface/inference/text-generation-streaming diff --git a/.github/workflows/sdk-foundation-models-jais-litellm.yml b/.github/workflows/sdk-foundation-models-jais-litellm.yml new file mode 100644 index 0000000000..43a9b3576d --- /dev/null +++ b/.github/workflows/sdk-foundation-models-jais-litellm.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-jais-litellm +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "13 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/jais/** + - .github/workflows/sdk-foundation-models-jais-litellm.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/jais" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/jais/litellm.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "litellm.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python litellm.ipynb litellm.output.ipynb + working-directory: sdk/python/foundation-models/jais + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: litellm + path: sdk/python/foundation-models/jais diff --git a/.github/workflows/sdk-foundation-models-jais-openaisdk.yml b/.github/workflows/sdk-foundation-models-jais-openaisdk.yml new file mode 100644 index 0000000000..3b7a16bc27 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-jais-openaisdk.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-jais-openaisdk +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/jais/** + - .github/workflows/sdk-foundation-models-jais-openaisdk.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/jais" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/jais/openaisdk.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openaisdk.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python openaisdk.ipynb openaisdk.output.ipynb + working-directory: sdk/python/foundation-models/jais + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: openaisdk + path: sdk/python/foundation-models/jais diff --git a/.github/workflows/sdk-foundation-models-jais-webrequests.yml b/.github/workflows/sdk-foundation-models-jais-webrequests.yml new file mode 100644 index 0000000000..d3f44d1a9b --- /dev/null +++ b/.github/workflows/sdk-foundation-models-jais-webrequests.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-jais-webrequests +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "5 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/jais/** + - .github/workflows/sdk-foundation-models-jais-webrequests.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/jais" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/jais/webrequests.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "webrequests.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python webrequests.ipynb webrequests.output.ipynb + working-directory: sdk/python/foundation-models/jais + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: webrequests + path: sdk/python/foundation-models/jais diff --git a/.github/workflows/sdk-foundation-models-llama2-langchain.yml b/.github/workflows/sdk-foundation-models-llama2-langchain.yml new file mode 100644 index 0000000000..b6a341867d --- /dev/null +++ b/.github/workflows/sdk-foundation-models-llama2-langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-llama2-langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "41 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/llama2/** + - .github/workflows/sdk-foundation-models-llama2-langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/llama2" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/llama2/langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python langchain.ipynb langchain.output.ipynb + working-directory: sdk/python/foundation-models/llama2 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: langchain + path: sdk/python/foundation-models/llama2 diff --git a/.github/workflows/sdk-foundation-models-llama2-litellm.yml b/.github/workflows/sdk-foundation-models-llama2-litellm.yml new file mode 100644 index 0000000000..ed91a9bca7 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-llama2-litellm.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-llama2-litellm +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "13 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/llama2/** + - .github/workflows/sdk-foundation-models-llama2-litellm.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/llama2" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/llama2/litellm.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "litellm.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python litellm.ipynb litellm.output.ipynb + working-directory: sdk/python/foundation-models/llama2 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: litellm + path: sdk/python/foundation-models/llama2 diff --git a/.github/workflows/sdk-foundation-models-llama2-openaisdk.yml b/.github/workflows/sdk-foundation-models-llama2-openaisdk.yml new file mode 100644 index 0000000000..1edec73633 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-llama2-openaisdk.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-llama2-openaisdk +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/llama2/** + - .github/workflows/sdk-foundation-models-llama2-openaisdk.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/llama2" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/llama2/openaisdk.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openaisdk.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python openaisdk.ipynb openaisdk.output.ipynb + working-directory: sdk/python/foundation-models/llama2 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: openaisdk + path: sdk/python/foundation-models/llama2 diff --git a/.github/workflows/sdk-foundation-models-llama2-webrequests.yml b/.github/workflows/sdk-foundation-models-llama2-webrequests.yml new file mode 100644 index 0000000000..378a21453c --- /dev/null +++ b/.github/workflows/sdk-foundation-models-llama2-webrequests.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-llama2-webrequests +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "5 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/llama2/** + - .github/workflows/sdk-foundation-models-llama2-webrequests.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/llama2" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/llama2/webrequests.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "webrequests.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python webrequests.ipynb webrequests.output.ipynb + working-directory: sdk/python/foundation-models/llama2 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: webrequests + path: sdk/python/foundation-models/llama2 diff --git a/.github/workflows/sdk-foundation-models-meta-llama-3.1-langchain.yml b/.github/workflows/sdk-foundation-models-meta-llama-3.1-langchain.yml new file mode 100644 index 0000000000..1232ee225d --- /dev/null +++ b/.github/workflows/sdk-foundation-models-meta-llama-3.1-langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-meta-llama-3.1-langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "41 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/meta-llama-3.1/** + - .github/workflows/sdk-foundation-models-meta-llama-3.1-langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/meta-llama-3.1" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/meta-llama-3.1/langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python langchain.ipynb langchain.output.ipynb + working-directory: sdk/python/foundation-models/meta-llama-3.1 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: langchain + path: sdk/python/foundation-models/meta-llama-3.1 diff --git a/.github/workflows/sdk-foundation-models-meta-llama-3.1-litellm.yml b/.github/workflows/sdk-foundation-models-meta-llama-3.1-litellm.yml new file mode 100644 index 0000000000..0cd1fd9755 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-meta-llama-3.1-litellm.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-meta-llama-3.1-litellm +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "13 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/meta-llama-3.1/** + - .github/workflows/sdk-foundation-models-meta-llama-3.1-litellm.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/meta-llama-3.1" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/meta-llama-3.1/litellm.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "litellm.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python litellm.ipynb litellm.output.ipynb + working-directory: sdk/python/foundation-models/meta-llama-3.1 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: litellm + path: sdk/python/foundation-models/meta-llama-3.1 diff --git a/.github/workflows/sdk-foundation-models-meta-llama-3.1-openaisdk.yml b/.github/workflows/sdk-foundation-models-meta-llama-3.1-openaisdk.yml new file mode 100644 index 0000000000..509af9ca8c --- /dev/null +++ b/.github/workflows/sdk-foundation-models-meta-llama-3.1-openaisdk.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-meta-llama-3.1-openaisdk +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/meta-llama-3.1/** + - .github/workflows/sdk-foundation-models-meta-llama-3.1-openaisdk.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/meta-llama-3.1" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/meta-llama-3.1/openaisdk.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openaisdk.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python openaisdk.ipynb openaisdk.output.ipynb + working-directory: sdk/python/foundation-models/meta-llama-3.1 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: openaisdk + path: sdk/python/foundation-models/meta-llama-3.1 diff --git a/.github/workflows/sdk-foundation-models-meta-llama-3.1-webrequests.yml b/.github/workflows/sdk-foundation-models-meta-llama-3.1-webrequests.yml new file mode 100644 index 0000000000..46481b1092 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-meta-llama-3.1-webrequests.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-meta-llama-3.1-webrequests +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "5 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/meta-llama-3.1/** + - .github/workflows/sdk-foundation-models-meta-llama-3.1-webrequests.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/meta-llama-3.1" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/meta-llama-3.1/webrequests.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "webrequests.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python webrequests.ipynb webrequests.output.ipynb + working-directory: sdk/python/foundation-models/meta-llama-3.1 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: webrequests + path: sdk/python/foundation-models/meta-llama-3.1 diff --git a/.github/workflows/sdk-foundation-models-meta-llama3-langchain.yml b/.github/workflows/sdk-foundation-models-meta-llama3-langchain.yml new file mode 100644 index 0000000000..e826b312fa --- /dev/null +++ b/.github/workflows/sdk-foundation-models-meta-llama3-langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-meta-llama3-langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "41 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/meta-llama3/** + - .github/workflows/sdk-foundation-models-meta-llama3-langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/meta-llama3" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/meta-llama3/langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python langchain.ipynb langchain.output.ipynb + working-directory: sdk/python/foundation-models/meta-llama3 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: langchain + path: sdk/python/foundation-models/meta-llama3 diff --git a/.github/workflows/sdk-foundation-models-meta-llama3-litellm.yml b/.github/workflows/sdk-foundation-models-meta-llama3-litellm.yml new file mode 100644 index 0000000000..54729ee446 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-meta-llama3-litellm.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-meta-llama3-litellm +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "13 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/meta-llama3/** + - .github/workflows/sdk-foundation-models-meta-llama3-litellm.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/meta-llama3" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/meta-llama3/litellm.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "litellm.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python litellm.ipynb litellm.output.ipynb + working-directory: sdk/python/foundation-models/meta-llama3 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: litellm + path: sdk/python/foundation-models/meta-llama3 diff --git a/.github/workflows/sdk-foundation-models-meta-llama3-openaisdk.yml b/.github/workflows/sdk-foundation-models-meta-llama3-openaisdk.yml new file mode 100644 index 0000000000..c2dbb8ba10 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-meta-llama3-openaisdk.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-meta-llama3-openaisdk +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/meta-llama3/** + - .github/workflows/sdk-foundation-models-meta-llama3-openaisdk.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/meta-llama3" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/meta-llama3/openaisdk.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openaisdk.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python openaisdk.ipynb openaisdk.output.ipynb + working-directory: sdk/python/foundation-models/meta-llama3 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: openaisdk + path: sdk/python/foundation-models/meta-llama3 diff --git a/.github/workflows/sdk-foundation-models-meta-llama3-webrequests.yml b/.github/workflows/sdk-foundation-models-meta-llama3-webrequests.yml new file mode 100644 index 0000000000..2487921750 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-meta-llama3-webrequests.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-meta-llama3-webrequests +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "5 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/meta-llama3/** + - .github/workflows/sdk-foundation-models-meta-llama3-webrequests.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/meta-llama3" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/meta-llama3/webrequests.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "webrequests.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python webrequests.ipynb webrequests.output.ipynb + working-directory: sdk/python/foundation-models/meta-llama3 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: webrequests + path: sdk/python/foundation-models/meta-llama3 diff --git a/.github/workflows/sdk-foundation-models-mistral-langchain.yml b/.github/workflows/sdk-foundation-models-mistral-langchain.yml new file mode 100644 index 0000000000..42fdae664a --- /dev/null +++ b/.github/workflows/sdk-foundation-models-mistral-langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-mistral-langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "41 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/mistral/** + - .github/workflows/sdk-foundation-models-mistral-langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/mistral" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/mistral/langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python langchain.ipynb langchain.output.ipynb + working-directory: sdk/python/foundation-models/mistral + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: langchain + path: sdk/python/foundation-models/mistral diff --git a/.github/workflows/sdk-foundation-models-mistral-litellm.yml b/.github/workflows/sdk-foundation-models-mistral-litellm.yml new file mode 100644 index 0000000000..e066ae37b0 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-mistral-litellm.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-mistral-litellm +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "13 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/mistral/** + - .github/workflows/sdk-foundation-models-mistral-litellm.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/mistral" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/mistral/litellm.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "litellm.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python litellm.ipynb litellm.output.ipynb + working-directory: sdk/python/foundation-models/mistral + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: litellm + path: sdk/python/foundation-models/mistral diff --git a/.github/workflows/sdk-foundation-models-mistral-mistralai.yml b/.github/workflows/sdk-foundation-models-mistral-mistralai.yml new file mode 100644 index 0000000000..b9f412380d --- /dev/null +++ b/.github/workflows/sdk-foundation-models-mistral-mistralai.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-mistral-mistralai +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "21 6/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/mistral/** + - .github/workflows/sdk-foundation-models-mistral-mistralai.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/mistral" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/mistral/mistralai.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "mistralai.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python mistralai.ipynb mistralai.output.ipynb + working-directory: sdk/python/foundation-models/mistral + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: mistralai + path: sdk/python/foundation-models/mistral diff --git a/.github/workflows/sdk-foundation-models-mistral-openaisdk.yml b/.github/workflows/sdk-foundation-models-mistral-openaisdk.yml new file mode 100644 index 0000000000..af3ad5d3b0 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-mistral-openaisdk.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-mistral-openaisdk +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/mistral/** + - .github/workflows/sdk-foundation-models-mistral-openaisdk.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/mistral" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/mistral/openaisdk.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openaisdk.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python openaisdk.ipynb openaisdk.output.ipynb + working-directory: sdk/python/foundation-models/mistral + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: openaisdk + path: sdk/python/foundation-models/mistral diff --git a/.github/workflows/sdk-foundation-models-mistral-webrequests.yml b/.github/workflows/sdk-foundation-models-mistral-webrequests.yml new file mode 100644 index 0000000000..347d6bb364 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-mistral-webrequests.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-mistral-webrequests +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "5 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/mistral/** + - .github/workflows/sdk-foundation-models-mistral-webrequests.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/mistral" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/mistral/webrequests.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "webrequests.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python webrequests.ipynb webrequests.output.ipynb + working-directory: sdk/python/foundation-models/mistral + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: webrequests + path: sdk/python/foundation-models/mistral diff --git a/.github/workflows/sdk-foundation-models-nixtla-01_quickstart_forecast.yml b/.github/workflows/sdk-foundation-models-nixtla-01_quickstart_forecast.yml new file mode 100644 index 0000000000..5386607532 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nixtla-01_quickstart_forecast.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nixtla-01_quickstart_forecast +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "24 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nixtla/** + - .github/workflows/sdk-foundation-models-nixtla-01_quickstart_forecast.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nixtla" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nixtla/01_quickstart_forecast.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "01_quickstart_forecast.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 01_quickstart_forecast.ipynb 01_quickstart_forecast.output.ipynb + working-directory: sdk/python/foundation-models/nixtla + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 01_quickstart_forecast + path: sdk/python/foundation-models/nixtla diff --git a/.github/workflows/sdk-foundation-models-nixtla-02_finetuning.yml b/.github/workflows/sdk-foundation-models-nixtla-02_finetuning.yml new file mode 100644 index 0000000000..ce8508c56b --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nixtla-02_finetuning.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nixtla-02_finetuning +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "47 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nixtla/** + - .github/workflows/sdk-foundation-models-nixtla-02_finetuning.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nixtla" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nixtla/02_finetuning.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "02_finetuning.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 02_finetuning.ipynb 02_finetuning.output.ipynb + working-directory: sdk/python/foundation-models/nixtla + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 02_finetuning + path: sdk/python/foundation-models/nixtla diff --git a/.github/workflows/sdk-foundation-models-nixtla-03_anomaly_detection.yml b/.github/workflows/sdk-foundation-models-nixtla-03_anomaly_detection.yml new file mode 100644 index 0000000000..509ded0aa5 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nixtla-03_anomaly_detection.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nixtla-03_anomaly_detection +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "59 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nixtla/** + - .github/workflows/sdk-foundation-models-nixtla-03_anomaly_detection.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nixtla" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nixtla/03_anomaly_detection.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "03_anomaly_detection.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 03_anomaly_detection.ipynb 03_anomaly_detection.output.ipynb + working-directory: sdk/python/foundation-models/nixtla + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 03_anomaly_detection + path: sdk/python/foundation-models/nixtla diff --git a/.github/workflows/sdk-foundation-models-nixtla-04_exogenous_variables.yml b/.github/workflows/sdk-foundation-models-nixtla-04_exogenous_variables.yml new file mode 100644 index 0000000000..f963c48b20 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nixtla-04_exogenous_variables.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nixtla-04_exogenous_variables +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "13 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nixtla/** + - .github/workflows/sdk-foundation-models-nixtla-04_exogenous_variables.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nixtla" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nixtla/04_exogenous_variables.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "04_exogenous_variables.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 04_exogenous_variables.ipynb 04_exogenous_variables.output.ipynb + working-directory: sdk/python/foundation-models/nixtla + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 04_exogenous_variables + path: sdk/python/foundation-models/nixtla diff --git a/.github/workflows/sdk-foundation-models-nixtla-05_demand_forecasting.yml b/.github/workflows/sdk-foundation-models-nixtla-05_demand_forecasting.yml new file mode 100644 index 0000000000..1690072c2e --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nixtla-05_demand_forecasting.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nixtla-05_demand_forecasting +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "42 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nixtla/** + - .github/workflows/sdk-foundation-models-nixtla-05_demand_forecasting.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nixtla" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nixtla/05_demand_forecasting.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "05_demand_forecasting.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python 05_demand_forecasting.ipynb 05_demand_forecasting.output.ipynb + working-directory: sdk/python/foundation-models/nixtla + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: 05_demand_forecasting + path: sdk/python/foundation-models/nixtla diff --git a/.github/workflows/sdk-foundation-models-nvidia-nemo-deploy_infer.yml b/.github/workflows/sdk-foundation-models-nvidia-nemo-deploy_infer.yml new file mode 100644 index 0000000000..ff4c3f9342 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nvidia-nemo-deploy_infer.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nvidia-nemo-deploy_infer +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "9 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nvidia-nemo/** + - .github/workflows/sdk-foundation-models-nvidia-nemo-deploy_infer.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nvidia-nemo" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nvidia-nemo/deploy_infer.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "deploy_infer.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python deploy_infer.ipynb deploy_infer.output.ipynb + working-directory: sdk/python/foundation-models/nvidia-nemo + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: deploy_infer + path: sdk/python/foundation-models/nvidia-nemo diff --git a/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nemo_ptuning_summarization_with_text_gen.yml b/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nemo_ptuning_summarization_with_text_gen.yml new file mode 100644 index 0000000000..c580aad752 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nemo_ptuning_summarization_with_text_gen.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nvidia-nemo-evaluate-nemo_ptuning_summarization_with_text_gen +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "29 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nvidia-nemo/evaluate/** + - .github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nemo_ptuning_summarization_with_text_gen.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nvidia-nemo/evaluate" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nvidia-nemo/evaluate/nemo_ptuning_summarization_with_text_gen.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "nemo_ptuning_summarization_with_text_gen.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python nemo_ptuning_summarization_with_text_gen.ipynb nemo_ptuning_summarization_with_text_gen.output.ipynb + working-directory: sdk/python/foundation-models/nvidia-nemo/evaluate + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: nemo_ptuning_summarization_with_text_gen + path: sdk/python/foundation-models/nvidia-nemo/evaluate diff --git a/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-classification-evaluation.yml b/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-classification-evaluation.yml new file mode 100644 index 0000000000..2b6a343f97 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-classification-evaluation.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-classification-evaluation +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "22 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nvidia-nemo/evaluate/** + - .github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-classification-evaluation.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nvidia-nemo/evaluate" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nvidia-nemo/evaluate/nvidia-text-classification-evaluation.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "nvidia-text-classification-evaluation.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python nvidia-text-classification-evaluation.ipynb nvidia-text-classification-evaluation.output.ipynb + working-directory: sdk/python/foundation-models/nvidia-nemo/evaluate + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: nvidia-text-classification-evaluation + path: sdk/python/foundation-models/nvidia-nemo/evaluate diff --git a/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-gen-evaluation.yml b/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-gen-evaluation.yml new file mode 100644 index 0000000000..311f330a8b --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-gen-evaluation.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-gen-evaluation +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "27 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nvidia-nemo/evaluate/** + - .github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-gen-evaluation.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nvidia-nemo/evaluate" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nvidia-nemo/evaluate/nvidia-text-gen-evaluation.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "nvidia-text-gen-evaluation.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python nvidia-text-gen-evaluation.ipynb nvidia-text-gen-evaluation.output.ipynb + working-directory: sdk/python/foundation-models/nvidia-nemo/evaluate + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: nvidia-text-gen-evaluation + path: sdk/python/foundation-models/nvidia-nemo/evaluate diff --git a/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-qna-evaluation.yml b/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-qna-evaluation.yml new file mode 100644 index 0000000000..a8f070d6a1 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-qna-evaluation.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-qna-evaluation +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "5 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nvidia-nemo/evaluate/** + - .github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-qna-evaluation.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nvidia-nemo/evaluate" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nvidia-nemo/evaluate/nvidia-text-qna-evaluation.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "nvidia-text-qna-evaluation.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python nvidia-text-qna-evaluation.ipynb nvidia-text-qna-evaluation.output.ipynb + working-directory: sdk/python/foundation-models/nvidia-nemo/evaluate + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: nvidia-text-qna-evaluation + path: sdk/python/foundation-models/nvidia-nemo/evaluate diff --git a/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-translation-evaluation.yml b/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-translation-evaluation.yml new file mode 100644 index 0000000000..ce56b73561 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-translation-evaluation.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-translation-evaluation +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "45 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nvidia-nemo/evaluate/** + - .github/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-translation-evaluation.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nvidia-nemo/evaluate" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nvidia-nemo/evaluate/nvidia-text-translation-evaluation.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "nvidia-text-translation-evaluation.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python nvidia-text-translation-evaluation.ipynb nvidia-text-translation-evaluation.output.ipynb + working-directory: sdk/python/foundation-models/nvidia-nemo/evaluate + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: nvidia-text-translation-evaluation + path: sdk/python/foundation-models/nvidia-nemo/evaluate diff --git a/.github/workflows/sdk-foundation-models-nvidia-nemo-ptune_evaluate.yml b/.github/workflows/sdk-foundation-models-nvidia-nemo-ptune_evaluate.yml new file mode 100644 index 0000000000..eabdde723a --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nvidia-nemo-ptune_evaluate.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nvidia-nemo-ptune_evaluate +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "58 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nvidia-nemo/** + - .github/workflows/sdk-foundation-models-nvidia-nemo-ptune_evaluate.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nvidia-nemo" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nvidia-nemo/ptune_evaluate.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "ptune_evaluate.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python ptune_evaluate.ipynb ptune_evaluate.output.ipynb + working-directory: sdk/python/foundation-models/nvidia-nemo + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: ptune_evaluate + path: sdk/python/foundation-models/nvidia-nemo diff --git a/.github/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-aiinference.yml b/.github/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-aiinference.yml new file mode 100644 index 0000000000..72bd16f9ba --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-aiinference.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nvidia-nim-llama3-8b-aiinference +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "30 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nvidia-nim-llama3-8b/** + - .github/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-aiinference.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nvidia-nim-llama3-8b" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nvidia-nim-llama3-8b/aiinference.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "aiinference.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python aiinference.ipynb aiinference.output.ipynb + working-directory: sdk/python/foundation-models/nvidia-nim-llama3-8b + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: aiinference + path: sdk/python/foundation-models/nvidia-nim-llama3-8b diff --git a/.github/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-openaisdk.yml b/.github/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-openaisdk.yml new file mode 100644 index 0000000000..24f5586f93 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-openaisdk.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nvidia-nim-llama3-8b-openaisdk +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nvidia-nim-llama3-8b/** + - .github/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-openaisdk.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nvidia-nim-llama3-8b" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nvidia-nim-llama3-8b/openaisdk.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openaisdk.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python openaisdk.ipynb openaisdk.output.ipynb + working-directory: sdk/python/foundation-models/nvidia-nim-llama3-8b + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: openaisdk + path: sdk/python/foundation-models/nvidia-nim-llama3-8b diff --git a/.github/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-webrequests.yml b/.github/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-webrequests.yml new file mode 100644 index 0000000000..0b7b46dfde --- /dev/null +++ b/.github/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-webrequests.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-nvidia-nim-llama3-8b-webrequests +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "5 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/nvidia-nim-llama3-8b/** + - .github/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-webrequests.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/nvidia-nim-llama3-8b" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/nvidia-nim-llama3-8b/webrequests.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "webrequests.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python webrequests.ipynb webrequests.output.ipynb + working-directory: sdk/python/foundation-models/nvidia-nim-llama3-8b + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: webrequests + path: sdk/python/foundation-models/nvidia-nim-llama3-8b diff --git a/.github/workflows/sdk-foundation-models-phi-3-litellm.yml b/.github/workflows/sdk-foundation-models-phi-3-litellm.yml new file mode 100644 index 0000000000..b4f28f91bd --- /dev/null +++ b/.github/workflows/sdk-foundation-models-phi-3-litellm.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-phi-3-litellm +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "13 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/phi-3/** + - .github/workflows/sdk-foundation-models-phi-3-litellm.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/phi-3" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/phi-3/litellm.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "litellm.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python litellm.ipynb litellm.output.ipynb + working-directory: sdk/python/foundation-models/phi-3 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: litellm + path: sdk/python/foundation-models/phi-3 diff --git a/.github/workflows/sdk-foundation-models-phi-3-openaisdk.yml b/.github/workflows/sdk-foundation-models-phi-3-openaisdk.yml new file mode 100644 index 0000000000..09695ab4ca --- /dev/null +++ b/.github/workflows/sdk-foundation-models-phi-3-openaisdk.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-phi-3-openaisdk +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/phi-3/** + - .github/workflows/sdk-foundation-models-phi-3-openaisdk.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/phi-3" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/phi-3/openaisdk.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openaisdk.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python openaisdk.ipynb openaisdk.output.ipynb + working-directory: sdk/python/foundation-models/phi-3 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: openaisdk + path: sdk/python/foundation-models/phi-3 diff --git a/.github/workflows/sdk-foundation-models-phi-3-webrequests.yml b/.github/workflows/sdk-foundation-models-phi-3-webrequests.yml new file mode 100644 index 0000000000..3fc5ae7f51 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-phi-3-webrequests.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-phi-3-webrequests +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "5 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/phi-3/** + - .github/workflows/sdk-foundation-models-phi-3-webrequests.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/phi-3" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/phi-3/webrequests.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "webrequests.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python webrequests.ipynb webrequests.output.ipynb + working-directory: sdk/python/foundation-models/phi-3 + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: webrequests + path: sdk/python/foundation-models/phi-3 diff --git a/.github/workflows/sdk-foundation-models-stabilityai-Image_to_Image.yml b/.github/workflows/sdk-foundation-models-stabilityai-Image_to_Image.yml new file mode 100644 index 0000000000..1127758053 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-stabilityai-Image_to_Image.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-stabilityai-Image_to_Image +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "36 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/stabilityai/** + - .github/workflows/sdk-foundation-models-stabilityai-Image_to_Image.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/stabilityai" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/stabilityai/Image_to_Image.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "Image_to_Image.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python Image_to_Image.ipynb Image_to_Image.output.ipynb + working-directory: sdk/python/foundation-models/stabilityai + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: Image_to_Image + path: sdk/python/foundation-models/stabilityai diff --git a/.github/workflows/sdk-foundation-models-stabilityai-Text_to_Image_openai_library.yml b/.github/workflows/sdk-foundation-models-stabilityai-Text_to_Image_openai_library.yml new file mode 100644 index 0000000000..6aeeaa0f16 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-stabilityai-Text_to_Image_openai_library.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-stabilityai-Text_to_Image_openai_library +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "36 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/stabilityai/** + - .github/workflows/sdk-foundation-models-stabilityai-Text_to_Image_openai_library.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/stabilityai" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/stabilityai/Text_to_Image_openai_library.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "Text_to_Image_openai_library.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python Text_to_Image_openai_library.ipynb Text_to_Image_openai_library.output.ipynb + working-directory: sdk/python/foundation-models/stabilityai + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: Text_to_Image_openai_library + path: sdk/python/foundation-models/stabilityai diff --git a/.github/workflows/sdk-foundation-models-stabilityai-Text_to_Image_requests_library.yml b/.github/workflows/sdk-foundation-models-stabilityai-Text_to_Image_requests_library.yml new file mode 100644 index 0000000000..1f33120786 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-stabilityai-Text_to_Image_requests_library.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-stabilityai-Text_to_Image_requests_library +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "59 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/stabilityai/** + - .github/workflows/sdk-foundation-models-stabilityai-Text_to_Image_requests_library.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/stabilityai" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/stabilityai/Text_to_Image_requests_library.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "Text_to_Image_requests_library.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python Text_to_Image_requests_library.ipynb Text_to_Image_requests_library.output.ipynb + working-directory: sdk/python/foundation-models/stabilityai + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: Text_to_Image_requests_library + path: sdk/python/foundation-models/stabilityai diff --git a/.github/workflows/sdk-foundation-models-system-distillation-conversation-distillation_conversational_task.yml b/.github/workflows/sdk-foundation-models-system-distillation-conversation-distillation_conversational_task.yml new file mode 100644 index 0000000000..d55198c0ca --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-distillation-conversation-distillation_conversational_task.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-distillation-conversation-distillation_conversational_task +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/distillation/conversation/** + - .github/workflows/sdk-foundation-models-system-distillation-conversation-distillation_conversational_task.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/distillation/conversation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/distillation/conversation/distillation_conversational_task.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "distillation_conversational_task.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python distillation_conversational_task.ipynb distillation_conversational_task.output.ipynb + working-directory: sdk/python/foundation-models/system/distillation/conversation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: distillation_conversational_task + path: sdk/python/foundation-models/system/distillation/conversation diff --git a/.github/workflows/sdk-foundation-models-system-distillation-math-distillation_math.yml b/.github/workflows/sdk-foundation-models-system-distillation-math-distillation_math.yml new file mode 100644 index 0000000000..8aa70897f2 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-distillation-math-distillation_math.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-distillation-math-distillation_math +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "51 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/distillation/math/** + - .github/workflows/sdk-foundation-models-system-distillation-math-distillation_math.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/distillation/math" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/distillation/math/distillation_math.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "distillation_math.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python distillation_math.ipynb distillation_math.output.ipynb + working-directory: sdk/python/foundation-models/system/distillation/math + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: distillation_math + path: sdk/python/foundation-models/system/distillation/math diff --git a/.github/workflows/sdk-foundation-models-system-distillation-nli-distillation_chat_completion.yml b/.github/workflows/sdk-foundation-models-system-distillation-nli-distillation_chat_completion.yml index 4b3be84b67..56f97aa178 100644 --- a/.github/workflows/sdk-foundation-models-system-distillation-nli-distillation_chat_completion.yml +++ b/.github/workflows/sdk-foundation-models-system-distillation-nli-distillation_chat_completion.yml @@ -1,7 +1,7 @@ # This code is autogenerated. # Code is generated by running custom script: python3 readme.py # Any manual changes to this file may cause incorrect behavior. -# Any manual changes will be overwritten if the code is regenerated. +# Any manual changes will be overwritten if the code is regenerated. name: sdk-foundation-models-system-distillation-nli-distillation_chat_completion # This file is created by sdk/python/readme.py. @@ -14,7 +14,7 @@ on: branches: - main paths: - - sdk/python/foundation-models/system/distillation/nli/ + - sdk/python/foundation-models/system/distillation/nli/** - .github/workflows/sdk-foundation-models-system-distillation-nli-distillation_chat_completion.yml - sdk/python/dev-requirements.txt - infra/bootstrapping/** @@ -34,9 +34,11 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login uses: azure/login@v1 with: @@ -56,11 +58,11 @@ jobs: bash setup.sh working-directory: sdk/python continue-on-error: true - # - name: validate readme - # run: | - # python check-readme.py "${{ github.workspace }}" "${{ github.workspace }}/sdk/python/foundation-models/system/distillation" - # working-directory: infra/bootstrapping - # continue-on-error: false + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/distillation/nli" + working-directory: infra/bootstrapping + continue-on-error: false - name: setup-cli run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; @@ -68,6 +70,15 @@ jobs: bash setup.sh working-directory: cli continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none - name: run foundation-models/system/distillation/nli/distillation_chat_completion.ipynb run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; @@ -82,4 +93,4 @@ jobs: uses: ./.github/actions/upload-artifact with: name: distillation_chat_completion - path: sdk/python/foundation-models/system/distillation/nli \ No newline at end of file + path: sdk/python/foundation-models/system/distillation/nli diff --git a/.github/workflows/sdk-foundation-models-system-distillation-nlu_qa-distillation_nlu_qa_task.yml b/.github/workflows/sdk-foundation-models-system-distillation-nlu_qa-distillation_nlu_qa_task.yml new file mode 100644 index 0000000000..9bc06f51b1 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-distillation-nlu_qa-distillation_nlu_qa_task.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-distillation-nlu_qa-distillation_nlu_qa_task +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "53 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/distillation/nlu_qa/** + - .github/workflows/sdk-foundation-models-system-distillation-nlu_qa-distillation_nlu_qa_task.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/distillation/nlu_qa" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/distillation/nlu_qa/distillation_nlu_qa_task.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "distillation_nlu_qa_task.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python distillation_nlu_qa_task.ipynb distillation_nlu_qa_task.output.ipynb + working-directory: sdk/python/foundation-models/system/distillation/nlu_qa + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: distillation_nlu_qa_task + path: sdk/python/foundation-models/system/distillation/nlu_qa diff --git a/.github/workflows/sdk-foundation-models-system-distillation-nlu_qa-distillation_qa_math.yml b/.github/workflows/sdk-foundation-models-system-distillation-nlu_qa-distillation_qa_math.yml new file mode 100644 index 0000000000..c872edeb38 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-distillation-nlu_qa-distillation_qa_math.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-distillation-nlu_qa-distillation_qa_math +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "52 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/distillation/nlu_qa/** + - .github/workflows/sdk-foundation-models-system-distillation-nlu_qa-distillation_qa_math.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/distillation/nlu_qa" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/distillation/nlu_qa/distillation_qa_math.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "distillation_qa_math.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python distillation_qa_math.ipynb distillation_qa_math.output.ipynb + working-directory: sdk/python/foundation-models/system/distillation/nlu_qa + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: distillation_qa_math + path: sdk/python/foundation-models/system/distillation/nlu_qa diff --git a/.github/workflows/sdk-foundation-models-system-distillation-summarization-distillation_summarization.yml b/.github/workflows/sdk-foundation-models-system-distillation-summarization-distillation_summarization.yml new file mode 100644 index 0000000000..e79b6ed9a3 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-distillation-summarization-distillation_summarization.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-distillation-summarization-distillation_summarization +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/distillation/summarization/** + - .github/workflows/sdk-foundation-models-system-distillation-summarization-distillation_summarization.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/distillation/summarization" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/distillation/summarization/distillation_summarization.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "distillation_summarization.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python distillation_summarization.ipynb distillation_summarization.output.ipynb + working-directory: sdk/python/foundation-models/system/distillation/summarization + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: distillation_summarization + path: sdk/python/foundation-models/system/distillation/summarization diff --git a/.github/workflows/sdk-foundation-models-system-evaluation-chat-completion-chat-completion.yml b/.github/workflows/sdk-foundation-models-system-evaluation-chat-completion-chat-completion.yml new file mode 100644 index 0000000000..4e1f974393 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-evaluation-chat-completion-chat-completion.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-evaluation-chat-completion-chat-completion +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "8 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/evaluation/chat-completion/** + - .github/workflows/sdk-foundation-models-system-evaluation-chat-completion-chat-completion.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/evaluation/chat-completion" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/evaluation/chat-completion/chat-completion.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "chat-completion.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python chat-completion.ipynb chat-completion.output.ipynb + working-directory: sdk/python/foundation-models/system/evaluation/chat-completion + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: chat-completion + path: sdk/python/foundation-models/system/evaluation/chat-completion diff --git a/.github/workflows/sdk-foundation-models-system-evaluation-fill-mask-fill-mask.yml b/.github/workflows/sdk-foundation-models-system-evaluation-fill-mask-fill-mask.yml new file mode 100644 index 0000000000..1b921dfe41 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-evaluation-fill-mask-fill-mask.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-evaluation-fill-mask-fill-mask +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "34 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/evaluation/fill-mask/** + - .github/workflows/sdk-foundation-models-system-evaluation-fill-mask-fill-mask.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/evaluation/fill-mask" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/evaluation/fill-mask/fill-mask.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "fill-mask.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python fill-mask.ipynb fill-mask.output.ipynb + working-directory: sdk/python/foundation-models/system/evaluation/fill-mask + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: fill-mask + path: sdk/python/foundation-models/system/evaluation/fill-mask diff --git a/.github/workflows/sdk-foundation-models-system-evaluation-image-classification-multiclass-classification-image-multiclass-classification.yml b/.github/workflows/sdk-foundation-models-system-evaluation-image-classification-multiclass-classification-image-multiclass-classification.yml new file mode 100644 index 0000000000..2f9a6e6cb3 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-evaluation-image-classification-multiclass-classification-image-multiclass-classification.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-evaluation-image-classification-multiclass-classification-image-multiclass-classification +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "54 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/evaluation/image-classification/multiclass-classification/** + - .github/workflows/sdk-foundation-models-system-evaluation-image-classification-multiclass-classification-image-multiclass-classification.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/evaluation/image-classification/multiclass-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/evaluation/image-classification/multiclass-classification/image-multiclass-classification.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-multiclass-classification.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-multiclass-classification.ipynb image-multiclass-classification.output.ipynb + working-directory: sdk/python/foundation-models/system/evaluation/image-classification/multiclass-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-multiclass-classification + path: sdk/python/foundation-models/system/evaluation/image-classification/multiclass-classification diff --git a/.github/workflows/sdk-foundation-models-system-evaluation-image-classification-multilabel-classification-image-multilabel-classification.yml b/.github/workflows/sdk-foundation-models-system-evaluation-image-classification-multilabel-classification-image-multilabel-classification.yml new file mode 100644 index 0000000000..8b67e5138f --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-evaluation-image-classification-multilabel-classification-image-multilabel-classification.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-evaluation-image-classification-multilabel-classification-image-multilabel-classification +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "32 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/evaluation/image-classification/multilabel-classification/** + - .github/workflows/sdk-foundation-models-system-evaluation-image-classification-multilabel-classification-image-multilabel-classification.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/evaluation/image-classification/multilabel-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/evaluation/image-classification/multilabel-classification/image-multilabel-classification.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-multilabel-classification.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-multilabel-classification.ipynb image-multilabel-classification.output.ipynb + working-directory: sdk/python/foundation-models/system/evaluation/image-classification/multilabel-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-multilabel-classification + path: sdk/python/foundation-models/system/evaluation/image-classification/multilabel-classification diff --git a/.github/workflows/sdk-foundation-models-system-evaluation-image-instance-segmentation-image-instance-segmentation.yml b/.github/workflows/sdk-foundation-models-system-evaluation-image-instance-segmentation-image-instance-segmentation.yml new file mode 100644 index 0000000000..d2136c5a17 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-evaluation-image-instance-segmentation-image-instance-segmentation.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-evaluation-image-instance-segmentation-image-instance-segmentation +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "16 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/evaluation/image-instance-segmentation/** + - .github/workflows/sdk-foundation-models-system-evaluation-image-instance-segmentation-image-instance-segmentation.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/evaluation/image-instance-segmentation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/evaluation/image-instance-segmentation/image-instance-segmentation.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-instance-segmentation.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-instance-segmentation.ipynb image-instance-segmentation.output.ipynb + working-directory: sdk/python/foundation-models/system/evaluation/image-instance-segmentation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-instance-segmentation + path: sdk/python/foundation-models/system/evaluation/image-instance-segmentation diff --git a/.github/workflows/sdk-foundation-models-system-evaluation-image-object-detection-image-object-detection.yml b/.github/workflows/sdk-foundation-models-system-evaluation-image-object-detection-image-object-detection.yml new file mode 100644 index 0000000000..25407d9309 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-evaluation-image-object-detection-image-object-detection.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-evaluation-image-object-detection-image-object-detection +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "39 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/evaluation/image-object-detection/** + - .github/workflows/sdk-foundation-models-system-evaluation-image-object-detection-image-object-detection.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/evaluation/image-object-detection" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/evaluation/image-object-detection/image-object-detection.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-object-detection.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-object-detection.ipynb image-object-detection.output.ipynb + working-directory: sdk/python/foundation-models/system/evaluation/image-object-detection + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-object-detection + path: sdk/python/foundation-models/system/evaluation/image-object-detection diff --git a/.github/workflows/sdk-foundation-models-system-evaluation-question-answering-question-answering.yml b/.github/workflows/sdk-foundation-models-system-evaluation-question-answering-question-answering.yml new file mode 100644 index 0000000000..6b3570456a --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-evaluation-question-answering-question-answering.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-evaluation-question-answering-question-answering +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "12 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/evaluation/question-answering/** + - .github/workflows/sdk-foundation-models-system-evaluation-question-answering-question-answering.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/evaluation/question-answering" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/evaluation/question-answering/question-answering.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "question-answering.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python question-answering.ipynb question-answering.output.ipynb + working-directory: sdk/python/foundation-models/system/evaluation/question-answering + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: question-answering + path: sdk/python/foundation-models/system/evaluation/question-answering diff --git a/.github/workflows/sdk-foundation-models-system-evaluation-summarization-abstractive-and-extractive-summarization.yml b/.github/workflows/sdk-foundation-models-system-evaluation-summarization-abstractive-and-extractive-summarization.yml new file mode 100644 index 0000000000..f373e2e1ff --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-evaluation-summarization-abstractive-and-extractive-summarization.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-evaluation-summarization-abstractive-and-extractive-summarization +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 9/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/evaluation/summarization/** + - .github/workflows/sdk-foundation-models-system-evaluation-summarization-abstractive-and-extractive-summarization.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/evaluation/summarization" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/evaluation/summarization/abstractive-and-extractive-summarization.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "abstractive-and-extractive-summarization.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python abstractive-and-extractive-summarization.ipynb abstractive-and-extractive-summarization.output.ipynb + working-directory: sdk/python/foundation-models/system/evaluation/summarization + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: abstractive-and-extractive-summarization + path: sdk/python/foundation-models/system/evaluation/summarization diff --git a/.github/workflows/sdk-foundation-models-system-evaluation-text-classification-entailment-contradiction.yml b/.github/workflows/sdk-foundation-models-system-evaluation-text-classification-entailment-contradiction.yml new file mode 100644 index 0000000000..cdd67f22f6 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-evaluation-text-classification-entailment-contradiction.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-evaluation-text-classification-entailment-contradiction +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "56 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/evaluation/text-classification/** + - .github/workflows/sdk-foundation-models-system-evaluation-text-classification-entailment-contradiction.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/evaluation/text-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/evaluation/text-classification/entailment-contradiction.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "entailment-contradiction.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python entailment-contradiction.ipynb entailment-contradiction.output.ipynb + working-directory: sdk/python/foundation-models/system/evaluation/text-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: entailment-contradiction + path: sdk/python/foundation-models/system/evaluation/text-classification diff --git a/.github/workflows/sdk-foundation-models-system-evaluation-text-classification-sentiment-analysis.yml b/.github/workflows/sdk-foundation-models-system-evaluation-text-classification-sentiment-analysis.yml new file mode 100644 index 0000000000..de7cbecd6b --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-evaluation-text-classification-sentiment-analysis.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-evaluation-text-classification-sentiment-analysis +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "20 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/evaluation/text-classification/** + - .github/workflows/sdk-foundation-models-system-evaluation-text-classification-sentiment-analysis.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/evaluation/text-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/evaluation/text-classification/sentiment-analysis.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "sentiment-analysis.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python sentiment-analysis.ipynb sentiment-analysis.output.ipynb + working-directory: sdk/python/foundation-models/system/evaluation/text-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: sentiment-analysis + path: sdk/python/foundation-models/system/evaluation/text-classification diff --git a/.github/workflows/sdk-foundation-models-system-evaluation-text-generation-text-generation.yml b/.github/workflows/sdk-foundation-models-system-evaluation-text-generation-text-generation.yml new file mode 100644 index 0000000000..84d5c14274 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-evaluation-text-generation-text-generation.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-evaluation-text-generation-text-generation +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "57 9/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/evaluation/text-generation/** + - .github/workflows/sdk-foundation-models-system-evaluation-text-generation-text-generation.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/evaluation/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/evaluation/text-generation/text-generation.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text-generation.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text-generation.ipynb text-generation.output.ipynb + working-directory: sdk/python/foundation-models/system/evaluation/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text-generation + path: sdk/python/foundation-models/system/evaluation/text-generation diff --git a/.github/workflows/sdk-foundation-models-system-evaluation-token-classification-news-articles-entity-recognition.yml b/.github/workflows/sdk-foundation-models-system-evaluation-token-classification-news-articles-entity-recognition.yml new file mode 100644 index 0000000000..cd4e12e7c9 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-evaluation-token-classification-news-articles-entity-recognition.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-evaluation-token-classification-news-articles-entity-recognition +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "8 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/evaluation/token-classification/** + - .github/workflows/sdk-foundation-models-system-evaluation-token-classification-news-articles-entity-recognition.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/evaluation/token-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/evaluation/token-classification/news-articles-entity-recognition.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "news-articles-entity-recognition.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python news-articles-entity-recognition.ipynb news-articles-entity-recognition.output.ipynb + working-directory: sdk/python/foundation-models/system/evaluation/token-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: news-articles-entity-recognition + path: sdk/python/foundation-models/system/evaluation/token-classification diff --git a/.github/workflows/sdk-foundation-models-system-evaluation-translation-translation-romanian-to-english.yml b/.github/workflows/sdk-foundation-models-system-evaluation-translation-translation-romanian-to-english.yml new file mode 100644 index 0000000000..b3fc6f9b09 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-evaluation-translation-translation-romanian-to-english.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-evaluation-translation-translation-romanian-to-english +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/evaluation/translation/** + - .github/workflows/sdk-foundation-models-system-evaluation-translation-translation-romanian-to-english.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/evaluation/translation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/evaluation/translation/translation-romanian-to-english.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "translation-romanian-to-english.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python translation-romanian-to-english.ipynb translation-romanian-to-english.output.ipynb + working-directory: sdk/python/foundation-models/system/evaluation/translation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: translation-romanian-to-english + path: sdk/python/foundation-models/system/evaluation/translation diff --git a/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-datagen-synthetic-data-generation.yml b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-datagen-synthetic-data-generation.yml new file mode 100644 index 0000000000..c9aa4d6a58 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-datagen-synthetic-data-generation.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-Llama-notebooks-datagen-synthetic-data-generation +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "8 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/Llama-notebooks/datagen/** + - .github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-datagen-synthetic-data-generation.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/Llama-notebooks/datagen" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/Llama-notebooks/datagen/synthetic-data-generation.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "synthetic-data-generation.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python synthetic-data-generation.ipynb synthetic-data-generation.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/Llama-notebooks/datagen + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: synthetic-data-generation + path: sdk/python/foundation-models/system/finetune/Llama-notebooks/datagen diff --git a/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode-serverless.yml b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode-serverless.yml new file mode 100644 index 0000000000..c80bd1fcf2 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode-serverless.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode-serverless +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "46 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/** + - .github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode-serverless.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/emotion-detection-llama-multinode-serverless.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "emotion-detection-llama-multinode-serverless.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python emotion-detection-llama-multinode-serverless.ipynb emotion-detection-llama-multinode-serverless.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: emotion-detection-llama-multinode-serverless + path: sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification diff --git a/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode.yml b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode.yml new file mode 100644 index 0000000000..3e858b25b2 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "51 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/** + - .github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/emotion-detection-llama-multinode.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "emotion-detection-llama-multinode.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python emotion-detection-llama-multinode.ipynb emotion-detection-llama-multinode.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: emotion-detection-llama-multinode + path: sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification diff --git a/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama-serverless-compute.yml b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama-serverless-compute.yml new file mode 100644 index 0000000000..eb967a3a68 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama-serverless-compute.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama-serverless-compute +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "36 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification/** + - .github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama-serverless-compute.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/Llama-notebooks/text-classification/emotion-detection-llama-serverless-compute.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "emotion-detection-llama-serverless-compute.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python emotion-detection-llama-serverless-compute.ipynb emotion-detection-llama-serverless-compute.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: emotion-detection-llama-serverless-compute + path: sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification diff --git a/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama.yml b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama.yml new file mode 100644 index 0000000000..0f5c1eb095 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "55 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification/** + - .github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/Llama-notebooks/text-classification/emotion-detection-llama.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "emotion-detection-llama.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python emotion-detection-llama.ipynb emotion-detection-llama.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: emotion-detection-llama + path: sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification diff --git a/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-abstractive_qna_with_text_gen.yml b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-abstractive_qna_with_text_gen.yml new file mode 100644 index 0000000000..d704f1fe4f --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-abstractive_qna_with_text_gen.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-abstractive_qna_with_text_gen +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "59 6/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation/** + - .github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-abstractive_qna_with_text_gen.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/Llama-notebooks/text-generation/abstractive_qna_with_text_gen.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "abstractive_qna_with_text_gen.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python abstractive_qna_with_text_gen.ipynb abstractive_qna_with_text_gen.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: abstractive_qna_with_text_gen + path: sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation diff --git a/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-summarization_with_text_gen.yml b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-summarization_with_text_gen.yml new file mode 100644 index 0000000000..cfdc6fe8fd --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-summarization_with_text_gen.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-summarization_with_text_gen +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "57 9/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation/** + - .github/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-summarization_with_text_gen.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/Llama-notebooks/text-generation/summarization_with_text_gen.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "summarization_with_text_gen.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python summarization_with_text_gen.ipynb summarization_with_text_gen.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: summarization_with_text_gen + path: sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation diff --git a/.github/workflows/sdk-foundation-models-system-finetune-chat-completion-chat-completion.yml b/.github/workflows/sdk-foundation-models-system-finetune-chat-completion-chat-completion.yml new file mode 100644 index 0000000000..0e0b8ba440 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-chat-completion-chat-completion.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-chat-completion-chat-completion +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "8 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/chat-completion/** + - .github/workflows/sdk-foundation-models-system-finetune-chat-completion-chat-completion.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/chat-completion" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/chat-completion/chat-completion.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "chat-completion.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python chat-completion.ipynb chat-completion.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/chat-completion + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: chat-completion + path: sdk/python/foundation-models/system/finetune/chat-completion diff --git a/.github/workflows/sdk-foundation-models-system-finetune-image-classification-multiclass-classification-hftransformers-fridgeobjects-multiclass-classification.yml b/.github/workflows/sdk-foundation-models-system-finetune-image-classification-multiclass-classification-hftransformers-fridgeobjects-multiclass-classification.yml new file mode 100644 index 0000000000..71faf88933 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-image-classification-multiclass-classification-hftransformers-fridgeobjects-multiclass-classification.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-image-classification-multiclass-classification-hftransformers-fridgeobjects-multiclass-classification +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "22 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/image-classification/multiclass-classification/** + - .github/workflows/sdk-foundation-models-system-finetune-image-classification-multiclass-classification-hftransformers-fridgeobjects-multiclass-classification.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/image-classification/multiclass-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/image-classification/multiclass-classification/hftransformers-fridgeobjects-multiclass-classification.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "hftransformers-fridgeobjects-multiclass-classification.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python hftransformers-fridgeobjects-multiclass-classification.ipynb hftransformers-fridgeobjects-multiclass-classification.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/image-classification/multiclass-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: hftransformers-fridgeobjects-multiclass-classification + path: sdk/python/foundation-models/system/finetune/image-classification/multiclass-classification diff --git a/.github/workflows/sdk-foundation-models-system-finetune-image-classification-multilabel-classification-hftransformers-fridgeobjects-multilabel-classification.yml b/.github/workflows/sdk-foundation-models-system-finetune-image-classification-multilabel-classification-hftransformers-fridgeobjects-multilabel-classification.yml new file mode 100644 index 0000000000..4cb3b01353 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-image-classification-multilabel-classification-hftransformers-fridgeobjects-multilabel-classification.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-image-classification-multilabel-classification-hftransformers-fridgeobjects-multilabel-classification +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "17 6/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/image-classification/multilabel-classification/** + - .github/workflows/sdk-foundation-models-system-finetune-image-classification-multilabel-classification-hftransformers-fridgeobjects-multilabel-classification.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/image-classification/multilabel-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/image-classification/multilabel-classification/hftransformers-fridgeobjects-multilabel-classification.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "hftransformers-fridgeobjects-multilabel-classification.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python hftransformers-fridgeobjects-multilabel-classification.ipynb hftransformers-fridgeobjects-multilabel-classification.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/image-classification/multilabel-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: hftransformers-fridgeobjects-multilabel-classification + path: sdk/python/foundation-models/system/finetune/image-classification/multilabel-classification diff --git a/.github/workflows/sdk-foundation-models-system-finetune-image-instance-segmentation-mmdetection-fridgeobjects-instance-segmentation.yml b/.github/workflows/sdk-foundation-models-system-finetune-image-instance-segmentation-mmdetection-fridgeobjects-instance-segmentation.yml new file mode 100644 index 0000000000..8589ef86f6 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-image-instance-segmentation-mmdetection-fridgeobjects-instance-segmentation.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-image-instance-segmentation-mmdetection-fridgeobjects-instance-segmentation +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "16 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/image-instance-segmentation/** + - .github/workflows/sdk-foundation-models-system-finetune-image-instance-segmentation-mmdetection-fridgeobjects-instance-segmentation.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/image-instance-segmentation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/image-instance-segmentation/mmdetection-fridgeobjects-instance-segmentation.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "mmdetection-fridgeobjects-instance-segmentation.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python mmdetection-fridgeobjects-instance-segmentation.ipynb mmdetection-fridgeobjects-instance-segmentation.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/image-instance-segmentation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: mmdetection-fridgeobjects-instance-segmentation + path: sdk/python/foundation-models/system/finetune/image-instance-segmentation diff --git a/.github/workflows/sdk-foundation-models-system-finetune-image-object-detection-mmdetection-fridgeobjects-object-detection.yml b/.github/workflows/sdk-foundation-models-system-finetune-image-object-detection-mmdetection-fridgeobjects-object-detection.yml new file mode 100644 index 0000000000..12fba7cc53 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-image-object-detection-mmdetection-fridgeobjects-object-detection.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-image-object-detection-mmdetection-fridgeobjects-object-detection +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "52 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/image-object-detection/** + - .github/workflows/sdk-foundation-models-system-finetune-image-object-detection-mmdetection-fridgeobjects-object-detection.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/image-object-detection" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/image-object-detection/mmdetection-fridgeobjects-object-detection.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "mmdetection-fridgeobjects-object-detection.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python mmdetection-fridgeobjects-object-detection.ipynb mmdetection-fridgeobjects-object-detection.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/image-object-detection + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: mmdetection-fridgeobjects-object-detection + path: sdk/python/foundation-models/system/finetune/image-object-detection diff --git a/.github/workflows/sdk-foundation-models-system-finetune-pipeline_validations-common.yml b/.github/workflows/sdk-foundation-models-system-finetune-pipeline_validations-common.yml new file mode 100644 index 0000000000..15d6e5d5d6 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-pipeline_validations-common.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-pipeline_validations-common +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "7 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/pipeline_validations/** + - .github/workflows/sdk-foundation-models-system-finetune-pipeline_validations-common.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/pipeline_validations" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/pipeline_validations/common.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "common.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python common.ipynb common.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/pipeline_validations + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: common + path: sdk/python/foundation-models/system/finetune/pipeline_validations diff --git a/.github/workflows/sdk-foundation-models-system-finetune-question-answering-extractive-qa.yml b/.github/workflows/sdk-foundation-models-system-finetune-question-answering-extractive-qa.yml new file mode 100644 index 0000000000..1cd22605ad --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-question-answering-extractive-qa.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-question-answering-extractive-qa +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "26 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/question-answering/** + - .github/workflows/sdk-foundation-models-system-finetune-question-answering-extractive-qa.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/question-answering" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/question-answering/extractive-qa.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "extractive-qa.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python extractive-qa.ipynb extractive-qa.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/question-answering + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: extractive-qa + path: sdk/python/foundation-models/system/finetune/question-answering diff --git a/.github/workflows/sdk-foundation-models-system-finetune-scenarios-very-large-model-deepspeed-stage3-finetuning-summarization_with_text_gen.yml b/.github/workflows/sdk-foundation-models-system-finetune-scenarios-very-large-model-deepspeed-stage3-finetuning-summarization_with_text_gen.yml new file mode 100644 index 0000000000..403b26fa3d --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-scenarios-very-large-model-deepspeed-stage3-finetuning-summarization_with_text_gen.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-scenarios-very-large-model-deepspeed-stage3-finetuning-summarization_with_text_gen +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "57 9/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/scenarios/very-large-model-deepspeed-stage3-finetuning/** + - .github/workflows/sdk-foundation-models-system-finetune-scenarios-very-large-model-deepspeed-stage3-finetuning-summarization_with_text_gen.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/scenarios/very-large-model-deepspeed-stage3-finetuning" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/scenarios/very-large-model-deepspeed-stage3-finetuning/summarization_with_text_gen.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "summarization_with_text_gen.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python summarization_with_text_gen.ipynb summarization_with_text_gen.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/scenarios/very-large-model-deepspeed-stage3-finetuning + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: summarization_with_text_gen + path: sdk/python/foundation-models/system/finetune/scenarios/very-large-model-deepspeed-stage3-finetuning diff --git a/.github/workflows/sdk-foundation-models-system-finetune-summarization-news-summary.yml b/.github/workflows/sdk-foundation-models-system-finetune-summarization-news-summary.yml new file mode 100644 index 0000000000..1fe28cf369 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-summarization-news-summary.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-summarization-news-summary +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "21 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/summarization/** + - .github/workflows/sdk-foundation-models-system-finetune-summarization-news-summary.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/summarization" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/summarization/news-summary.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "news-summary.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python news-summary.ipynb news-summary.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/summarization + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: news-summary + path: sdk/python/foundation-models/system/finetune/summarization diff --git a/.github/workflows/sdk-foundation-models-system-finetune-text-classification-emotion-detection.yml b/.github/workflows/sdk-foundation-models-system-finetune-text-classification-emotion-detection.yml new file mode 100644 index 0000000000..545c1e0d3a --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-text-classification-emotion-detection.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-text-classification-emotion-detection +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "52 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/text-classification/** + - .github/workflows/sdk-foundation-models-system-finetune-text-classification-emotion-detection.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/text-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/text-classification/emotion-detection.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "emotion-detection.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python emotion-detection.ipynb emotion-detection.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/text-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: emotion-detection + path: sdk/python/foundation-models/system/finetune/text-classification diff --git a/.github/workflows/sdk-foundation-models-system-finetune-text-generation-abstractive_qna_with_text_gen.yml b/.github/workflows/sdk-foundation-models-system-finetune-text-generation-abstractive_qna_with_text_gen.yml new file mode 100644 index 0000000000..589e7f8cf2 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-text-generation-abstractive_qna_with_text_gen.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-text-generation-abstractive_qna_with_text_gen +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "59 6/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/text-generation/** + - .github/workflows/sdk-foundation-models-system-finetune-text-generation-abstractive_qna_with_text_gen.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/text-generation/abstractive_qna_with_text_gen.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "abstractive_qna_with_text_gen.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python abstractive_qna_with_text_gen.ipynb abstractive_qna_with_text_gen.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: abstractive_qna_with_text_gen + path: sdk/python/foundation-models/system/finetune/text-generation diff --git a/.github/workflows/sdk-foundation-models-system-finetune-text-generation-summarization_with_text_gen.yml b/.github/workflows/sdk-foundation-models-system-finetune-text-generation-summarization_with_text_gen.yml new file mode 100644 index 0000000000..0e5921a36e --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-text-generation-summarization_with_text_gen.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-text-generation-summarization_with_text_gen +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "57 9/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/text-generation/** + - .github/workflows/sdk-foundation-models-system-finetune-text-generation-summarization_with_text_gen.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/text-generation/summarization_with_text_gen.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "summarization_with_text_gen.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python summarization_with_text_gen.ipynb summarization_with_text_gen.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: summarization_with_text_gen + path: sdk/python/foundation-models/system/finetune/text-generation diff --git a/.github/workflows/sdk-foundation-models-system-finetune-text-to-image-diffusers-dreambooth-dog-text-to-image.yml b/.github/workflows/sdk-foundation-models-system-finetune-text-to-image-diffusers-dreambooth-dog-text-to-image.yml new file mode 100644 index 0000000000..ba5db814cc --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-text-to-image-diffusers-dreambooth-dog-text-to-image.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-text-to-image-diffusers-dreambooth-dog-text-to-image +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "47 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/text-to-image/** + - .github/workflows/sdk-foundation-models-system-finetune-text-to-image-diffusers-dreambooth-dog-text-to-image.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/text-to-image/diffusers-dreambooth-dog-text-to-image.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "diffusers-dreambooth-dog-text-to-image.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python diffusers-dreambooth-dog-text-to-image.ipynb diffusers-dreambooth-dog-text-to-image.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: diffusers-dreambooth-dog-text-to-image + path: sdk/python/foundation-models/system/finetune/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-finetune-token-classification-token-classification.yml b/.github/workflows/sdk-foundation-models-system-finetune-token-classification-token-classification.yml new file mode 100644 index 0000000000..1579370d05 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-token-classification-token-classification.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-token-classification-token-classification +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "45 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/token-classification/** + - .github/workflows/sdk-foundation-models-system-finetune-token-classification-token-classification.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/token-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/token-classification/token-classification.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "token-classification.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python token-classification.ipynb token-classification.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/token-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: token-classification + path: sdk/python/foundation-models/system/finetune/token-classification diff --git a/.github/workflows/sdk-foundation-models-system-finetune-translation-translation.yml b/.github/workflows/sdk-foundation-models-system-finetune-translation-translation.yml new file mode 100644 index 0000000000..9c693f3171 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-translation-translation.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-translation-translation +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "20 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/translation/** + - .github/workflows/sdk-foundation-models-system-finetune-translation-translation.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/translation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/translation/translation.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "translation.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python translation.ipynb translation.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/translation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: translation + path: sdk/python/foundation-models/system/finetune/translation diff --git a/.github/workflows/sdk-foundation-models-system-finetune-video-multi-object-tracking-mmtracking-video-multi-object-tracking.yml b/.github/workflows/sdk-foundation-models-system-finetune-video-multi-object-tracking-mmtracking-video-multi-object-tracking.yml new file mode 100644 index 0000000000..d9efa04eed --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-finetune-video-multi-object-tracking-mmtracking-video-multi-object-tracking.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-finetune-video-multi-object-tracking-mmtracking-video-multi-object-tracking +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "30 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/finetune/video-multi-object-tracking/** + - .github/workflows/sdk-foundation-models-system-finetune-video-multi-object-tracking-mmtracking-video-multi-object-tracking.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/finetune/video-multi-object-tracking" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/finetune/video-multi-object-tracking/mmtracking-video-multi-object-tracking.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "mmtracking-video-multi-object-tracking.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python mmtracking-video-multi-object-tracking.ipynb mmtracking-video-multi-object-tracking.output.ipynb + working-directory: sdk/python/foundation-models/system/finetune/video-multi-object-tracking + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: mmtracking-video-multi-object-tracking + path: sdk/python/foundation-models/system/finetune/video-multi-object-tracking diff --git a/.github/workflows/sdk-foundation-models-system-import-import_mmdetection_model_into_registry.yml b/.github/workflows/sdk-foundation-models-system-import-import_mmdetection_model_into_registry.yml new file mode 100644 index 0000000000..74ba17933b --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-import-import_mmdetection_model_into_registry.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-import-import_mmdetection_model_into_registry +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "56 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/import/** + - .github/workflows/sdk-foundation-models-system-import-import_mmdetection_model_into_registry.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/import" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/import/import_mmdetection_model_into_registry.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "import_mmdetection_model_into_registry.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python import_mmdetection_model_into_registry.ipynb import_mmdetection_model_into_registry.output.ipynb + working-directory: sdk/python/foundation-models/system/import + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: import_mmdetection_model_into_registry + path: sdk/python/foundation-models/system/import diff --git a/.github/workflows/sdk-foundation-models-system-import-import_model_into_registry.yml b/.github/workflows/sdk-foundation-models-system-import-import_model_into_registry.yml index cf756261e0..d74a4ea5ca 100644 --- a/.github/workflows/sdk-foundation-models-system-import-import_model_into_registry.yml +++ b/.github/workflows/sdk-foundation-models-system-import-import_model_into_registry.yml @@ -19,9 +19,7 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh -env: - MMDETECTION_MODEL_ID: None - MMDETECTION_TASK_NAME: image-object-detection + permissions: id-token: write concurrency: @@ -36,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-foundation-models-system-inference-automatic-speech-recognition-asr-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-automatic-speech-recognition-asr-batch-endpoint.yml new file mode 100644 index 0000000000..0cac57c776 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-automatic-speech-recognition-asr-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-automatic-speech-recognition-asr-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "12 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/automatic-speech-recognition/** + - .github/workflows/sdk-foundation-models-system-inference-automatic-speech-recognition-asr-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/automatic-speech-recognition" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/automatic-speech-recognition/asr-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "asr-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python asr-batch-endpoint.ipynb asr-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/automatic-speech-recognition + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: asr-batch-endpoint + path: sdk/python/foundation-models/system/inference/automatic-speech-recognition diff --git a/.github/workflows/sdk-foundation-models-system-inference-automatic-speech-recognition-asr-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-automatic-speech-recognition-asr-online-endpoint.yml new file mode 100644 index 0000000000..41696f07cb --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-automatic-speech-recognition-asr-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-automatic-speech-recognition-asr-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "35 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/automatic-speech-recognition/** + - .github/workflows/sdk-foundation-models-system-inference-automatic-speech-recognition-asr-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/automatic-speech-recognition" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/automatic-speech-recognition/asr-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "asr-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python asr-online-endpoint.ipynb asr-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/automatic-speech-recognition + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: asr-online-endpoint + path: sdk/python/foundation-models/system/inference/automatic-speech-recognition diff --git a/.github/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-batch-endpoint.yml new file mode 100644 index 0000000000..4cd4411d12 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-fill-mask-fill-mask-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "46 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/fill-mask/** + - .github/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/fill-mask" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/fill-mask/fill-mask-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "fill-mask-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python fill-mask-batch-endpoint.ipynb fill-mask-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/fill-mask + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: fill-mask-batch-endpoint + path: sdk/python/foundation-models/system/inference/fill-mask diff --git a/.github/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-online-endpoint-oss.yml b/.github/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-online-endpoint-oss.yml new file mode 100644 index 0000000000..a5bbe764e3 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-online-endpoint-oss.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-fill-mask-fill-mask-online-endpoint-oss +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "58 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/fill-mask/** + - .github/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-online-endpoint-oss.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/fill-mask" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/fill-mask/fill-mask-online-endpoint-oss.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "fill-mask-online-endpoint-oss.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python fill-mask-online-endpoint-oss.ipynb fill-mask-online-endpoint-oss.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/fill-mask + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: fill-mask-online-endpoint-oss + path: sdk/python/foundation-models/system/inference/fill-mask diff --git a/.github/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-online-endpoint.yml new file mode 100644 index 0000000000..2adc121ef7 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-fill-mask-fill-mask-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "2 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/fill-mask/** + - .github/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/fill-mask" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/fill-mask/fill-mask-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "fill-mask-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python fill-mask-online-endpoint.ipynb fill-mask-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/fill-mask + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: fill-mask-online-endpoint + path: sdk/python/foundation-models/system/inference/fill-mask diff --git a/.github/workflows/sdk-foundation-models-system-inference-image-classification-image-classification-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-image-classification-image-classification-batch-endpoint.yml new file mode 100644 index 0000000000..dd1925c184 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-image-classification-image-classification-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-image-classification-image-classification-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "57 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/image-classification/** + - .github/workflows/sdk-foundation-models-system-inference-image-classification-image-classification-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/image-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/image-classification/image-classification-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-classification-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-classification-batch-endpoint.ipynb image-classification-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/image-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-classification-batch-endpoint + path: sdk/python/foundation-models/system/inference/image-classification diff --git a/.github/workflows/sdk-foundation-models-system-inference-image-classification-image-classification-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-image-classification-image-classification-online-endpoint.yml new file mode 100644 index 0000000000..f51efb2899 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-image-classification-image-classification-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-image-classification-image-classification-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "48 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/image-classification/** + - .github/workflows/sdk-foundation-models-system-inference-image-classification-image-classification-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/image-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/image-classification/image-classification-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-classification-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-classification-online-endpoint.ipynb image-classification-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/image-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-classification-online-endpoint + path: sdk/python/foundation-models/system/inference/image-classification diff --git a/.github/workflows/sdk-foundation-models-system-inference-image-embeddings-image-embeddings-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-image-embeddings-image-embeddings-batch-endpoint.yml new file mode 100644 index 0000000000..b4199d538a --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-image-embeddings-image-embeddings-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-image-embeddings-image-embeddings-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "45 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/image-embeddings/** + - .github/workflows/sdk-foundation-models-system-inference-image-embeddings-image-embeddings-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/image-embeddings" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/image-embeddings/image-embeddings-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-embeddings-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-embeddings-batch-endpoint.ipynb image-embeddings-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/image-embeddings + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-embeddings-batch-endpoint + path: sdk/python/foundation-models/system/inference/image-embeddings diff --git a/.github/workflows/sdk-foundation-models-system-inference-image-embeddings-image-embeddings-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-image-embeddings-image-embeddings-online-endpoint.yml new file mode 100644 index 0000000000..73e7c8af86 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-image-embeddings-image-embeddings-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-image-embeddings-image-embeddings-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "0 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/image-embeddings/** + - .github/workflows/sdk-foundation-models-system-inference-image-embeddings-image-embeddings-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/image-embeddings" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/image-embeddings/image-embeddings-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-embeddings-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-embeddings-online-endpoint.ipynb image-embeddings-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/image-embeddings + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-embeddings-online-endpoint + path: sdk/python/foundation-models/system/inference/image-embeddings diff --git a/.github/workflows/sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-batch-endpoint.yml new file mode 100644 index 0000000000..2fdee77cba --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "49 6/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/image-instance-segmentation/** + - .github/workflows/sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/image-instance-segmentation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/image-instance-segmentation/image-instance-segmentation-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-instance-segmentation-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-instance-segmentation-batch-endpoint.ipynb image-instance-segmentation-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/image-instance-segmentation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-instance-segmentation-batch-endpoint + path: sdk/python/foundation-models/system/inference/image-instance-segmentation diff --git a/.github/workflows/sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-online-endpoint.yml new file mode 100644 index 0000000000..967f132db8 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "12 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/image-instance-segmentation/** + - .github/workflows/sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/image-instance-segmentation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/image-instance-segmentation/image-instance-segmentation-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-instance-segmentation-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-instance-segmentation-online-endpoint.ipynb image-instance-segmentation-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/image-instance-segmentation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-instance-segmentation-online-endpoint + path: sdk/python/foundation-models/system/inference/image-instance-segmentation diff --git a/.github/workflows/sdk-foundation-models-system-inference-image-object-detection-image-object-detection-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-image-object-detection-image-object-detection-batch-endpoint.yml new file mode 100644 index 0000000000..a6f989441c --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-image-object-detection-image-object-detection-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-image-object-detection-image-object-detection-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "4 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/image-object-detection/** + - .github/workflows/sdk-foundation-models-system-inference-image-object-detection-image-object-detection-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/image-object-detection" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/image-object-detection/image-object-detection-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-object-detection-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-object-detection-batch-endpoint.ipynb image-object-detection-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/image-object-detection + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-object-detection-batch-endpoint + path: sdk/python/foundation-models/system/inference/image-object-detection diff --git a/.github/workflows/sdk-foundation-models-system-inference-image-object-detection-image-object-detection-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-image-object-detection-image-object-detection-online-endpoint.yml new file mode 100644 index 0000000000..c5076f4239 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-image-object-detection-image-object-detection-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-image-object-detection-image-object-detection-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "1 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/image-object-detection/** + - .github/workflows/sdk-foundation-models-system-inference-image-object-detection-image-object-detection-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/image-object-detection" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/image-object-detection/image-object-detection-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-object-detection-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-object-detection-online-endpoint.ipynb image-object-detection-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/image-object-detection + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-object-detection-online-endpoint + path: sdk/python/foundation-models/system/inference/image-object-detection diff --git a/.github/workflows/sdk-foundation-models-system-inference-image-text-embeddings-image-text-embeddings-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-image-text-embeddings-image-text-embeddings-batch-endpoint.yml new file mode 100644 index 0000000000..ce308c01da --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-image-text-embeddings-image-text-embeddings-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-image-text-embeddings-image-text-embeddings-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "11 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/image-text-embeddings/** + - .github/workflows/sdk-foundation-models-system-inference-image-text-embeddings-image-text-embeddings-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/image-text-embeddings" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/image-text-embeddings/image-text-embeddings-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-text-embeddings-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-text-embeddings-batch-endpoint.ipynb image-text-embeddings-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/image-text-embeddings + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-text-embeddings-batch-endpoint + path: sdk/python/foundation-models/system/inference/image-text-embeddings diff --git a/.github/workflows/sdk-foundation-models-system-inference-image-text-embeddings-image-text-embeddings-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-image-text-embeddings-image-text-embeddings-online-endpoint.yml new file mode 100644 index 0000000000..e857828575 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-image-text-embeddings-image-text-embeddings-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-image-text-embeddings-image-text-embeddings-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "53 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/image-text-embeddings/** + - .github/workflows/sdk-foundation-models-system-inference-image-text-embeddings-image-text-embeddings-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/image-text-embeddings" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/image-text-embeddings/image-text-embeddings-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-text-embeddings-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-text-embeddings-online-endpoint.ipynb image-text-embeddings-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/image-text-embeddings + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-text-embeddings-online-endpoint + path: sdk/python/foundation-models/system/inference/image-text-embeddings diff --git a/.github/workflows/sdk-foundation-models-system-inference-image-text-embeddings-text-to-image-retrieval.yml b/.github/workflows/sdk-foundation-models-system-inference-image-text-embeddings-text-to-image-retrieval.yml new file mode 100644 index 0000000000..4dc262e7e4 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-image-text-embeddings-text-to-image-retrieval.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-image-text-embeddings-text-to-image-retrieval +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "4 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/image-text-embeddings/** + - .github/workflows/sdk-foundation-models-system-inference-image-text-embeddings-text-to-image-retrieval.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/image-text-embeddings" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/image-text-embeddings/text-to-image-retrieval.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text-to-image-retrieval.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text-to-image-retrieval.ipynb text-to-image-retrieval.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/image-text-embeddings + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text-to-image-retrieval + path: sdk/python/foundation-models/system/inference/image-text-embeddings diff --git a/.github/workflows/sdk-foundation-models-system-inference-image-to-text-image-to-text-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-image-to-text-image-to-text-batch-endpoint.yml new file mode 100644 index 0000000000..7363109d1e --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-image-to-text-image-to-text-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-image-to-text-image-to-text-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "37 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/image-to-text/** + - .github/workflows/sdk-foundation-models-system-inference-image-to-text-image-to-text-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/image-to-text" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/image-to-text/image-to-text-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-to-text-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-to-text-batch-endpoint.ipynb image-to-text-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/image-to-text + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-to-text-batch-endpoint + path: sdk/python/foundation-models/system/inference/image-to-text diff --git a/.github/workflows/sdk-foundation-models-system-inference-image-to-text-image-to-text-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-image-to-text-image-to-text-online-endpoint.yml new file mode 100644 index 0000000000..6631f9cee5 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-image-to-text-image-to-text-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-image-to-text-image-to-text-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "52 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/image-to-text/** + - .github/workflows/sdk-foundation-models-system-inference-image-to-text-image-to-text-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/image-to-text" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/image-to-text/image-to-text-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-to-text-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-to-text-online-endpoint.ipynb image-to-text-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/image-to-text + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-to-text-online-endpoint + path: sdk/python/foundation-models/system/inference/image-to-text diff --git a/.github/workflows/sdk-foundation-models-system-inference-mask-generation-mask-generation-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-mask-generation-mask-generation-batch-endpoint.yml new file mode 100644 index 0000000000..40294ea82c --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-mask-generation-mask-generation-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-mask-generation-mask-generation-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "20 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/mask-generation/** + - .github/workflows/sdk-foundation-models-system-inference-mask-generation-mask-generation-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/mask-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/mask-generation/mask-generation-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "mask-generation-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python mask-generation-batch-endpoint.ipynb mask-generation-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/mask-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: mask-generation-batch-endpoint + path: sdk/python/foundation-models/system/inference/mask-generation diff --git a/.github/workflows/sdk-foundation-models-system-inference-mask-generation-mask-generation-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-mask-generation-mask-generation-online-endpoint.yml new file mode 100644 index 0000000000..ec76d25ee5 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-mask-generation-mask-generation-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-mask-generation-mask-generation-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "53 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/mask-generation/** + - .github/workflows/sdk-foundation-models-system-inference-mask-generation-mask-generation-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/mask-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/mask-generation/mask-generation-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "mask-generation-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python mask-generation-online-endpoint.ipynb mask-generation-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/mask-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: mask-generation-online-endpoint + path: sdk/python/foundation-models/system/inference/mask-generation diff --git a/.github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-instruct-online-endpoint-chat-completions-inference.yml b/.github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-instruct-online-endpoint-chat-completions-inference.yml new file mode 100644 index 0000000000..5111fac064 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-instruct-online-endpoint-chat-completions-inference.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-instruct-online-endpoint-chat-completions-inference +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "54 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b-instruct/** + - .github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-instruct-online-endpoint-chat-completions-inference.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b-instruct" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/meta-llama-3/meta-llama-3-70b-instruct/online-endpoint-chat-completions-inference.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "online-endpoint-chat-completions-inference.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python online-endpoint-chat-completions-inference.ipynb online-endpoint-chat-completions-inference.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b-instruct + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: online-endpoint-chat-completions-inference + path: sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b-instruct diff --git a/.github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-online-endpoint-completions-inference.yml b/.github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-online-endpoint-completions-inference.yml new file mode 100644 index 0000000000..4d009a5814 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-online-endpoint-completions-inference.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-online-endpoint-completions-inference +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "42 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b/** + - .github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-online-endpoint-completions-inference.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/meta-llama-3/meta-llama-3-70b/online-endpoint-completions-inference.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "online-endpoint-completions-inference.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python online-endpoint-completions-inference.ipynb online-endpoint-completions-inference.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: online-endpoint-completions-inference + path: sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b diff --git a/.github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-instruct-online-endpoint-chat-completions-inference.yml b/.github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-instruct-online-endpoint-chat-completions-inference.yml new file mode 100644 index 0000000000..0d7a565ecd --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-instruct-online-endpoint-chat-completions-inference.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-instruct-online-endpoint-chat-completions-inference +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "54 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b-instruct/** + - .github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-instruct-online-endpoint-chat-completions-inference.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b-instruct" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/meta-llama-3/meta-llama-3-8b-instruct/online-endpoint-chat-completions-inference.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "online-endpoint-chat-completions-inference.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python online-endpoint-chat-completions-inference.ipynb online-endpoint-chat-completions-inference.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b-instruct + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: online-endpoint-chat-completions-inference + path: sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b-instruct diff --git a/.github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-online-endpoint-completions-inference.yml b/.github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-online-endpoint-completions-inference.yml new file mode 100644 index 0000000000..8421492a12 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-online-endpoint-completions-inference.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-online-endpoint-completions-inference +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "42 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b/** + - .github/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-online-endpoint-completions-inference.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/meta-llama-3/meta-llama-3-8b/online-endpoint-completions-inference.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "online-endpoint-completions-inference.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python online-endpoint-completions-inference.ipynb online-endpoint-completions-inference.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: online-endpoint-completions-inference + path: sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b diff --git a/.github/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-batch-endpoint.yml new file mode 100644 index 0000000000..d3de197da0 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-question-answering-question-answering-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "7 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/question-answering/** + - .github/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/question-answering" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/question-answering/question-answering-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "question-answering-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python question-answering-batch-endpoint.ipynb question-answering-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/question-answering + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: question-answering-batch-endpoint + path: sdk/python/foundation-models/system/inference/question-answering diff --git a/.github/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint-oss.yml b/.github/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint-oss.yml new file mode 100644 index 0000000000..fbbfecc400 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint-oss.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint-oss +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "13 9/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/question-answering/** + - .github/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint-oss.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/question-answering" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/question-answering/question-answering-online-endpoint-oss.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "question-answering-online-endpoint-oss.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python question-answering-online-endpoint-oss.ipynb question-answering-online-endpoint-oss.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/question-answering + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: question-answering-online-endpoint-oss + path: sdk/python/foundation-models/system/inference/question-answering diff --git a/.github/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint.yml new file mode 100644 index 0000000000..0cc03e279b --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "29 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/question-answering/** + - .github/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/question-answering" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/question-answering/question-answering-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "question-answering-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python question-answering-online-endpoint.ipynb question-answering-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/question-answering + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: question-answering-online-endpoint + path: sdk/python/foundation-models/system/inference/question-answering diff --git a/.github/workflows/sdk-foundation-models-system-inference-summarization-summarization-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-summarization-summarization-batch-endpoint.yml new file mode 100644 index 0000000000..b24c45d6fc --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-summarization-summarization-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-summarization-summarization-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "54 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/summarization/** + - .github/workflows/sdk-foundation-models-system-inference-summarization-summarization-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/summarization" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/summarization/summarization-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "summarization-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python summarization-batch-endpoint.ipynb summarization-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/summarization + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: summarization-batch-endpoint + path: sdk/python/foundation-models/system/inference/summarization diff --git a/.github/workflows/sdk-foundation-models-system-inference-summarization-summarization-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-summarization-summarization-online-endpoint.yml new file mode 100644 index 0000000000..cb7b5505e9 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-summarization-summarization-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-summarization-summarization-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "20 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/summarization/** + - .github/workflows/sdk-foundation-models-system-inference-summarization-summarization-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/summarization" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/summarization/summarization-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "summarization-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python summarization-online-endpoint.ipynb summarization-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/summarization + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: summarization-online-endpoint + path: sdk/python/foundation-models/system/inference/summarization diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-classification-entailment-contradiction-batch.yml b/.github/workflows/sdk-foundation-models-system-inference-text-classification-entailment-contradiction-batch.yml new file mode 100644 index 0000000000..9a7cac9557 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-classification-entailment-contradiction-batch.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-classification-entailment-contradiction-batch +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "25 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-classification/** + - .github/workflows/sdk-foundation-models-system-inference-text-classification-entailment-contradiction-batch.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-classification/entailment-contradiction-batch.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "entailment-contradiction-batch.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python entailment-contradiction-batch.ipynb entailment-contradiction-batch.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: entailment-contradiction-batch + path: sdk/python/foundation-models/system/inference/text-classification diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-classification-text-classification-online-endpoint-oss.yml b/.github/workflows/sdk-foundation-models-system-inference-text-classification-text-classification-online-endpoint-oss.yml new file mode 100644 index 0000000000..77e31a3adb --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-classification-text-classification-online-endpoint-oss.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-classification-text-classification-online-endpoint-oss +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "54 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-classification/** + - .github/workflows/sdk-foundation-models-system-inference-text-classification-text-classification-online-endpoint-oss.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-classification/text-classification-online-endpoint-oss.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text-classification-online-endpoint-oss.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text-classification-online-endpoint-oss.ipynb text-classification-online-endpoint-oss.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text-classification-online-endpoint-oss + path: sdk/python/foundation-models/system/inference/text-classification diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-classification-text-classification-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-text-classification-text-classification-online-endpoint.yml new file mode 100644 index 0000000000..40a373a215 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-classification-text-classification-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-classification-text-classification-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "53 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-classification/** + - .github/workflows/sdk-foundation-models-system-inference-text-classification-text-classification-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-classification/text-classification-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text-classification-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text-classification-online-endpoint.ipynb text-classification-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text-classification-online-endpoint + path: sdk/python/foundation-models/system/inference/text-classification diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-generation-aacs-prepare-uai.yml b/.github/workflows/sdk-foundation-models-system-inference-text-generation-aacs-prepare-uai.yml new file mode 100644 index 0000000000..99e8bb28a9 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-generation-aacs-prepare-uai.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-generation-aacs-prepare-uai +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "35 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-generation/** + - .github/workflows/sdk-foundation-models-system-inference-text-generation-aacs-prepare-uai.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-generation/aacs-prepare-uai.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "aacs-prepare-uai.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python aacs-prepare-uai.ipynb aacs-prepare-uai.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: aacs-prepare-uai + path: sdk/python/foundation-models/system/inference/text-generation diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-generation-falcon-safe-online-deployment.yml b/.github/workflows/sdk-foundation-models-system-inference-text-generation-falcon-safe-online-deployment.yml new file mode 100644 index 0000000000..5079d85c90 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-generation-falcon-safe-online-deployment.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-generation-falcon-safe-online-deployment +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "54 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-generation/** + - .github/workflows/sdk-foundation-models-system-inference-text-generation-falcon-safe-online-deployment.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-generation/falcon-safe-online-deployment.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "falcon-safe-online-deployment.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python falcon-safe-online-deployment.ipynb falcon-safe-online-deployment.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: falcon-safe-online-deployment + path: sdk/python/foundation-models/system/inference/text-generation diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-generation-llama-safe-batch-deployment.yml b/.github/workflows/sdk-foundation-models-system-inference-text-generation-llama-safe-batch-deployment.yml new file mode 100644 index 0000000000..91238f6248 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-generation-llama-safe-batch-deployment.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-generation-llama-safe-batch-deployment +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "40 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-generation/** + - .github/workflows/sdk-foundation-models-system-inference-text-generation-llama-safe-batch-deployment.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-generation/llama-safe-batch-deployment.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "llama-safe-batch-deployment.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python llama-safe-batch-deployment.ipynb llama-safe-batch-deployment.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: llama-safe-batch-deployment + path: sdk/python/foundation-models/system/inference/text-generation diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-generation-llama-safe-online-deployment.yml b/.github/workflows/sdk-foundation-models-system-inference-text-generation-llama-safe-online-deployment.yml new file mode 100644 index 0000000000..707f546c3c --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-generation-llama-safe-online-deployment.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-generation-llama-safe-online-deployment +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "12 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-generation/** + - .github/workflows/sdk-foundation-models-system-inference-text-generation-llama-safe-online-deployment.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-generation/llama-safe-online-deployment.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "llama-safe-online-deployment.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python llama-safe-online-deployment.ipynb llama-safe-online-deployment.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: llama-safe-online-deployment + path: sdk/python/foundation-models/system/inference/text-generation diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-batch-endpoint.yml new file mode 100644 index 0000000000..ed3b0ea7b3 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-generation-text-generation-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "55 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-generation/** + - .github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-generation/text-generation-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text-generation-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text-generation-batch-endpoint.ipynb text-generation-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text-generation-batch-endpoint + path: sdk/python/foundation-models/system/inference/text-generation diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint-dolly.yml b/.github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint-dolly.yml new file mode 100644 index 0000000000..53beba0f8e --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint-dolly.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint-dolly +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "26 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-generation/** + - .github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint-dolly.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-generation/text-generation-online-endpoint-dolly.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text-generation-online-endpoint-dolly.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text-generation-online-endpoint-dolly.ipynb text-generation-online-endpoint-dolly.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text-generation-online-endpoint-dolly + path: sdk/python/foundation-models/system/inference/text-generation diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint-oss.yml b/.github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint-oss.yml new file mode 100644 index 0000000000..7dd85541df --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint-oss.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint-oss +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "15 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-generation/** + - .github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint-oss.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-generation/text-generation-online-endpoint-oss.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text-generation-online-endpoint-oss.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text-generation-online-endpoint-oss.ipynb text-generation-online-endpoint-oss.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text-generation-online-endpoint-oss + path: sdk/python/foundation-models/system/inference/text-generation diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint.yml new file mode 100644 index 0000000000..4bf9fc9125 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "38 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-generation/** + - .github/workflows/sdk-foundation-models-system-inference-text-generation-text-generation-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-generation/text-generation-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text-generation-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text-generation-online-endpoint.ipynb text-generation-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text-generation-online-endpoint + path: sdk/python/foundation-models/system/inference/text-generation diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-to-image-aacs-prepare-uai.yml b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-aacs-prepare-uai.yml new file mode 100644 index 0000000000..f8383bdff7 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-aacs-prepare-uai.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-to-image-aacs-prepare-uai +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "35 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-to-image/** + - .github/workflows/sdk-foundation-models-system-inference-text-to-image-aacs-prepare-uai.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-to-image/aacs-prepare-uai.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "aacs-prepare-uai.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python aacs-prepare-uai.ipynb aacs-prepare-uai.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: aacs-prepare-uai + path: sdk/python/foundation-models/system/inference/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-to-image-image-text-to-image-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-image-text-to-image-batch-endpoint.yml new file mode 100644 index 0000000000..1466abda16 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-image-text-to-image-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-to-image-image-text-to-image-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "17 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-to-image/** + - .github/workflows/sdk-foundation-models-system-inference-text-to-image-image-text-to-image-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-to-image/image-text-to-image-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-text-to-image-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-text-to-image-batch-endpoint.ipynb image-text-to-image-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-text-to-image-batch-endpoint + path: sdk/python/foundation-models/system/inference/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-to-image-image-text-to-image-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-image-text-to-image-online-endpoint.yml new file mode 100644 index 0000000000..65408146f5 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-image-text-to-image-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-to-image-image-text-to-image-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "33 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-to-image/** + - .github/workflows/sdk-foundation-models-system-inference-text-to-image-image-text-to-image-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-to-image/image-text-to-image-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-text-to-image-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-text-to-image-online-endpoint.ipynb image-text-to-image-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-text-to-image-online-endpoint + path: sdk/python/foundation-models/system/inference/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-image-text-to-image-batch-deployment.yml b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-image-text-to-image-batch-deployment.yml new file mode 100644 index 0000000000..9557b70349 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-image-text-to-image-batch-deployment.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-to-image-safe-image-text-to-image-batch-deployment +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "15 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-to-image/** + - .github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-image-text-to-image-batch-deployment.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-to-image/safe-image-text-to-image-batch-deployment.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "safe-image-text-to-image-batch-deployment.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python safe-image-text-to-image-batch-deployment.ipynb safe-image-text-to-image-batch-deployment.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: safe-image-text-to-image-batch-deployment + path: sdk/python/foundation-models/system/inference/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-image-text-to-image-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-image-text-to-image-online-endpoint.yml new file mode 100644 index 0000000000..3bdc2a8022 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-image-text-to-image-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-to-image-safe-image-text-to-image-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "20 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-to-image/** + - .github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-image-text-to-image-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-to-image/safe-image-text-to-image-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "safe-image-text-to-image-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python safe-image-text-to-image-online-endpoint.ipynb safe-image-text-to-image-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: safe-image-text-to-image-online-endpoint + path: sdk/python/foundation-models/system/inference/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-batch-deployment.yml b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-batch-deployment.yml new file mode 100644 index 0000000000..bed95b6964 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-batch-deployment.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-batch-deployment +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "2 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-to-image/** + - .github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-batch-deployment.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-to-image/safe-text-to-image-batch-deployment.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "safe-text-to-image-batch-deployment.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python safe-text-to-image-batch-deployment.ipynb safe-text-to-image-batch-deployment.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: safe-text-to-image-batch-deployment + path: sdk/python/foundation-models/system/inference/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-inpainting-batch-deployment.yml b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-inpainting-batch-deployment.yml new file mode 100644 index 0000000000..3aea6d9b2f --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-inpainting-batch-deployment.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-inpainting-batch-deployment +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "47 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-to-image/** + - .github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-inpainting-batch-deployment.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-to-image/safe-text-to-image-inpainting-batch-deployment.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "safe-text-to-image-inpainting-batch-deployment.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python safe-text-to-image-inpainting-batch-deployment.ipynb safe-text-to-image-inpainting-batch-deployment.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: safe-text-to-image-inpainting-batch-deployment + path: sdk/python/foundation-models/system/inference/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-inpainting-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-inpainting-online-endpoint.yml new file mode 100644 index 0000000000..9282bf293c --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-inpainting-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-inpainting-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "29 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-to-image/** + - .github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-inpainting-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-to-image/safe-text-to-image-inpainting-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "safe-text-to-image-inpainting-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python safe-text-to-image-inpainting-online-endpoint.ipynb safe-text-to-image-inpainting-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: safe-text-to-image-inpainting-online-endpoint + path: sdk/python/foundation-models/system/inference/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-online-deployment.yml b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-online-deployment.yml new file mode 100644 index 0000000000..f76fd4d64a --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-online-deployment.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-online-deployment +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "52 9/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-to-image/** + - .github/workflows/sdk-foundation-models-system-inference-text-to-image-safe-text-to-image-online-deployment.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-to-image/safe-text-to-image-online-deployment.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "safe-text-to-image-online-deployment.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python safe-text-to-image-online-deployment.ipynb safe-text-to-image-online-deployment.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: safe-text-to-image-online-deployment + path: sdk/python/foundation-models/system/inference/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-batch-endpoint.yml new file mode 100644 index 0000000000..9d0f06dd43 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-to-image-text-to-image-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "20 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-to-image/** + - .github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-to-image/text-to-image-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text-to-image-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text-to-image-batch-endpoint.ipynb text-to-image-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text-to-image-batch-endpoint + path: sdk/python/foundation-models/system/inference/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-inpainting-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-inpainting-batch-endpoint.yml new file mode 100644 index 0000000000..d80bc2d1f1 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-inpainting-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-to-image-text-to-image-inpainting-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "1 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-to-image/** + - .github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-inpainting-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-to-image/text-to-image-inpainting-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text-to-image-inpainting-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text-to-image-inpainting-batch-endpoint.ipynb text-to-image-inpainting-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text-to-image-inpainting-batch-endpoint + path: sdk/python/foundation-models/system/inference/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-inpainting-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-inpainting-online-endpoint.yml new file mode 100644 index 0000000000..df27e94c57 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-inpainting-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-to-image-text-to-image-inpainting-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "24 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-to-image/** + - .github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-inpainting-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-to-image/text-to-image-inpainting-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text-to-image-inpainting-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text-to-image-inpainting-online-endpoint.ipynb text-to-image-inpainting-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text-to-image-inpainting-online-endpoint + path: sdk/python/foundation-models/system/inference/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-online-endpoint.yml new file mode 100644 index 0000000000..151842fbfd --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-text-to-image-text-to-image-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "29 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/text-to-image/** + - .github/workflows/sdk-foundation-models-system-inference-text-to-image-text-to-image-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/text-to-image" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/text-to-image/text-to-image-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text-to-image-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text-to-image-online-endpoint.ipynb text-to-image-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/text-to-image + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text-to-image-online-endpoint + path: sdk/python/foundation-models/system/inference/text-to-image diff --git a/.github/workflows/sdk-foundation-models-system-inference-token-classification-token-classification-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-token-classification-token-classification-batch-endpoint.yml new file mode 100644 index 0000000000..376501ad46 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-token-classification-token-classification-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-token-classification-token-classification-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "15 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/token-classification/** + - .github/workflows/sdk-foundation-models-system-inference-token-classification-token-classification-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/token-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/token-classification/token-classification-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "token-classification-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python token-classification-batch-endpoint.ipynb token-classification-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/token-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: token-classification-batch-endpoint + path: sdk/python/foundation-models/system/inference/token-classification diff --git a/.github/workflows/sdk-foundation-models-system-inference-token-classification-token-classification-online-endpoint-oss.yml b/.github/workflows/sdk-foundation-models-system-inference-token-classification-token-classification-online-endpoint-oss.yml new file mode 100644 index 0000000000..5031abcc3e --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-token-classification-token-classification-online-endpoint-oss.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-token-classification-token-classification-online-endpoint-oss +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "17 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/token-classification/** + - .github/workflows/sdk-foundation-models-system-inference-token-classification-token-classification-online-endpoint-oss.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/token-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/token-classification/token-classification-online-endpoint-oss.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "token-classification-online-endpoint-oss.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python token-classification-online-endpoint-oss.ipynb token-classification-online-endpoint-oss.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/token-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: token-classification-online-endpoint-oss + path: sdk/python/foundation-models/system/inference/token-classification diff --git a/.github/workflows/sdk-foundation-models-system-inference-token-classification-token-classification-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-token-classification-token-classification-online-endpoint.yml new file mode 100644 index 0000000000..6e423f860a --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-token-classification-token-classification-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-token-classification-token-classification-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "17 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/token-classification/** + - .github/workflows/sdk-foundation-models-system-inference-token-classification-token-classification-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/token-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/token-classification/token-classification-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "token-classification-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python token-classification-online-endpoint.ipynb token-classification-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/token-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: token-classification-online-endpoint + path: sdk/python/foundation-models/system/inference/token-classification diff --git a/.github/workflows/sdk-foundation-models-system-inference-translation-translation-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-translation-translation-batch-endpoint.yml new file mode 100644 index 0000000000..1fc7291bda --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-translation-translation-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-translation-translation-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "53 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/translation/** + - .github/workflows/sdk-foundation-models-system-inference-translation-translation-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/translation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/translation/translation-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "translation-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python translation-batch-endpoint.ipynb translation-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/translation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: translation-batch-endpoint + path: sdk/python/foundation-models/system/inference/translation diff --git a/.github/workflows/sdk-foundation-models-system-inference-translation-translation-online-endpoint-oss.yml b/.github/workflows/sdk-foundation-models-system-inference-translation-translation-online-endpoint-oss.yml new file mode 100644 index 0000000000..950c0868c3 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-translation-translation-online-endpoint-oss.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-translation-translation-online-endpoint-oss +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "36 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/translation/** + - .github/workflows/sdk-foundation-models-system-inference-translation-translation-online-endpoint-oss.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/translation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/translation/translation-online-endpoint-oss.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "translation-online-endpoint-oss.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python translation-online-endpoint-oss.ipynb translation-online-endpoint-oss.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/translation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: translation-online-endpoint-oss + path: sdk/python/foundation-models/system/inference/translation diff --git a/.github/workflows/sdk-foundation-models-system-inference-translation-translation-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-translation-translation-online-endpoint.yml new file mode 100644 index 0000000000..49b49d5e9c --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-translation-translation-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-translation-translation-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/translation/** + - .github/workflows/sdk-foundation-models-system-inference-translation-translation-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/translation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/translation/translation-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "translation-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python translation-online-endpoint.ipynb translation-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/translation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: translation-online-endpoint + path: sdk/python/foundation-models/system/inference/translation diff --git a/.github/workflows/sdk-foundation-models-system-inference-video-multi-object-tracking-video-multi-object-tracking-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-video-multi-object-tracking-video-multi-object-tracking-online-endpoint.yml new file mode 100644 index 0000000000..2c0be51caf --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-video-multi-object-tracking-video-multi-object-tracking-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-video-multi-object-tracking-video-multi-object-tracking-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "6 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/video-multi-object-tracking/** + - .github/workflows/sdk-foundation-models-system-inference-video-multi-object-tracking-video-multi-object-tracking-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/video-multi-object-tracking" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/video-multi-object-tracking/video-multi-object-tracking-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "video-multi-object-tracking-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python video-multi-object-tracking-online-endpoint.ipynb video-multi-object-tracking-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/video-multi-object-tracking + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: video-multi-object-tracking-online-endpoint + path: sdk/python/foundation-models/system/inference/video-multi-object-tracking diff --git a/.github/workflows/sdk-foundation-models-system-inference-visual-chat-completion-image-text-to-text-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-visual-chat-completion-image-text-to-text-online-endpoint.yml new file mode 100644 index 0000000000..12df13e327 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-visual-chat-completion-image-text-to-text-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-visual-chat-completion-image-text-to-text-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "7 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/visual-chat-completion/** + - .github/workflows/sdk-foundation-models-system-inference-visual-chat-completion-image-text-to-text-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/visual-chat-completion" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/visual-chat-completion/image-text-to-text-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "image-text-to-text-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python image-text-to-text-online-endpoint.ipynb image-text-to-text-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/visual-chat-completion + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: image-text-to-text-online-endpoint + path: sdk/python/foundation-models/system/inference/visual-chat-completion diff --git a/.github/workflows/sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-batch-endpoint.yml new file mode 100644 index 0000000000..5f8240888d --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "59 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/visual-question-answering/** + - .github/workflows/sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/visual-question-answering" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/visual-question-answering/visual-question-answering-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "visual-question-answering-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python visual-question-answering-batch-endpoint.ipynb visual-question-answering-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/visual-question-answering + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: visual-question-answering-batch-endpoint + path: sdk/python/foundation-models/system/inference/visual-question-answering diff --git a/.github/workflows/sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-online-endpoint.yml new file mode 100644 index 0000000000..e4a1806b7c --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "11 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/visual-question-answering/** + - .github/workflows/sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/visual-question-answering" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/visual-question-answering/visual-question-answering-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "visual-question-answering-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python visual-question-answering-online-endpoint.ipynb visual-question-answering-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/visual-question-answering + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: visual-question-answering-online-endpoint + path: sdk/python/foundation-models/system/inference/visual-question-answering diff --git a/.github/workflows/sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-batch-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-batch-endpoint.yml new file mode 100644 index 0000000000..fbc5646ff7 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-batch-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-batch-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "36 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/zero-shot-image-classification/** + - .github/workflows/sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-batch-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/zero-shot-image-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/zero-shot-image-classification/zero-shot-image-classification-batch-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "zero-shot-image-classification-batch-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python zero-shot-image-classification-batch-endpoint.ipynb zero-shot-image-classification-batch-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/zero-shot-image-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: zero-shot-image-classification-batch-endpoint + path: sdk/python/foundation-models/system/inference/zero-shot-image-classification diff --git a/.github/workflows/sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-online-endpoint.yml b/.github/workflows/sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-online-endpoint.yml new file mode 100644 index 0000000000..79f9a0e055 --- /dev/null +++ b/.github/workflows/sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-online-endpoint.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-online-endpoint +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "7 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/foundation-models/system/inference/zero-shot-image-classification/** + - .github/workflows/sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-online-endpoint.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/foundation-models/system/inference/zero-shot-image-classification" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run foundation-models/system/inference/zero-shot-image-classification/zero-shot-image-classification-online-endpoint.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "zero-shot-image-classification-online-endpoint.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python zero-shot-image-classification-online-endpoint.ipynb zero-shot-image-classification-online-endpoint.output.ipynb + working-directory: sdk/python/foundation-models/system/inference/zero-shot-image-classification + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: zero-shot-image-classification-online-endpoint + path: sdk/python/foundation-models/system/inference/zero-shot-image-classification diff --git a/.github/workflows/sdk-generative-ai-promptflow-create_faiss_index.yml b/.github/workflows/sdk-generative-ai-promptflow-create_faiss_index.yml new file mode 100644 index 0000000000..ce58b1b20e --- /dev/null +++ b/.github/workflows/sdk-generative-ai-promptflow-create_faiss_index.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-promptflow-create_faiss_index +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "34 9/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/promptflow/** + - .github/workflows/sdk-generative-ai-promptflow-create_faiss_index.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/promptflow" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/promptflow/create_faiss_index.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "create_faiss_index.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python create_faiss_index.ipynb create_faiss_index.output.ipynb + working-directory: sdk/python/generative-ai/promptflow + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: create_faiss_index + path: sdk/python/generative-ai/promptflow diff --git a/.github/workflows/sdk-generative-ai-promptflow-deploy-flow-deploy-flow.yml b/.github/workflows/sdk-generative-ai-promptflow-deploy-flow-deploy-flow.yml new file mode 100644 index 0000000000..3414401d8f --- /dev/null +++ b/.github/workflows/sdk-generative-ai-promptflow-deploy-flow-deploy-flow.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-promptflow-deploy-flow-deploy-flow +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "14 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/promptflow/deploy-flow/** + - .github/workflows/sdk-generative-ai-promptflow-deploy-flow-deploy-flow.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/promptflow/deploy-flow" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/promptflow/deploy-flow/deploy-flow.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "deploy-flow.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python deploy-flow.ipynb deploy-flow.output.ipynb + working-directory: sdk/python/generative-ai/promptflow/deploy-flow + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: deploy-flow + path: sdk/python/generative-ai/promptflow/deploy-flow diff --git a/.github/workflows/sdk-generative-ai-rag-code_first-flows-chat-with-index-src-test.yml b/.github/workflows/sdk-generative-ai-rag-code_first-flows-chat-with-index-src-test.yml new file mode 100644 index 0000000000..d684bbfc2d --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-code_first-flows-chat-with-index-src-test.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-code_first-flows-chat-with-index-src-test +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "23 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/code_first/flows/chat-with-index/src/** + - .github/workflows/sdk-generative-ai-rag-code_first-flows-chat-with-index-src-test.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/code_first/flows/chat-with-index/src" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/code_first/flows/chat-with-index/src/test.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "test.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python test.ipynb test.output.ipynb + working-directory: sdk/python/generative-ai/rag/code_first/flows/chat-with-index/src + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: test + path: sdk/python/generative-ai/rag/code_first/flows/chat-with-index/src diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-azure_cognitive_search-acs_mlindex_with_langchain.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-azure_cognitive_search-acs_mlindex_with_langchain.yml new file mode 100644 index 0000000000..3f3c4a9eaa --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-azure_cognitive_search-acs_mlindex_with_langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-azure_cognitive_search-acs_mlindex_with_langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "6 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/azure_cognitive_search/** + - .github/workflows/sdk-generative-ai-rag-notebooks-azure_cognitive_search-acs_mlindex_with_langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks/azure_cognitive_search" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/azure_cognitive_search/acs_mlindex_with_langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "acs_mlindex_with_langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python acs_mlindex_with_langchain.ipynb acs_mlindex_with_langchain.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks/azure_cognitive_search + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: acs_mlindex_with_langchain + path: sdk/python/generative-ai/rag/notebooks/azure_cognitive_search diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-azure_cognitive_search-s3_to_acs_mlindex_with_langchain.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-azure_cognitive_search-s3_to_acs_mlindex_with_langchain.yml new file mode 100644 index 0000000000..d369bf5e93 --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-azure_cognitive_search-s3_to_acs_mlindex_with_langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-azure_cognitive_search-s3_to_acs_mlindex_with_langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "41 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/azure_cognitive_search/** + - .github/workflows/sdk-generative-ai-rag-notebooks-azure_cognitive_search-s3_to_acs_mlindex_with_langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks/azure_cognitive_search" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/azure_cognitive_search/s3_to_acs_mlindex_with_langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "s3_to_acs_mlindex_with_langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python s3_to_acs_mlindex_with_langchain.ipynb s3_to_acs_mlindex_with_langchain.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks/azure_cognitive_search + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: s3_to_acs_mlindex_with_langchain + path: sdk/python/generative-ai/rag/notebooks/azure_cognitive_search diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk-crack_pdfs_with_azure_document_intelligence.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk-crack_pdfs_with_azure_document_intelligence.yml new file mode 100644 index 0000000000..3922bcde9b --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk-crack_pdfs_with_azure_document_intelligence.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-custom_crack_and_chunk-crack_pdfs_with_azure_document_intelligence +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "9 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk/** + - .github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk-crack_pdfs_with_azure_document_intelligence.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/custom_crack_and_chunk/crack_pdfs_with_azure_document_intelligence.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "crack_pdfs_with_azure_document_intelligence.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python crack_pdfs_with_azure_document_intelligence.ipynb crack_pdfs_with_azure_document_intelligence.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: crack_pdfs_with_azure_document_intelligence + path: sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-crack_and_chunk_with_doc_intel_parallel_component.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-crack_and_chunk_with_doc_intel_parallel_component.yml new file mode 100644 index 0000000000..dc53237677 --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-crack_and_chunk_with_doc_intel_parallel_component.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-crack_and_chunk_with_doc_intel_parallel_component +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "7 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/** + - .github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-crack_and_chunk_with_doc_intel_parallel_component.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/crack_and_chunk_with_doc_intel_parallel_component.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "crack_and_chunk_with_doc_intel_parallel_component.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python crack_and_chunk_with_doc_intel_parallel_component.ipynb crack_and_chunk_with_doc_intel_parallel_component.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: crack_and_chunk_with_doc_intel_parallel_component + path: sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-d_parallel_crack_pdfs_with_azure_document_intelligence.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-d_parallel_crack_pdfs_with_azure_document_intelligence.yml new file mode 100644 index 0000000000..2901564f2c --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-d_parallel_crack_pdfs_with_azure_document_intelligence.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-d_parallel_crack_pdfs_with_azure_document_intelligence +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "4 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/** + - .github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-d_parallel_crack_pdfs_with_azure_document_intelligence.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/d_parallel_crack_pdfs_with_azure_document_intelligence.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "d_parallel_crack_pdfs_with_azure_document_intelligence.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python d_parallel_crack_pdfs_with_azure_document_intelligence.ipynb d_parallel_crack_pdfs_with_azure_document_intelligence.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: d_parallel_crack_pdfs_with_azure_document_intelligence + path: sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-e_parallel_crack_pdfs_with_azure_document_intelligence.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-e_parallel_crack_pdfs_with_azure_document_intelligence.yml new file mode 100644 index 0000000000..57c927e40f --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-e_parallel_crack_pdfs_with_azure_document_intelligence.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-e_parallel_crack_pdfs_with_azure_document_intelligence +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "51 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/** + - .github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-e_parallel_crack_pdfs_with_azure_document_intelligence.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/e_parallel_crack_pdfs_with_azure_document_intelligence.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "e_parallel_crack_pdfs_with_azure_document_intelligence.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python e_parallel_crack_pdfs_with_azure_document_intelligence.ipynb e_parallel_crack_pdfs_with_azure_document_intelligence.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: e_parallel_crack_pdfs_with_azure_document_intelligence + path: sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-f_parallel_crack_pdfs_with_azure_document_intelligence.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-f_parallel_crack_pdfs_with_azure_document_intelligence.yml new file mode 100644 index 0000000000..4ed7cfec08 --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-f_parallel_crack_pdfs_with_azure_document_intelligence.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-f_parallel_crack_pdfs_with_azure_document_intelligence +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "44 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/** + - .github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-f_parallel_crack_pdfs_with_azure_document_intelligence.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/f_parallel_crack_pdfs_with_azure_document_intelligence.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "f_parallel_crack_pdfs_with_azure_document_intelligence.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python f_parallel_crack_pdfs_with_azure_document_intelligence.ipynb f_parallel_crack_pdfs_with_azure_document_intelligence.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: f_parallel_crack_pdfs_with_azure_document_intelligence + path: sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-local_parallel_crack_pdfs_with_azure_document_intelligence.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-local_parallel_crack_pdfs_with_azure_document_intelligence.yml new file mode 100644 index 0000000000..2435f31f31 --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-local_parallel_crack_pdfs_with_azure_document_intelligence.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-local_parallel_crack_pdfs_with_azure_document_intelligence +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "50 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/** + - .github/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-local_parallel_crack_pdfs_with_azure_document_intelligence.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/local_parallel_crack_pdfs_with_azure_document_intelligence.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "local_parallel_crack_pdfs_with_azure_document_intelligence.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python local_parallel_crack_pdfs_with_azure_document_intelligence.ipynb local_parallel_crack_pdfs_with_azure_document_intelligence.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: local_parallel_crack_pdfs_with_azure_document_intelligence + path: sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-db_copilot_with_rag.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-db_copilot_with_rag.yml new file mode 100644 index 0000000000..3591850bf5 --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-db_copilot_with_rag.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-db_copilot_with_rag +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "57 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/** + - .github/workflows/sdk-generative-ai-rag-notebooks-db_copilot_with_rag.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/db_copilot_with_rag.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "db_copilot_with_rag.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python db_copilot_with_rag.ipynb db_copilot_with_rag.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: db_copilot_with_rag + path: sdk/python/generative-ai/rag/notebooks diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-faiss-faiss_mlindex_with_langchain.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-faiss-faiss_mlindex_with_langchain.yml new file mode 100644 index 0000000000..9951f1d424 --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-faiss-faiss_mlindex_with_langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-faiss-faiss_mlindex_with_langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "2 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/faiss/** + - .github/workflows/sdk-generative-ai-rag-notebooks-faiss-faiss_mlindex_with_langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks/faiss" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/faiss/faiss_mlindex_with_langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "faiss_mlindex_with_langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python faiss_mlindex_with_langchain.ipynb faiss_mlindex_with_langchain.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks/faiss + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: faiss_mlindex_with_langchain + path: sdk/python/generative-ai/rag/notebooks/faiss diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-faiss-scheduled_update_faiss_index.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-faiss-scheduled_update_faiss_index.yml new file mode 100644 index 0000000000..dc0df6f229 --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-faiss-scheduled_update_faiss_index.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-faiss-scheduled_update_faiss_index +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "25 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/faiss/** + - .github/workflows/sdk-generative-ai-rag-notebooks-faiss-scheduled_update_faiss_index.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks/faiss" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/faiss/scheduled_update_faiss_index.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "scheduled_update_faiss_index.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python scheduled_update_faiss_index.ipynb scheduled_update_faiss_index.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks/faiss + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: scheduled_update_faiss_index + path: sdk/python/generative-ai/rag/notebooks/faiss diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-faiss-url_to_faiss_incremental_embeddings_with_tabular_data.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-faiss-url_to_faiss_incremental_embeddings_with_tabular_data.yml new file mode 100644 index 0000000000..2065df1720 --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-faiss-url_to_faiss_incremental_embeddings_with_tabular_data.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-faiss-url_to_faiss_incremental_embeddings_with_tabular_data +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "29 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/faiss/** + - .github/workflows/sdk-generative-ai-rag-notebooks-faiss-url_to_faiss_incremental_embeddings_with_tabular_data.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks/faiss" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/faiss/url_to_faiss_incremental_embeddings_with_tabular_data.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "url_to_faiss_incremental_embeddings_with_tabular_data.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python url_to_faiss_incremental_embeddings_with_tabular_data.ipynb url_to_faiss_incremental_embeddings_with_tabular_data.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks/faiss + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: url_to_faiss_incremental_embeddings_with_tabular_data + path: sdk/python/generative-ai/rag/notebooks/faiss diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-mlindex_with_testgen.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-mlindex_with_testgen.yml new file mode 100644 index 0000000000..3fbade2b97 --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-mlindex_with_testgen.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-mlindex_with_testgen +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "37 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/** + - .github/workflows/sdk-generative-ai-rag-notebooks-mlindex_with_testgen.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/mlindex_with_testgen.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "mlindex_with_testgen.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python mlindex_with_testgen.ipynb mlindex_with_testgen.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: mlindex_with_testgen + path: sdk/python/generative-ai/rag/notebooks diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-mlindex_with_testgen_autoprompt.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-mlindex_with_testgen_autoprompt.yml new file mode 100644 index 0000000000..40a71c35f3 --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-mlindex_with_testgen_autoprompt.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-mlindex_with_testgen_autoprompt +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "28 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/** + - .github/workflows/sdk-generative-ai-rag-notebooks-mlindex_with_testgen_autoprompt.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/mlindex_with_testgen_autoprompt.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "mlindex_with_testgen_autoprompt.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python mlindex_with_testgen_autoprompt.ipynb mlindex_with_testgen_autoprompt.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: mlindex_with_testgen_autoprompt + path: sdk/python/generative-ai/rag/notebooks diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-pinecone-pinecone_mlindex_with_langchain.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-pinecone-pinecone_mlindex_with_langchain.yml new file mode 100644 index 0000000000..92955f07fe --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-pinecone-pinecone_mlindex_with_langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-pinecone-pinecone_mlindex_with_langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "50 3/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/pinecone/** + - .github/workflows/sdk-generative-ai-rag-notebooks-pinecone-pinecone_mlindex_with_langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks/pinecone" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/pinecone/pinecone_mlindex_with_langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "pinecone_mlindex_with_langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python pinecone_mlindex_with_langchain.ipynb pinecone_mlindex_with_langchain.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks/pinecone + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: pinecone_mlindex_with_langchain + path: sdk/python/generative-ai/rag/notebooks/pinecone diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-pinecone-s3_to_pinecone_mlindex_with_langchain.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-pinecone-s3_to_pinecone_mlindex_with_langchain.yml new file mode 100644 index 0000000000..818a97ee5a --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-pinecone-s3_to_pinecone_mlindex_with_langchain.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-pinecone-s3_to_pinecone_mlindex_with_langchain +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "32 8/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/pinecone/** + - .github/workflows/sdk-generative-ai-rag-notebooks-pinecone-s3_to_pinecone_mlindex_with_langchain.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks/pinecone" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/pinecone/s3_to_pinecone_mlindex_with_langchain.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "s3_to_pinecone_mlindex_with_langchain.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python s3_to_pinecone_mlindex_with_langchain.ipynb s3_to_pinecone_mlindex_with_langchain.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks/pinecone + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: s3_to_pinecone_mlindex_with_langchain + path: sdk/python/generative-ai/rag/notebooks/pinecone diff --git a/.github/workflows/sdk-generative-ai-rag-notebooks-qa_data_generation.yml b/.github/workflows/sdk-generative-ai-rag-notebooks-qa_data_generation.yml new file mode 100644 index 0000000000..581975dae9 --- /dev/null +++ b/.github/workflows/sdk-generative-ai-rag-notebooks-qa_data_generation.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-generative-ai-rag-notebooks-qa_data_generation +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "32 1/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/generative-ai/rag/notebooks/** + - .github/workflows/sdk-generative-ai-rag-notebooks-qa_data_generation.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/generative-ai/rag/notebooks" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run generative-ai/rag/notebooks/qa_data_generation.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "qa_data_generation.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python qa_data_generation.ipynb qa_data_generation.output.ipynb + working-directory: sdk/python/generative-ai/rag/notebooks + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: qa_data_generation + path: sdk/python/generative-ai/rag/notebooks diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing-serverless.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing-serverless.yml new file mode 100644 index 0000000000..c4513b4cab --- /dev/null +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing-serverless.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing-serverless +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "45 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/automl-standalone-jobs/automl-classification-task-bankmarketing/** + - .github/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing-serverless.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/automl-standalone-jobs/automl-classification-task-bankmarketing" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/automl-standalone-jobs/automl-classification-task-bankmarketing/automl-classification-task-bankmarketing-serverless.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "automl-classification-task-bankmarketing-serverless.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python -p compute_name automl-cpu-cluster automl-classification-task-bankmarketing-serverless.ipynb automl-classification-task-bankmarketing-serverless.output.ipynb + working-directory: sdk/python/jobs/automl-standalone-jobs/automl-classification-task-bankmarketing + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: automl-classification-task-bankmarketing-serverless + path: sdk/python/jobs/automl-standalone-jobs/automl-classification-task-bankmarketing diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing.yml index e76ca96f62..9c4aa777b4 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-distributed-tcn-automl-forecasting-distributed-tcn.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-distributed-tcn-automl-forecasting-distributed-tcn.yml index fcea17f2ea..1d228b1190 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-distributed-tcn-automl-forecasting-distributed-tcn.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-distributed-tcn-automl-forecasting-distributed-tcn.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login @@ -70,14 +69,6 @@ jobs: source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; bash setup.sh working-directory: cli - - name: Eagerly cache access tokens for required scopes - run: | - # Workaround for azure-cli's lack of support for ID token refresh - # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 - # Management - az account get-access-token --scope https://management.azure.com/.default --output none - # ML - az account get-access-token --scope https://ml.azure.com/.default --output none continue-on-error: true - name: Eagerly cache access tokens for required scopes run: | diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-batch-inference.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-batch-inference.yml new file mode 100644 index 0000000000..cf8900863b --- /dev/null +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-batch-inference.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-batch-inference +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "54 6/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/** + - .github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-batch-inference.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-batch-inference.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "auto-ml-forecasting-function-gap-batch-inference.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python -p compute_name automl-cpu-cluster auto-ml-forecasting-function-gap-batch-inference.ipynb auto-ml-forecasting-function-gap-batch-inference.output.ipynb + working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: auto-ml-forecasting-function-gap-batch-inference + path: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-local-inference.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-local-inference.yml new file mode 100644 index 0000000000..0e466f4f48 --- /dev/null +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-local-inference.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-local-inference +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "41 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/** + - .github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-local-inference.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-local-inference.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "auto-ml-forecasting-function-gap-local-inference.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python -p compute_name automl-cpu-cluster auto-ml-forecasting-function-gap-local-inference.ipynb auto-ml-forecasting-function-gap-local-inference.output.ipynb + working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: auto-ml-forecasting-function-gap-local-inference + path: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-training.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-training.yml new file mode 100644 index 0000000000..59f4c32524 --- /dev/null +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-training.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-training +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "59 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/** + - .github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-training.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-training.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "auto-ml-forecasting-function-gap-training.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python -p compute_name automl-cpu-cluster auto-ml-forecasting-function-gap-training.ipynb auto-ml-forecasting-function-gap-training.output.ipynb + working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: auto-ml-forecasting-function-gap-training + path: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-github-dau-auto-ml-forecasting-github-dau.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-github-dau-auto-ml-forecasting-github-dau.yml index ce5196e089..c15e0e17f6 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-github-dau-auto-ml-forecasting-github-dau.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-github-dau-auto-ml-forecasting-github-dau.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: pip install forecasting reqs diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-orange-juice-sales-automl-forecasting-orange-juice-sales-mlflow.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-orange-juice-sales-automl-forecasting-orange-juice-sales-mlflow.yml index 7f8cda05d0..1ed6fb4899 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-orange-juice-sales-automl-forecasting-orange-juice-sales-mlflow.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-orange-juice-sales-automl-forecasting-orange-juice-sales-mlflow.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: pip install forecasting reqs diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-recipes-univariate-automl-forecasting-recipe-univariate-experiment-settings.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-recipes-univariate-automl-forecasting-recipe-univariate-experiment-settings.yml index 4a0920a99e..4581418a6a 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-recipes-univariate-automl-forecasting-recipe-univariate-experiment-settings.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-recipes-univariate-automl-forecasting-recipe-univariate-experiment-settings.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install forecasting reqs run: pip install -r sdk/python/forecasting-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-recipes-univariate-automl-forecasting-recipe-univariate-run.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-recipes-univariate-automl-forecasting-recipe-univariate-run.yml index 8bc97b1cd1..77f6ce0aea 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-recipes-univariate-automl-forecasting-recipe-univariate-run.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-recipes-univariate-automl-forecasting-recipe-univariate-run.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: pip install forecasting reqs diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-bike-share-auto-ml-forecasting-bike-share.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-bike-share-auto-ml-forecasting-bike-share.yml index 29252a6de0..03562a519e 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-bike-share-auto-ml-forecasting-bike-share.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-bike-share-auto-ml-forecasting-bike-share.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: pip install forecasting reqs diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-energy-demand-automl-forecasting-task-energy-demand-advanced-mlflow.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-energy-demand-automl-forecasting-task-energy-demand-advanced-mlflow.yml index 940ffcec94..d492b606c0 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-energy-demand-automl-forecasting-task-energy-demand-advanced-mlflow.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-energy-demand-automl-forecasting-task-energy-demand-advanced-mlflow.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: pip install forecasting reqs diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-energy-demand-automl-forecasting-task-energy-demand-advanced.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-energy-demand-automl-forecasting-task-energy-demand-advanced.yml index 20aa9f9f2a..a51673eb58 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-energy-demand-automl-forecasting-task-energy-demand-advanced.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-energy-demand-automl-forecasting-task-energy-demand-advanced.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-classification-multiclass-task-fridge-items-automl-image-classification-multiclass-task-fridge-items.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-classification-multiclass-task-fridge-items-automl-image-classification-multiclass-task-fridge-items.yml index 65a389a2b3..a413cda08a 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-classification-multiclass-task-fridge-items-automl-image-classification-multiclass-task-fridge-items.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-classification-multiclass-task-fridge-items-automl-image-classification-multiclass-task-fridge-items.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-classification-multilabel-task-fridge-items-automl-image-classification-multilabel-task-fridge-items.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-classification-multilabel-task-fridge-items-automl-image-classification-multilabel-task-fridge-items.yml index 0e1d9a8de7..ddb782a559 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-classification-multilabel-task-fridge-items-automl-image-classification-multilabel-task-fridge-items.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-classification-multilabel-task-fridge-items-automl-image-classification-multilabel-task-fridge-items.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-instance-segmentation-task-fridge-items-automl-image-instance-segmentation-task-fridge-items.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-instance-segmentation-task-fridge-items-automl-image-instance-segmentation-task-fridge-items.yml index 1202d4031d..0e8b3f9f69 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-instance-segmentation-task-fridge-items-automl-image-instance-segmentation-task-fridge-items.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-instance-segmentation-task-fridge-items-automl-image-instance-segmentation-task-fridge-items.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-instance-segmentation-task-fridge-items-odFridgeObjects_jsonl2coco.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-instance-segmentation-task-fridge-items-odFridgeObjects_jsonl2coco.yml new file mode 100644 index 0000000000..6cbcff905b --- /dev/null +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-instance-segmentation-task-fridge-items-odFridgeObjects_jsonl2coco.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-automl-standalone-jobs-automl-image-instance-segmentation-task-fridge-items-odFridgeObjects_jsonl2coco +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "17 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/automl-standalone-jobs/automl-image-instance-segmentation-task-fridge-items/** + - .github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-instance-segmentation-task-fridge-items-odFridgeObjects_jsonl2coco.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/automl-standalone-jobs/automl-image-instance-segmentation-task-fridge-items" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/automl-standalone-jobs/automl-image-instance-segmentation-task-fridge-items/odFridgeObjects_jsonl2coco.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "odFridgeObjects_jsonl2coco.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python -p compute_name automl-gpu-cluster odFridgeObjects_jsonl2coco.ipynb odFridgeObjects_jsonl2coco.output.ipynb + working-directory: sdk/python/jobs/automl-standalone-jobs/automl-image-instance-segmentation-task-fridge-items + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: odFridgeObjects_jsonl2coco + path: sdk/python/jobs/automl-standalone-jobs/automl-image-instance-segmentation-task-fridge-items diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-object-detection-task-fridge-items-automl-image-object-detection-task-fridge-items.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-object-detection-task-fridge-items-automl-image-object-detection-task-fridge-items.yml index ca9ea8235e..34976940d4 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-object-detection-task-fridge-items-automl-image-object-detection-task-fridge-items.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-object-detection-task-fridge-items-automl-image-object-detection-task-fridge-items.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login @@ -94,4 +93,4 @@ jobs: uses: ./.github/actions/upload-artifact with: name: automl-image-object-detection-task-fridge-items - path: sdk/python/jobs/automl-standalone-jobs/automl-image-object-detection-task-fridge-items \ No newline at end of file + path: sdk/python/jobs/automl-standalone-jobs/automl-image-object-detection-task-fridge-items diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-object-detection-task-fridge-items-batch-scoring-image-object-detection-batch-scoring-non-mlflow-model.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-object-detection-task-fridge-items-batch-scoring-image-object-detection-batch-scoring-non-mlflow-model.yml index a65f12ebc0..19bf2a1017 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-object-detection-task-fridge-items-batch-scoring-image-object-detection-batch-scoring-non-mlflow-model.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-image-object-detection-task-fridge-items-batch-scoring-image-object-detection-batch-scoring-non-mlflow-model.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login @@ -94,4 +93,4 @@ jobs: uses: ./.github/actions/upload-artifact with: name: image-object-detection-batch-scoring-non-mlflow-model - path: sdk/python/jobs/automl-standalone-jobs/automl-image-object-detection-task-fridge-items-batch-scoring \ No newline at end of file + path: sdk/python/jobs/automl-standalone-jobs/automl-image-object-detection-task-fridge-items-batch-scoring diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-classification-multiclass-task-sentiment-analysis-automl-nlp-multiclass-sentiment-mlflow.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-classification-multiclass-task-sentiment-analysis-automl-nlp-multiclass-sentiment-mlflow.yml index 1ee042d01c..e52cda8410 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-classification-multiclass-task-sentiment-analysis-automl-nlp-multiclass-sentiment-mlflow.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-classification-multiclass-task-sentiment-analysis-automl-nlp-multiclass-sentiment-mlflow.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login @@ -94,4 +93,4 @@ jobs: uses: ./.github/actions/upload-artifact with: name: automl-nlp-multiclass-sentiment-mlflow - path: sdk/python/jobs/automl-standalone-jobs/automl-nlp-text-classification-multiclass-task-sentiment-analysis \ No newline at end of file + path: sdk/python/jobs/automl-standalone-jobs/automl-nlp-text-classification-multiclass-task-sentiment-analysis diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-classification-multiclass-task-sentiment-analysis-automl-nlp-multiclass-sentiment.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-classification-multiclass-task-sentiment-analysis-automl-nlp-multiclass-sentiment.yml index 26fbfe5613..a2ab199b0f 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-classification-multiclass-task-sentiment-analysis-automl-nlp-multiclass-sentiment.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-classification-multiclass-task-sentiment-analysis-automl-nlp-multiclass-sentiment.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-classification-multilabel-task-paper-categorization-automl-nlp-multilabel-paper-cat.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-classification-multilabel-task-paper-categorization-automl-nlp-multilabel-paper-cat.yml index aef6406f6c..9b29d506b2 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-classification-multilabel-task-paper-categorization-automl-nlp-multilabel-paper-cat.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-classification-multilabel-task-paper-categorization-automl-nlp-multilabel-paper-cat.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-named-entity-recognition-task-automl-nlp-text-ner-task.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-named-entity-recognition-task-automl-nlp-text-ner-task.yml index 74d91e28dc..7b8d1d2cd6 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-named-entity-recognition-task-automl-nlp-text-ner-task.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-named-entity-recognition-task-automl-nlp-text-ner-task.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-named-entity-recognition-task-distributed-sweeping-automl-nlp-text-ner-task-distributed-with-sweeping.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-named-entity-recognition-task-distributed-sweeping-automl-nlp-text-ner-task-distributed-with-sweeping.yml index 08ec11ce36..60097ba769 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-named-entity-recognition-task-distributed-sweeping-automl-nlp-text-ner-task-distributed-with-sweeping.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-named-entity-recognition-task-distributed-sweeping-automl-nlp-text-ner-task-distributed-with-sweeping.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-regression-task-hardware-performance-automl-regression-task-hardware-performance.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-regression-task-hardware-performance-automl-regression-task-hardware-performance.yml index f1c606c4d7..2c9baea9e4 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-regression-task-hardware-performance-automl-regression-task-hardware-performance.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-regression-task-hardware-performance-automl-regression-task-hardware-performance.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-jobs-configuration.yml b/.github/workflows/sdk-jobs-configuration.yml index b301d55034..c67f8cd13e 100644 --- a/.github/workflows/sdk-jobs-configuration.yml +++ b/.github/workflows/sdk-jobs-configuration.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-chat_completion_with_model_as_platform.yml b/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-chat_completion_with_model_as_platform.yml new file mode 100644 index 0000000000..996065eca6 --- /dev/null +++ b/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-chat_completion_with_model_as_platform.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-chat_completion_with_model_as_platform +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "1 11/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion/** + - .github/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-chat_completion_with_model_as_platform.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/finetuning/standalone/model-as-a-platform/chat-completion/chat_completion_with_model_as_platform.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "chat_completion_with_model_as_platform.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python chat_completion_with_model_as_platform.ipynb chat_completion_with_model_as_platform.output.ipynb + working-directory: sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: chat_completion_with_model_as_platform + path: sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion diff --git a/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-gpt-oss-20b-gpt-oss-20b-chat-completion.yml b/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-gpt-oss-20b-gpt-oss-20b-chat-completion.yml new file mode 100644 index 0000000000..1f68beeaf4 --- /dev/null +++ b/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-gpt-oss-20b-gpt-oss-20b-chat-completion.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-gpt-oss-20b-gpt-oss-20b-chat-completion +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "58 5/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion/gpt-oss-20b/** + - .github/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-gpt-oss-20b-gpt-oss-20b-chat-completion.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion/gpt-oss-20b" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/finetuning/standalone/model-as-a-platform/chat-completion/gpt-oss-20b/gpt-oss-20b-chat-completion.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "gpt-oss-20b-chat-completion.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python gpt-oss-20b-chat-completion.ipynb gpt-oss-20b-chat-completion.output.ipynb + working-directory: sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion/gpt-oss-20b + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: gpt-oss-20b-chat-completion + path: sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion/gpt-oss-20b diff --git a/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-text-generation-text_generation_with_model_as_a_platform.yml b/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-text-generation-text_generation_with_model_as_a_platform.yml new file mode 100644 index 0000000000..b56922d98c --- /dev/null +++ b/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-text-generation-text_generation_with_model_as_a_platform.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-finetuning-standalone-model-as-a-platform-text-generation-text_generation_with_model_as_a_platform +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "23 10/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/finetuning/standalone/model-as-a-platform/text-generation/** + - .github/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-text-generation-text_generation_with_model_as_a_platform.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/finetuning/standalone/model-as-a-platform/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/finetuning/standalone/model-as-a-platform/text-generation/text_generation_with_model_as_a_platform.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text_generation_with_model_as_a_platform.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text_generation_with_model_as_a_platform.ipynb text_generation_with_model_as_a_platform.output.ipynb + working-directory: sdk/python/jobs/finetuning/standalone/model-as-a-platform/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text_generation_with_model_as_a_platform + path: sdk/python/jobs/finetuning/standalone/model-as-a-platform/text-generation diff --git a/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-service-chat-completion-chat_completion_with_model_as_service.yml b/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-service-chat-completion-chat_completion_with_model_as_service.yml new file mode 100644 index 0000000000..b8d705c447 --- /dev/null +++ b/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-service-chat-completion-chat_completion_with_model_as_service.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-finetuning-standalone-model-as-a-service-chat-completion-chat_completion_with_model_as_service +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "36 6/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/finetuning/standalone/model-as-a-service/chat-completion/** + - .github/workflows/sdk-jobs-finetuning-standalone-model-as-a-service-chat-completion-chat_completion_with_model_as_service.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/finetuning/standalone/model-as-a-service/chat-completion" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/finetuning/standalone/model-as-a-service/chat-completion/chat_completion_with_model_as_service.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "chat_completion_with_model_as_service.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python chat_completion_with_model_as_service.ipynb chat_completion_with_model_as_service.output.ipynb + working-directory: sdk/python/jobs/finetuning/standalone/model-as-a-service/chat-completion + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: chat_completion_with_model_as_service + path: sdk/python/jobs/finetuning/standalone/model-as-a-service/chat-completion diff --git a/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-service-text-generation-text_generation_with_model_as_a_service.yml b/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-service-text-generation-text_generation_with_model_as_a_service.yml new file mode 100644 index 0000000000..2d85c17d8a --- /dev/null +++ b/.github/workflows/sdk-jobs-finetuning-standalone-model-as-a-service-text-generation-text_generation_with_model_as_a_service.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-finetuning-standalone-model-as-a-service-text-generation-text_generation_with_model_as_a_service +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "31 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/finetuning/standalone/model-as-a-service/text-generation/** + - .github/workflows/sdk-jobs-finetuning-standalone-model-as-a-service-text-generation-text_generation_with_model_as_a_service.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/finetuning/standalone/model-as-a-service/text-generation" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/finetuning/standalone/model-as-a-service/text-generation/text_generation_with_model_as_a_service.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "text_generation_with_model_as_a_service.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python text_generation_with_model_as_a_service.ipynb text_generation_with_model_as_a_service.output.ipynb + working-directory: sdk/python/jobs/finetuning/standalone/model-as-a-service/text-generation + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: text_generation_with_model_as_a_service + path: sdk/python/jobs/finetuning/standalone/model-as-a-service/text-generation diff --git a/.github/workflows/sdk-jobs-grpo-grpo_from_aml_pipeline.yml b/.github/workflows/sdk-jobs-grpo-grpo_from_aml_pipeline.yml new file mode 100644 index 0000000000..3fca9793e6 --- /dev/null +++ b/.github/workflows/sdk-jobs-grpo-grpo_from_aml_pipeline.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-grpo-grpo_from_aml_pipeline +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "59 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/grpo/** + - .github/workflows/sdk-jobs-grpo-grpo_from_aml_pipeline.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/grpo" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/grpo/grpo_from_aml_pipeline.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "grpo_from_aml_pipeline.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python grpo_from_aml_pipeline.ipynb grpo_from_aml_pipeline.output.ipynb + working-directory: sdk/python/jobs/grpo + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: grpo_from_aml_pipeline + path: sdk/python/jobs/grpo diff --git a/.github/workflows/sdk-jobs-grpo-launch_grpo_command_job-med-mcqa-commented.yml b/.github/workflows/sdk-jobs-grpo-launch_grpo_command_job-med-mcqa-commented.yml new file mode 100644 index 0000000000..611ce91f5e --- /dev/null +++ b/.github/workflows/sdk-jobs-grpo-launch_grpo_command_job-med-mcqa-commented.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-grpo-launch_grpo_command_job-med-mcqa-commented +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "55 0/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/grpo/** + - .github/workflows/sdk-jobs-grpo-launch_grpo_command_job-med-mcqa-commented.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/grpo" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/grpo/launch_grpo_command_job-med-mcqa-commented.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "launch_grpo_command_job-med-mcqa-commented.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python launch_grpo_command_job-med-mcqa-commented.ipynb launch_grpo_command_job-med-mcqa-commented.output.ipynb + working-directory: sdk/python/jobs/grpo + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: launch_grpo_command_job-med-mcqa-commented + path: sdk/python/jobs/grpo diff --git a/.github/workflows/sdk-jobs-parallel-1a_oj_sales_prediction-oj_sales_prediction.yml b/.github/workflows/sdk-jobs-parallel-1a_oj_sales_prediction-oj_sales_prediction.yml index 68163af5cf..804f9a5998 100644 --- a/.github/workflows/sdk-jobs-parallel-1a_oj_sales_prediction-oj_sales_prediction.yml +++ b/.github/workflows/sdk-jobs-parallel-1a_oj_sales_prediction-oj_sales_prediction.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-parallel-2a_iris_batch_prediction-iris_batch_prediction.yml b/.github/workflows/sdk-jobs-parallel-2a_iris_batch_prediction-iris_batch_prediction.yml index 63045eb843..d7099bf177 100644 --- a/.github/workflows/sdk-jobs-parallel-2a_iris_batch_prediction-iris_batch_prediction.yml +++ b/.github/workflows/sdk-jobs-parallel-2a_iris_batch_prediction-iris_batch_prediction.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-parallel-3a_mnist_batch_identification-mnist_batch_prediction.yml b/.github/workflows/sdk-jobs-parallel-3a_mnist_batch_identification-mnist_batch_prediction.yml index 56acdd831f..112102fab7 100644 --- a/.github/workflows/sdk-jobs-parallel-3a_mnist_batch_identification-mnist_batch_prediction.yml +++ b/.github/workflows/sdk-jobs-parallel-3a_mnist_batch_identification-mnist_batch_prediction.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1a_pipeline_with_components_from_yaml-pipeline_with_components_from_yaml.yml b/.github/workflows/sdk-jobs-pipelines-1a_pipeline_with_components_from_yaml-pipeline_with_components_from_yaml.yml index 99a27a5d60..4a651ec92c 100644 --- a/.github/workflows/sdk-jobs-pipelines-1a_pipeline_with_components_from_yaml-pipeline_with_components_from_yaml.yml +++ b/.github/workflows/sdk-jobs-pipelines-1a_pipeline_with_components_from_yaml-pipeline_with_components_from_yaml.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1a_pipeline_with_components_from_yaml-pipeline_with_components_from_yaml_serverless.yml b/.github/workflows/sdk-jobs-pipelines-1a_pipeline_with_components_from_yaml-pipeline_with_components_from_yaml_serverless.yml new file mode 100644 index 0000000000..03ca3e9cfb --- /dev/null +++ b/.github/workflows/sdk-jobs-pipelines-1a_pipeline_with_components_from_yaml-pipeline_with_components_from_yaml_serverless.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-pipelines-1a_pipeline_with_components_from_yaml-pipeline_with_components_from_yaml_serverless +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "25 7/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml/** + - .github/workflows/sdk-jobs-pipelines-1a_pipeline_with_components_from_yaml-pipeline_with_components_from_yaml_serverless.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/pipelines/1a_pipeline_with_components_from_yaml/pipeline_with_components_from_yaml_serverless.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "pipeline_with_components_from_yaml_serverless.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python pipeline_with_components_from_yaml_serverless.ipynb pipeline_with_components_from_yaml_serverless.output.ipynb + working-directory: sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: pipeline_with_components_from_yaml_serverless + path: sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml diff --git a/.github/workflows/sdk-jobs-pipelines-1b_pipeline_with_python_function_components-pipeline_with_python_function_components.yml b/.github/workflows/sdk-jobs-pipelines-1b_pipeline_with_python_function_components-pipeline_with_python_function_components.yml index 401fac7113..e65d5378a8 100644 --- a/.github/workflows/sdk-jobs-pipelines-1b_pipeline_with_python_function_components-pipeline_with_python_function_components.yml +++ b/.github/workflows/sdk-jobs-pipelines-1b_pipeline_with_python_function_components-pipeline_with_python_function_components.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1c_pipeline_with_hyperparameter_sweep-pipeline_with_hyperparameter_sweep.yml b/.github/workflows/sdk-jobs-pipelines-1c_pipeline_with_hyperparameter_sweep-pipeline_with_hyperparameter_sweep.yml index 5ec9b99fbb..fa0fa3c28b 100644 --- a/.github/workflows/sdk-jobs-pipelines-1c_pipeline_with_hyperparameter_sweep-pipeline_with_hyperparameter_sweep.yml +++ b/.github/workflows/sdk-jobs-pipelines-1c_pipeline_with_hyperparameter_sweep-pipeline_with_hyperparameter_sweep.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1d_pipeline_with_non_python_components-pipeline_with_non_python_components.yml b/.github/workflows/sdk-jobs-pipelines-1d_pipeline_with_non_python_components-pipeline_with_non_python_components.yml index fb245b4818..40eff04549 100644 --- a/.github/workflows/sdk-jobs-pipelines-1d_pipeline_with_non_python_components-pipeline_with_non_python_components.yml +++ b/.github/workflows/sdk-jobs-pipelines-1d_pipeline_with_non_python_components-pipeline_with_non_python_components.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1e_pipeline_with_registered_components-pipeline_with_registered_components.yml b/.github/workflows/sdk-jobs-pipelines-1e_pipeline_with_registered_components-pipeline_with_registered_components.yml index 97862b0ca8..6b9aaf9e42 100644 --- a/.github/workflows/sdk-jobs-pipelines-1e_pipeline_with_registered_components-pipeline_with_registered_components.yml +++ b/.github/workflows/sdk-jobs-pipelines-1e_pipeline_with_registered_components-pipeline_with_registered_components.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1g_pipeline_with_parallel_nodes-pipeline_with_parallel_nodes.yml b/.github/workflows/sdk-jobs-pipelines-1g_pipeline_with_parallel_nodes-pipeline_with_parallel_nodes.yml index d2a920cbde..2bffcb49b5 100644 --- a/.github/workflows/sdk-jobs-pipelines-1g_pipeline_with_parallel_nodes-pipeline_with_parallel_nodes.yml +++ b/.github/workflows/sdk-jobs-pipelines-1g_pipeline_with_parallel_nodes-pipeline_with_parallel_nodes.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-classification-bankmarketing-in-pipeline-automl-classification-bankmarketing-in-pipeline.yml b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-classification-bankmarketing-in-pipeline-automl-classification-bankmarketing-in-pipeline.yml index 26d062cc20..d2955cd4d8 100644 --- a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-classification-bankmarketing-in-pipeline-automl-classification-bankmarketing-in-pipeline.yml +++ b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-classification-bankmarketing-in-pipeline-automl-classification-bankmarketing-in-pipeline.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-forecasting-in-pipeline-automl-forecasting-in-pipeline.yml b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-forecasting-in-pipeline-automl-forecasting-in-pipeline.yml index 502a8fb06f..9ac02aa17d 100644 --- a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-forecasting-in-pipeline-automl-forecasting-in-pipeline.yml +++ b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-forecasting-in-pipeline-automl-forecasting-in-pipeline.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-classification-multiclass-in-pipeline-automl-image-classification-multiclass-in-pipeline.yml b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-classification-multiclass-in-pipeline-automl-image-classification-multiclass-in-pipeline.yml index 86b571f586..96b0eed2a9 100644 --- a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-classification-multiclass-in-pipeline-automl-image-classification-multiclass-in-pipeline.yml +++ b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-classification-multiclass-in-pipeline-automl-image-classification-multiclass-in-pipeline.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-classification-multilabel-in-pipeline-automl-image-classification-multilabel-in-pipeline.yml b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-classification-multilabel-in-pipeline-automl-image-classification-multilabel-in-pipeline.yml index ced59e048b..29d3f0b5e2 100644 --- a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-classification-multilabel-in-pipeline-automl-image-classification-multilabel-in-pipeline.yml +++ b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-classification-multilabel-in-pipeline-automl-image-classification-multilabel-in-pipeline.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-instance-segmentation-in-pipeline-automl-image-instance-segmentation-in-pipeline.yml b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-instance-segmentation-in-pipeline-automl-image-instance-segmentation-in-pipeline.yml index 844e426a54..4f49e23560 100644 --- a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-instance-segmentation-in-pipeline-automl-image-instance-segmentation-in-pipeline.yml +++ b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-instance-segmentation-in-pipeline-automl-image-instance-segmentation-in-pipeline.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-object-detection-in-pipeline-automl-image-object-detection-in-pipeline.yml b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-object-detection-in-pipeline-automl-image-object-detection-in-pipeline.yml index 71ca4a7f15..dc11d09041 100644 --- a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-object-detection-in-pipeline-automl-image-object-detection-in-pipeline.yml +++ b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-image-object-detection-in-pipeline-automl-image-object-detection-in-pipeline.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-regression-house-pricing-in-pipeline-automl-regression-house-pricing-in-pipeline.yml b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-regression-house-pricing-in-pipeline-automl-regression-house-pricing-in-pipeline.yml index 8805ba724c..b484cace22 100644 --- a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-regression-house-pricing-in-pipeline-automl-regression-house-pricing-in-pipeline.yml +++ b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-regression-house-pricing-in-pipeline-automl-regression-house-pricing-in-pipeline.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-text-classification-in-pipeline-automl-text-classification-in-pipeline.yml b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-text-classification-in-pipeline-automl-text-classification-in-pipeline.yml index 45e9dfc599..a5da43098d 100644 --- a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-text-classification-in-pipeline-automl-text-classification-in-pipeline.yml +++ b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-text-classification-in-pipeline-automl-text-classification-in-pipeline.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-text-classification-multilabel-in-pipeline-automl-text-classification-multilabel-in-pipeline.yml b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-text-classification-multilabel-in-pipeline-automl-text-classification-multilabel-in-pipeline.yml index 1cee15efff..42cb997260 100644 --- a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-text-classification-multilabel-in-pipeline-automl-text-classification-multilabel-in-pipeline.yml +++ b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-text-classification-multilabel-in-pipeline-automl-text-classification-multilabel-in-pipeline.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-text-ner-named-entity-recognition-in-pipeline-automl-text-ner-named-entity-recognition-in-pipeline.yml b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-text-ner-named-entity-recognition-in-pipeline-automl-text-ner-named-entity-recognition-in-pipeline.yml index 4871f43f05..1ff50ee12f 100644 --- a/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-text-ner-named-entity-recognition-in-pipeline-automl-text-ner-named-entity-recognition-in-pipeline.yml +++ b/.github/workflows/sdk-jobs-pipelines-1h_automl_in_pipeline-automl-text-ner-named-entity-recognition-in-pipeline-automl-text-ner-named-entity-recognition-in-pipeline.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1i_pipeline_with_spark_nodes-pipeline_with_spark_nodes.yml b/.github/workflows/sdk-jobs-pipelines-1i_pipeline_with_spark_nodes-pipeline_with_spark_nodes.yml index 69056770ca..19d2e04c19 100644 --- a/.github/workflows/sdk-jobs-pipelines-1i_pipeline_with_spark_nodes-pipeline_with_spark_nodes.yml +++ b/.github/workflows/sdk-jobs-pipelines-1i_pipeline_with_spark_nodes-pipeline_with_spark_nodes.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component.yml b/.github/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component.yml index 2e68689874..6ee67e6c41 100644 --- a/.github/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component.yml +++ b/.github/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-pipeline_with_train_eval_pipeline_component.yml b/.github/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-pipeline_with_train_eval_pipeline_component.yml index 4319564eaa..fac6694568 100644 --- a/.github/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-pipeline_with_train_eval_pipeline_component.yml +++ b/.github/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-pipeline_with_train_eval_pipeline_component.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-hts-pipeline-aml-demand-forecast-hts.yml b/.github/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-hts-pipeline-aml-demand-forecast-hts.yml new file mode 100644 index 0000000000..7753c26192 --- /dev/null +++ b/.github/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-hts-pipeline-aml-demand-forecast-hts.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-hts-pipeline-aml-demand-forecast-hts +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "18 6/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-hts-pipeline/** + - .github/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-hts-pipeline-aml-demand-forecast-hts.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-hts-pipeline" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-hts-pipeline/aml-demand-forecast-hts.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "aml-demand-forecast-hts.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python aml-demand-forecast-hts.ipynb aml-demand-forecast-hts.output.ipynb + working-directory: sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-hts-pipeline + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: aml-demand-forecast-hts + path: sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-hts-pipeline diff --git a/.github/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-mm-pipeline-aml-demand-forecast-mm-pipeline.yml b/.github/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-mm-pipeline-aml-demand-forecast-mm-pipeline.yml new file mode 100644 index 0000000000..817d863bcc --- /dev/null +++ b/.github/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-mm-pipeline-aml-demand-forecast-mm-pipeline.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-mm-pipeline-aml-demand-forecast-mm-pipeline +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "49 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-mm-pipeline/** + - .github/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-mm-pipeline-aml-demand-forecast-mm-pipeline.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-mm-pipeline" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-mm-pipeline/aml-demand-forecast-mm-pipeline.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "aml-demand-forecast-mm-pipeline.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python aml-demand-forecast-mm-pipeline.ipynb aml-demand-forecast-mm-pipeline.output.ipynb + working-directory: sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-mm-pipeline + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: aml-demand-forecast-mm-pipeline + path: sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-mm-pipeline diff --git a/.github/workflows/sdk-jobs-pipelines-1l_flow_in_pipeline-flow_in_pipeline.yml b/.github/workflows/sdk-jobs-pipelines-1l_flow_in_pipeline-flow_in_pipeline.yml index 74c3453b02..e539ba11e1 100644 --- a/.github/workflows/sdk-jobs-pipelines-1l_flow_in_pipeline-flow_in_pipeline.yml +++ b/.github/workflows/sdk-jobs-pipelines-1l_flow_in_pipeline-flow_in_pipeline.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-2a_train_mnist_with_tensorflow-train_mnist_with_tensorflow.yml b/.github/workflows/sdk-jobs-pipelines-2a_train_mnist_with_tensorflow-train_mnist_with_tensorflow.yml index 9e9593a3ce..2c24c8f3d9 100644 --- a/.github/workflows/sdk-jobs-pipelines-2a_train_mnist_with_tensorflow-train_mnist_with_tensorflow.yml +++ b/.github/workflows/sdk-jobs-pipelines-2a_train_mnist_with_tensorflow-train_mnist_with_tensorflow.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-2b_train_cifar_10_with_pytorch-train_cifar_10_with_pytorch.yml b/.github/workflows/sdk-jobs-pipelines-2b_train_cifar_10_with_pytorch-train_cifar_10_with_pytorch.yml index d4099d1e74..a86f70a576 100644 --- a/.github/workflows/sdk-jobs-pipelines-2b_train_cifar_10_with_pytorch-train_cifar_10_with_pytorch.yml +++ b/.github/workflows/sdk-jobs-pipelines-2b_train_cifar_10_with_pytorch-train_cifar_10_with_pytorch.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-2c_nyc_taxi_data_regression-nyc_taxi_data_regression.yml b/.github/workflows/sdk-jobs-pipelines-2c_nyc_taxi_data_regression-nyc_taxi_data_regression.yml index f46b1ba7d2..8898b9c640 100644 --- a/.github/workflows/sdk-jobs-pipelines-2c_nyc_taxi_data_regression-nyc_taxi_data_regression.yml +++ b/.github/workflows/sdk-jobs-pipelines-2c_nyc_taxi_data_regression-nyc_taxi_data_regression.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-2d_image_classification_with_densenet-image_classification_with_densenet.yml b/.github/workflows/sdk-jobs-pipelines-2d_image_classification_with_densenet-image_classification_with_densenet.yml index 02ba29676d..7b14f65f10 100644 --- a/.github/workflows/sdk-jobs-pipelines-2d_image_classification_with_densenet-image_classification_with_densenet.yml +++ b/.github/workflows/sdk-jobs-pipelines-2d_image_classification_with_densenet-image_classification_with_densenet.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-2e_image_classification_keras_minist_convnet-image_classification_keras_minist_convnet.yml b/.github/workflows/sdk-jobs-pipelines-2e_image_classification_keras_minist_convnet-image_classification_keras_minist_convnet.yml index fe4879d38e..6d1f71f2b5 100644 --- a/.github/workflows/sdk-jobs-pipelines-2e_image_classification_keras_minist_convnet-image_classification_keras_minist_convnet.yml +++ b/.github/workflows/sdk-jobs-pipelines-2e_image_classification_keras_minist_convnet-image_classification_keras_minist_convnet.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-pipelines-stepsequence-pipeline_with_step_sequence_dummy_dependencies.yml b/.github/workflows/sdk-jobs-pipelines-stepsequence-pipeline_with_step_sequence_dummy_dependencies.yml index b69d4e7b2d..b817a27a33 100644 --- a/.github/workflows/sdk-jobs-pipelines-stepsequence-pipeline_with_step_sequence_dummy_dependencies.yml +++ b/.github/workflows/sdk-jobs-pipelines-stepsequence-pipeline_with_step_sequence_dummy_dependencies.yml @@ -1,15 +1,21 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + name: sdk-jobs-pipelines-stepsequence-pipeline_with_step_sequence_dummy_dependencies +# This file is created by sdk/python/readme.py. +# Please do not edit directly. on: workflow_dispatch: schedule: - - cron: "0 */8 * * *" + - cron: "38 7/12 * * *" pull_request: branches: - main paths: - - sdk/python/jobs/pipelines/stepsequence/pipeline_with_step_sequence_dummy_dependencies.ipynb + - sdk/python/jobs/pipelines/stepsequence/** - .github/workflows/sdk-jobs-pipelines-stepsequence-pipeline_with_step_sequence_dummy_dependencies.yml - - "!sdk/python/jobs/pipelines/stepsequence/**/*.md" - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh @@ -19,19 +25,18 @@ permissions: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true - jobs: build: runs-on: ubuntu-latest steps: - name: check out repo - uses: actions/checkout@v3 + uses: actions/checkout@v2 - name: setup python - uses: actions/setup-python@v4 + uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: @@ -51,6 +56,11 @@ jobs: bash setup.sh working-directory: sdk/python continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/jobs/pipelines/stepsequence" + working-directory: infra/bootstrapping + continue-on-error: false - name: setup-cli run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; @@ -67,8 +77,6 @@ jobs: az account get-access-token --scope https://management.azure.com/.default --output none # ML az account get-access-token --scope https://ml.azure.com/.default --output none - - name: install ml extension - run: az extension add -n ml -y - name: run jobs/pipelines/stepsequence/pipeline_with_step_sequence_dummy_dependencies.ipynb run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; @@ -82,5 +90,5 @@ jobs: if: ${{ always() }} uses: ./.github/actions/upload-artifact with: - name: stepsequence + name: pipeline_with_step_sequence_dummy_dependencies path: sdk/python/jobs/pipelines/stepsequence diff --git a/.github/workflows/sdk-jobs-single-step-lightgbm-iris-lightgbm-iris-sweep.yml b/.github/workflows/sdk-jobs-single-step-lightgbm-iris-lightgbm-iris-sweep.yml index 3224313d92..6591351d59 100644 --- a/.github/workflows/sdk-jobs-single-step-lightgbm-iris-lightgbm-iris-sweep.yml +++ b/.github/workflows/sdk-jobs-single-step-lightgbm-iris-lightgbm-iris-sweep.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-single-step-pytorch-distributed-training-distributed-cifar10.yml b/.github/workflows/sdk-jobs-single-step-pytorch-distributed-training-distributed-cifar10.yml index 693f35cabe..75caa7a5f2 100644 --- a/.github/workflows/sdk-jobs-single-step-pytorch-distributed-training-distributed-cifar10.yml +++ b/.github/workflows/sdk-jobs-single-step-pytorch-distributed-training-distributed-cifar10.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-single-step-pytorch-train-hyperparameter-tune-deploy-with-pytorch-train-hyperparameter-tune-deploy-with-pytorch.yml b/.github/workflows/sdk-jobs-single-step-pytorch-train-hyperparameter-tune-deploy-with-pytorch-train-hyperparameter-tune-deploy-with-pytorch.yml index 1376472e1e..70234e991e 100644 --- a/.github/workflows/sdk-jobs-single-step-pytorch-train-hyperparameter-tune-deploy-with-pytorch-train-hyperparameter-tune-deploy-with-pytorch.yml +++ b/.github/workflows/sdk-jobs-single-step-pytorch-train-hyperparameter-tune-deploy-with-pytorch-train-hyperparameter-tune-deploy-with-pytorch.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-single-step-tensorflow-mnist-distributed-tensorflow-mnist-distributed.yml b/.github/workflows/sdk-jobs-single-step-tensorflow-mnist-distributed-tensorflow-mnist-distributed.yml index 4574f3a285..b9a06189b7 100644 --- a/.github/workflows/sdk-jobs-single-step-tensorflow-mnist-distributed-tensorflow-mnist-distributed.yml +++ b/.github/workflows/sdk-jobs-single-step-tensorflow-mnist-distributed-tensorflow-mnist-distributed.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-spark-automation-run_interactive_session_notebook.yml b/.github/workflows/sdk-jobs-spark-automation-run_interactive_session_notebook.yml index 266a78e533..3c7de01d1a 100644 --- a/.github/workflows/sdk-jobs-spark-automation-run_interactive_session_notebook.yml +++ b/.github/workflows/sdk-jobs-spark-automation-run_interactive_session_notebook.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-spark-submit_spark_pipeline_jobs.yml b/.github/workflows/sdk-jobs-spark-submit_spark_pipeline_jobs.yml index eda607c9c3..fa5d8898cc 100644 --- a/.github/workflows/sdk-jobs-spark-submit_spark_pipeline_jobs.yml +++ b/.github/workflows/sdk-jobs-spark-submit_spark_pipeline_jobs.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-spark-submit_spark_standalone_jobs.yml b/.github/workflows/sdk-jobs-spark-submit_spark_standalone_jobs.yml index 180c266435..9057c327a5 100644 --- a/.github/workflows/sdk-jobs-spark-submit_spark_standalone_jobs.yml +++ b/.github/workflows/sdk-jobs-spark-submit_spark_standalone_jobs.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-jobs-spark-submit_spark_standalone_jobs_managed_vnet.yml b/.github/workflows/sdk-jobs-spark-submit_spark_standalone_jobs_managed_vnet.yml index b23b79957d..d66d53ddfe 100644 --- a/.github/workflows/sdk-jobs-spark-submit_spark_standalone_jobs_managed_vnet.yml +++ b/.github/workflows/sdk-jobs-spark-submit_spark_standalone_jobs_managed_vnet.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-resources-compute-attach_manage_spark_pools.yml b/.github/workflows/sdk-resources-compute-attach_manage_spark_pools.yml index 41e135a452..5a9a0ae94f 100644 --- a/.github/workflows/sdk-resources-compute-attach_manage_spark_pools.yml +++ b/.github/workflows/sdk-resources-compute-attach_manage_spark_pools.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-resources-compute-compute.yml b/.github/workflows/sdk-resources-compute-compute.yml index 8f6884671c..636a27d00f 100644 --- a/.github/workflows/sdk-resources-compute-compute.yml +++ b/.github/workflows/sdk-resources-compute-compute.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-resources-connections-connections.yml b/.github/workflows/sdk-resources-connections-connections.yml index cd1a5fed47..8ac3b58d0c 100644 --- a/.github/workflows/sdk-resources-connections-connections.yml +++ b/.github/workflows/sdk-resources-connections-connections.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-resources-registry-registry-create.yml b/.github/workflows/sdk-resources-registry-registry-create.yml index 93729afeaa..85cbbe4a43 100644 --- a/.github/workflows/sdk-resources-registry-registry-create.yml +++ b/.github/workflows/sdk-resources-registry-registry-create.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-resources-workspace-workspace-managed-network.yml b/.github/workflows/sdk-resources-workspace-workspace-managed-network.yml new file mode 100644 index 0000000000..7f951aacca --- /dev/null +++ b/.github/workflows/sdk-resources-workspace-workspace-managed-network.yml @@ -0,0 +1,94 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-resources-workspace-workspace-managed-network +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "59 4/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/resources/workspace/** + - .github/workflows/sdk-resources-workspace-workspace-managed-network.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/resources/workspace" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run resources/workspace/workspace-managed-network.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "workspace-managed-network.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python workspace-managed-network.ipynb workspace-managed-network.output.ipynb + working-directory: sdk/python/resources/workspace + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: workspace-managed-network + path: sdk/python/resources/workspace diff --git a/.github/workflows/sdk-resources-workspace-workspace.yml b/.github/workflows/sdk-resources-workspace-workspace.yml index 1f23b953a6..9123bc9230 100644 --- a/.github/workflows/sdk-resources-workspace-workspace.yml +++ b/.github/workflows/sdk-resources-workspace-workspace.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-responsible-ai-mlflow-deployment-with-explanations-mlflow-deployment-with-explanations.yml b/.github/workflows/sdk-responsible-ai-mlflow-deployment-with-explanations-mlflow-deployment-with-explanations.yml index d053d2db5a..2cc14010fc 100644 --- a/.github/workflows/sdk-responsible-ai-mlflow-deployment-with-explanations-mlflow-deployment-with-explanations.yml +++ b/.github/workflows/sdk-responsible-ai-mlflow-deployment-with-explanations-mlflow-deployment-with-explanations.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-diabetes-decision-making-responsibleaidashboard-diabetes-decision-making.yml b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-diabetes-decision-making-responsibleaidashboard-diabetes-decision-making.yml index 75425b7a39..c1d1e2132a 100644 --- a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-diabetes-decision-making-responsibleaidashboard-diabetes-decision-making.yml +++ b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-diabetes-decision-making-responsibleaidashboard-diabetes-decision-making.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-diabetes-regression-model-debugging-responsibleaidashboard-diabetes-regression-model-debugging.yml b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-diabetes-regression-model-debugging-responsibleaidashboard-diabetes-regression-model-debugging.yml index 54b063df42..10144eb787 100644 --- a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-diabetes-regression-model-debugging-responsibleaidashboard-diabetes-regression-model-debugging.yml +++ b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-diabetes-regression-model-debugging-responsibleaidashboard-diabetes-regression-model-debugging.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-ed-student-attrition-clf-responsibleaidashboard-ed-student-attrition-clf.yml b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-ed-student-attrition-clf-responsibleaidashboard-ed-student-attrition-clf.yml new file mode 100644 index 0000000000..e3f651d49c --- /dev/null +++ b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-ed-student-attrition-clf-responsibleaidashboard-ed-student-attrition-clf.yml @@ -0,0 +1,96 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-responsible-ai-tabular-responsibleaidashboard-ed-student-attrition-clf-responsibleaidashboard-ed-student-attrition-clf +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "43 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/responsible-ai/tabular/responsibleaidashboard-ed-student-attrition-clf/** + - .github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-ed-student-attrition-clf-responsibleaidashboard-ed-student-attrition-clf.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh + +permissions: + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: pip install notebook reqs + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/responsible-ai/tabular/responsibleaidashboard-ed-student-attrition-clf" + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: Eagerly cache access tokens for required scopes + run: | + # Workaround for azure-cli's lack of support for ID token refresh + # Taken from: https://github.com/Azure/login/issues/372#issuecomment-2056289617 + + # Management + az account get-access-token --scope https://management.azure.com/.default --output none + # ML + az account get-access-token --scope https://ml.azure.com/.default --output none + - name: run responsible-ai/tabular/responsibleaidashboard-ed-student-attrition-clf/responsibleaidashboard-ed-student-attrition-clf.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "responsibleaidashboard-ed-student-attrition-clf.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python responsibleaidashboard-ed-student-attrition-clf.ipynb responsibleaidashboard-ed-student-attrition-clf.output.ipynb + working-directory: sdk/python/responsible-ai/tabular/responsibleaidashboard-ed-student-attrition-clf + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: ./.github/actions/upload-artifact + with: + name: responsibleaidashboard-ed-student-attrition-clf + path: sdk/python/responsible-ai/tabular/responsibleaidashboard-ed-student-attrition-clf diff --git a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-finance-loan-classification-responsibleaidashboard-finance-loan-classification.yml b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-finance-loan-classification-responsibleaidashboard-finance-loan-classification.yml index 89836625e8..9484776760 100644 --- a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-finance-loan-classification-responsibleaidashboard-finance-loan-classification.yml +++ b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-finance-loan-classification-responsibleaidashboard-finance-loan-classification.yml @@ -19,7 +19,7 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt + permissions: id-token: write concurrency: @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login @@ -58,6 +58,11 @@ jobs: bash setup.sh working-directory: sdk/python continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/responsible-ai/tabular/responsibleaidashboard-finance-loan-classification" + working-directory: infra/bootstrapping + continue-on-error: false - name: setup-cli run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; @@ -89,3 +94,6 @@ jobs: with: name: responsibleaidashboard-finance-loan-classification path: sdk/python/responsible-ai/tabular/responsibleaidashboard-finance-loan-classification + + - name: Remove the compute if notebook did not done it properly. + run: bash "${{ github.workspace }}/infra/bootstrapping/remove_computes.sh" "raitextcluster" diff --git a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-healthcare-covid-classification-responsibleaidashboard-healthcare-covid-classification.yml b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-healthcare-covid-classification-responsibleaidashboard-healthcare-covid-classification.yml index 6c7bd03dd2..a42d54e76c 100644 --- a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-healthcare-covid-classification-responsibleaidashboard-healthcare-covid-classification.yml +++ b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-healthcare-covid-classification-responsibleaidashboard-healthcare-covid-classification.yml @@ -19,7 +19,7 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt + permissions: id-token: write concurrency: @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login @@ -58,6 +58,11 @@ jobs: bash setup.sh working-directory: sdk/python continue-on-error: true + - name: validate readme + run: | + python check-readme.py "${{ github.workspace }}/sdk/python/responsible-ai/tabular/responsibleaidashboard-healthcare-covid-classification" + working-directory: infra/bootstrapping + continue-on-error: false - name: setup-cli run: | source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; @@ -89,3 +94,6 @@ jobs: with: name: responsibleaidashboard-healthcare-covid-classification path: sdk/python/responsible-ai/tabular/responsibleaidashboard-healthcare-covid-classification + + - name: Remove the compute if notebook did not done it properly. + run: bash "${{ github.workspace }}/infra/bootstrapping/remove_computes.sh" "raitextcluster" diff --git a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-housing-classification-model-debugging-responsibleaidashboard-housing-classification-model-debugging.yml b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-housing-classification-model-debugging-responsibleaidashboard-housing-classification-model-debugging.yml index 45acf3f273..421a867b71 100644 --- a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-housing-classification-model-debugging-responsibleaidashboard-housing-classification-model-debugging.yml +++ b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-housing-classification-model-debugging-responsibleaidashboard-housing-classification-model-debugging.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-housing-decision-making-responsibleaidashboard-housing-decision-making.yml b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-housing-decision-making-responsibleaidashboard-housing-decision-making.yml index bc931f8723..ce4a5c6ca1 100644 --- a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-housing-decision-making-responsibleaidashboard-housing-decision-making.yml +++ b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-housing-decision-making-responsibleaidashboard-housing-decision-making.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-programmer-regression-model-debugging-responsibleaidashboard-programmer-regression-model-debugging.yml b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-programmer-regression-model-debugging-responsibleaidashboard-programmer-regression-model-debugging.yml index 3917f077c7..7839d2700d 100644 --- a/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-programmer-regression-model-debugging-responsibleaidashboard-programmer-regression-model-debugging.yml +++ b/.github/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-programmer-regression-model-debugging-responsibleaidashboard-programmer-regression-model-debugging.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write @@ -35,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs run: pip install -r sdk/python/mlflow-requirements.txt - name: azure login diff --git a/.github/workflows/sdk-schedules-job-schedule.yml b/.github/workflows/sdk-schedules-job-schedule.yml index 3be006164d..975f9d52ca 100644 --- a/.github/workflows/sdk-schedules-job-schedule.yml +++ b/.github/workflows/sdk-schedules-job-schedule.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-using-mltable-delimited-files-example-delimited-files-example.yml b/.github/workflows/sdk-using-mltable-delimited-files-example-delimited-files-example.yml index 9adfb69445..3ec4f0371b 100644 --- a/.github/workflows/sdk-using-mltable-delimited-files-example-delimited-files-example.yml +++ b/.github/workflows/sdk-using-mltable-delimited-files-example-delimited-files-example.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-using-mltable-delta-lake-example-delta-lake-example.yml b/.github/workflows/sdk-using-mltable-delta-lake-example-delta-lake-example.yml index aeeaa7cfd6..9ed0c32594 100644 --- a/.github/workflows/sdk-using-mltable-delta-lake-example-delta-lake-example.yml +++ b/.github/workflows/sdk-using-mltable-delta-lake-example-delta-lake-example.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-using-mltable-from-paths-example-from-paths-example.yml b/.github/workflows/sdk-using-mltable-from-paths-example-from-paths-example.yml index 2f482ea8b3..2d62094363 100644 --- a/.github/workflows/sdk-using-mltable-from-paths-example-from-paths-example.yml +++ b/.github/workflows/sdk-using-mltable-from-paths-example-from-paths-example.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-using-mltable-local-to-cloud-mltable-local-to-cloud.yml b/.github/workflows/sdk-using-mltable-local-to-cloud-mltable-local-to-cloud.yml index fea5592150..9a96fdc77b 100644 --- a/.github/workflows/sdk-using-mltable-local-to-cloud-mltable-local-to-cloud.yml +++ b/.github/workflows/sdk-using-mltable-local-to-cloud-mltable-local-to-cloud.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/sdk-using-mltable-quickstart-mltable-quickstart.yml b/.github/workflows/sdk-using-mltable-quickstart-mltable-quickstart.yml index 2f80d5b14a..b460c7757c 100644 --- a/.github/workflows/sdk-using-mltable-quickstart-mltable-quickstart.yml +++ b/.github/workflows/sdk-using-mltable-quickstart-mltable-quickstart.yml @@ -34,9 +34,9 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt + run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt - name: azure login uses: azure/login@v1 with: diff --git a/.github/workflows/tutorials-azureml-getting-started-azureml-getting-started-studio.yml b/.github/workflows/tutorials-azureml-getting-started-azureml-getting-started-studio.yml index 827d2bc8c5..e88d9db832 100644 --- a/.github/workflows/tutorials-azureml-getting-started-azureml-getting-started-studio.yml +++ b/.github/workflows/tutorials-azureml-getting-started-azureml-getting-started-studio.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write concurrency: @@ -34,7 +33,7 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: "3.11" - name: pip install notebook reqs run: pip install -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs diff --git a/.github/workflows/tutorials-azureml-in-a-day-azureml-in-a-day.yml b/.github/workflows/tutorials-azureml-in-a-day-azureml-in-a-day.yml index e7ab0b663a..3750d3753e 100644 --- a/.github/workflows/tutorials-azureml-in-a-day-azureml-in-a-day.yml +++ b/.github/workflows/tutorials-azureml-in-a-day-azureml-in-a-day.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write concurrency: @@ -34,7 +33,7 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: "3.11" - name: pip install notebook reqs run: pip install -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs diff --git a/.github/workflows/tutorials-e2e-distributed-pytorch-image-e2e-object-classification-distributed-pytorch.yml b/.github/workflows/tutorials-e2e-distributed-pytorch-image-e2e-object-classification-distributed-pytorch.yml index d3e4e91dbb..9a86bfa0a5 100644 --- a/.github/workflows/tutorials-e2e-distributed-pytorch-image-e2e-object-classification-distributed-pytorch.yml +++ b/.github/workflows/tutorials-e2e-distributed-pytorch-image-e2e-object-classification-distributed-pytorch.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write concurrency: @@ -34,7 +33,7 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: "3.11" - name: pip install notebook reqs run: pip install -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs diff --git a/.github/workflows/tutorials-e2e-ds-experience-e2e-ml-workflow.yml b/.github/workflows/tutorials-e2e-ds-experience-e2e-ml-workflow.yml index 4edf19a566..68e71eb820 100644 --- a/.github/workflows/tutorials-e2e-ds-experience-e2e-ml-workflow.yml +++ b/.github/workflows/tutorials-e2e-ds-experience-e2e-ml-workflow.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write concurrency: @@ -34,7 +33,7 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: "3.11" - name: pip install notebook reqs run: pip install -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs diff --git a/.github/workflows/tutorials-get-started-notebooks-cloud-workstation.yml b/.github/workflows/tutorials-get-started-notebooks-cloud-workstation.yml index e0d16d4906..542aeac60b 100644 --- a/.github/workflows/tutorials-get-started-notebooks-cloud-workstation.yml +++ b/.github/workflows/tutorials-get-started-notebooks-cloud-workstation.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write concurrency: @@ -34,7 +33,7 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: "3.11" - name: pip install notebook reqs run: pip install -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs diff --git a/.github/workflows/tutorials-get-started-notebooks-deploy-model.yml b/.github/workflows/tutorials-get-started-notebooks-deploy-model.yml index 1661957428..e724a0f056 100644 --- a/.github/workflows/tutorials-get-started-notebooks-deploy-model.yml +++ b/.github/workflows/tutorials-get-started-notebooks-deploy-model.yml @@ -33,7 +33,7 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: "3.11" - name: pip install notebook reqs run: pip install -r sdk/python/dev-requirements.txt - name: azure login diff --git a/.github/workflows/tutorials-get-started-notebooks-explore-data.yml b/.github/workflows/tutorials-get-started-notebooks-explore-data.yml index 339c26fcdc..c034e8ed77 100644 --- a/.github/workflows/tutorials-get-started-notebooks-explore-data.yml +++ b/.github/workflows/tutorials-get-started-notebooks-explore-data.yml @@ -26,14 +26,14 @@ concurrency: cancel-in-progress: true jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: check out repo uses: actions/checkout@v2 - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: "3.11" - name: pip install notebook reqs run: pip install -r sdk/python/dev-requirements.txt - name: azure login diff --git a/.github/workflows/tutorials-get-started-notebooks-pipeline.yml b/.github/workflows/tutorials-get-started-notebooks-pipeline.yml index 22c2d75fec..8092529bed 100644 --- a/.github/workflows/tutorials-get-started-notebooks-pipeline.yml +++ b/.github/workflows/tutorials-get-started-notebooks-pipeline.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write concurrency: @@ -34,7 +33,7 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: "3.11" - name: pip install notebook reqs run: pip install -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs diff --git a/.github/workflows/tutorials-get-started-notebooks-quickstart.yml b/.github/workflows/tutorials-get-started-notebooks-quickstart.yml index 7d6f3b475c..6aa7cdae35 100644 --- a/.github/workflows/tutorials-get-started-notebooks-quickstart.yml +++ b/.github/workflows/tutorials-get-started-notebooks-quickstart.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write concurrency: @@ -34,7 +33,7 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: "3.11" - name: pip install notebook reqs run: pip install -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs diff --git a/.github/workflows/tutorials-get-started-notebooks-train-model.yml b/.github/workflows/tutorials-get-started-notebooks-train-model.yml index 29489e25dc..f1171307bd 100644 --- a/.github/workflows/tutorials-get-started-notebooks-train-model.yml +++ b/.github/workflows/tutorials-get-started-notebooks-train-model.yml @@ -19,7 +19,6 @@ on: - sdk/python/dev-requirements.txt - infra/bootstrapping/** - sdk/python/setup.sh - - sdk/python/mlflow-requirements.txt permissions: id-token: write concurrency: @@ -34,7 +33,7 @@ jobs: - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: "3.11" - name: pip install notebook reqs run: pip install -r sdk/python/dev-requirements.txt - name: pip install mlflow reqs diff --git a/cli/README.md b/cli/README.md index 5bf9dc72b0..a97ae32b7b 100644 --- a/cli/README.md +++ b/cli/README.md @@ -31,15 +31,7 @@ az extension add --name ml --yes ``` ## Examples - - -**Schedules** - -path|status| --|- -[basic-installation/basic-installation.yml](basic-installation/basic-installation.yml)|| - - + **Scripts** path|status| @@ -58,6 +50,7 @@ path|status| [deploy-managed-online-endpoint-access-resource-sai.sh](deploy-managed-online-endpoint-access-resource-sai.sh)|[![deploy-managed-online-endpoint-access-resource-sai](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-managed-online-endpoint-access-resource-sai/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-managed-online-endpoint-access-resource-sai.yml) [deploy-managed-online-endpoint-access-resource-uai.sh](deploy-managed-online-endpoint-access-resource-uai.sh)|[![deploy-managed-online-endpoint-access-resource-uai](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-managed-online-endpoint-access-resource-uai/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-managed-online-endpoint-access-resource-uai.yml) [deploy-managed-online-endpoint-ncd.sh](deploy-managed-online-endpoint-ncd.sh)|[![deploy-managed-online-endpoint-ncd](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-managed-online-endpoint-ncd/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-managed-online-endpoint-ncd.yml) +[deploy-managed-online-endpoint-workspacevnet.sh](deploy-managed-online-endpoint-workspacevnet.sh)|[![deploy-managed-online-endpoint-workspacevnet](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-managed-online-endpoint-workspacevnet/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-managed-online-endpoint-workspacevnet.yml) [deploy-managed-online-endpoint.sh](deploy-managed-online-endpoint.sh)|[![deploy-managed-online-endpoint](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-managed-online-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-managed-online-endpoint.yml) [deploy-mlcompute-create_with-system-identity.sh](deploy-mlcompute-create_with-system-identity.sh)|[![deploy-mlcompute-create_with-system-identity](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-mlcompute-create_with-system-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-mlcompute-create_with-system-identity.yml) [deploy-mlcompute-update-to-system-identity.sh](deploy-mlcompute-update-to-system-identity.sh)|[![deploy-mlcompute-update-to-system-identity](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-mlcompute-update-to-system-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-mlcompute-update-to-system-identity.yml) @@ -68,8 +61,8 @@ path|status| [deploy-moe-keyvault.sh](deploy-moe-keyvault.sh)|[![deploy-moe-keyvault](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-moe-keyvault/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-moe-keyvault.yml) [deploy-moe-minimal-single-model-registered.sh](deploy-moe-minimal-single-model-registered.sh)|[![deploy-moe-minimal-single-model-registered](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-moe-minimal-single-model-registered/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-moe-minimal-single-model-registered.yml) [deploy-moe-openapi.sh](deploy-moe-openapi.sh)|[![deploy-moe-openapi](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-moe-openapi/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-moe-openapi.yml) -[deploy-moe-vnet-mlflow.sh](deploy-moe-vnet-mlflow.sh)|[![deploy-moe-vnet-mlflow](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-moe-vnet-mlflow/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-moe-vnet-mlflow.yml) -[deploy-moe-vnet.sh](deploy-moe-vnet.sh)|[![deploy-moe-vnet](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-moe-vnet/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-moe-vnet.yml) +[deploy-moe-vnet-legacy.sh](deploy-moe-vnet-legacy.sh)|[![deploy-moe-vnet-legacy](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-moe-vnet-legacy/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-moe-vnet-legacy.yml) +[deploy-moe-vnet-mlflow-legacy.sh](deploy-moe-vnet-mlflow-legacy.sh)|[![deploy-moe-vnet-mlflow-legacy](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-moe-vnet-mlflow-legacy/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-moe-vnet-mlflow-legacy.yml) [deploy-rest.sh](deploy-rest.sh)|[![deploy-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-rest.yml) [deploy-safe-rollout-kubernetes-online-endpoints.sh](deploy-safe-rollout-kubernetes-online-endpoints.sh)|[![deploy-safe-rollout-kubernetes-online-endpoints](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-safe-rollout-kubernetes-online-endpoints/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-safe-rollout-kubernetes-online-endpoints.yml) [deploy-safe-rollout-online-endpoints.sh](deploy-safe-rollout-online-endpoints.sh)|[![deploy-safe-rollout-online-endpoints](https://github.com/Azure/azureml-examples/workflows/cli-scripts-deploy-safe-rollout-online-endpoints/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-deploy-safe-rollout-online-endpoints.yml) @@ -78,6 +71,7 @@ path|status| [mlflow-uri.sh](mlflow-uri.sh)|[![mlflow-uri](https://github.com/Azure/azureml-examples/workflows/cli-scripts-mlflow-uri/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-mlflow-uri.yml) [train-rest.sh](train-rest.sh)|[![train-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-train-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-train-rest.yml) [train.sh](train.sh)|[![train](https://github.com/Azure/azureml-examples/workflows/cli-scripts-train/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-train.yml) +[upload-data-to-blob.sh](upload-data-to-blob.sh)|[![upload-data-to-blob](https://github.com/Azure/azureml-examples/workflows/cli-scripts-upload-data-to-blob/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-upload-data-to-blob.yml) **Jobs** ([jobs](jobs)) @@ -87,7 +81,6 @@ path|status|description [jobs/deepspeed/deepspeed-autotuning/job.yml](jobs/deepspeed/deepspeed-autotuning/job.yml)|[![jobs/deepspeed/deepspeed-autotuning/job](https://github.com/Azure/azureml-examples/workflows/cli-jobs-deepspeed-deepspeed-autotuning-job/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-deepspeed-deepspeed-autotuning-job.yml)|*no description* [jobs/deepspeed/deepspeed-training/job.yml](jobs/deepspeed/deepspeed-training/job.yml)|[![jobs/deepspeed/deepspeed-training/job](https://github.com/Azure/azureml-examples/workflows/cli-jobs-deepspeed-deepspeed-training-job/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-deepspeed-deepspeed-training-job.yml)|*no description* [jobs/nebulaml/PyTorch_CNN_MNIST/job.yml](jobs/nebulaml/PyTorch_CNN_MNIST/job.yml)|[![jobs/nebulaml/PyTorch_CNN_MNIST/job](https://github.com/Azure/azureml-examples/workflows/cli-jobs-nebulaml-PyTorch_CNN_MNIST-job/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-nebulaml-PyTorch_CNN_MNIST-job.yml)|*no description* -[jobs/nebulaml/bert-pretrain-deepspeed/job.yml](jobs/nebulaml/bert-pretrain-deepspeed/job.yml)|[![jobs/nebulaml/bert-pretrain-deepspeed/job](https://github.com/Azure/azureml-examples/workflows/cli-jobs-nebulaml-bert-pretrain-deepspeed-job/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-nebulaml-bert-pretrain-deepspeed-job.yml)|*no description* [jobs/nebulaml/cifar10_deepspeed/job.yml](jobs/nebulaml/cifar10_deepspeed/job.yml)|[![jobs/nebulaml/cifar10_deepspeed/job](https://github.com/Azure/azureml-examples/workflows/cli-jobs-nebulaml-cifar10_deepspeed-job/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-nebulaml-cifar10_deepspeed-job.yml)|*no description* [jobs/pipelines-with-components/nyc_taxi_data_regression/single-job-pipeline.yml](jobs/pipelines-with-components/nyc_taxi_data_regression/single-job-pipeline.yml)|[![jobs/pipelines-with-components/nyc_taxi_data_regression/single-job-pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-with-components-nyc_taxi_data_regression-single-job-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-with-components-nyc_taxi_data_regression-single-job-pipeline.yml)|Single job pipeline to train regression model based on nyc taxi dataset [jobs/single-step/dask/nyctaxi/job.yml](jobs/single-step/dask/nyctaxi/job.yml)|[![jobs/single-step/dask/nyctaxi/job](https://github.com/Azure/azureml-examples/workflows/cli-jobs-single-step-dask-nyctaxi-job/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-single-step-dask-nyctaxi-job.yml)|This sample shows how to run a distributed DASK job on AzureML. The 24GB NYC Taxi dataset is read in CSV format by a 4 node DASK cluster, processed and then written as job output in parquet format. @@ -105,6 +98,7 @@ path|status|description [jobs/single-step/scikit-learn/iris/job-docker-context.yml](jobs/single-step/scikit-learn/iris/job-docker-context.yml)|[![jobs/single-step/scikit-learn/iris/job-docker-context](https://github.com/Azure/azureml-examples/workflows/cli-jobs-single-step-scikit-learn-iris-job-docker-context/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-single-step-scikit-learn-iris-job-docker-context.yml)|Train a scikit-learn SVM on the Iris dataset using a custom Docker container build. [jobs/single-step/scikit-learn/iris/job-sweep.yml](jobs/single-step/scikit-learn/iris/job-sweep.yml)|[![jobs/single-step/scikit-learn/iris/job-sweep](https://github.com/Azure/azureml-examples/workflows/cli-jobs-single-step-scikit-learn-iris-job-sweep/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-single-step-scikit-learn-iris-job-sweep.yml)|Sweep hyperparemeters for training a scikit-learn SVM on the Iris dataset. [jobs/single-step/scikit-learn/iris/job.yml](jobs/single-step/scikit-learn/iris/job.yml)|[![jobs/single-step/scikit-learn/iris/job](https://github.com/Azure/azureml-examples/workflows/cli-jobs-single-step-scikit-learn-iris-job/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-single-step-scikit-learn-iris-job.yml)|Train a scikit-learn SVM on the Iris dataset. +[jobs/single-step/spark/nyctaxi/job.yml](jobs/single-step/spark/nyctaxi/job.yml)|[![jobs/single-step/spark/nyctaxi/job](https://github.com/Azure/azureml-examples/workflows/cli-jobs-single-step-spark-nyctaxi-job/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-single-step-spark-nyctaxi-job.yml)|This sample shows how to run a single node Spark job on Azure ML. The 47GB NYC Taxi dataset is read in parquet format by a 1 node Spark cluster, processed and then written as job output in parquet format. [jobs/single-step/tensorflow/mnist-distributed/job.yml](jobs/single-step/tensorflow/mnist-distributed/job.yml)|[![jobs/single-step/tensorflow/mnist-distributed/job](https://github.com/Azure/azureml-examples/workflows/cli-jobs-single-step-tensorflow-mnist-distributed-job/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-single-step-tensorflow-mnist-distributed-job.yml)|Train a basic neural network with TensorFlow on the MNIST dataset, distributed via TensorFlow. [jobs/single-step/tensorflow/mnist/job.yml](jobs/single-step/tensorflow/mnist/job.yml)|[![jobs/single-step/tensorflow/mnist/job](https://github.com/Azure/azureml-examples/workflows/cli-jobs-single-step-tensorflow-mnist-job/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-single-step-tensorflow-mnist-job.yml)|Train a basic neural network with TensorFlow on the MNIST dataset. [jobs/basics/hello-code.yml](jobs/basics/hello-code.yml)|[![jobs/basics/hello-code](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-code/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-code.yml)|*no description* @@ -121,11 +115,18 @@ path|status|description [jobs/basics/hello-model-as-input.yml](jobs/basics/hello-model-as-input.yml)|[![jobs/basics/hello-model-as-input](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-model-as-input/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-model-as-input.yml)|*no description* [jobs/basics/hello-model-as-output.yml](jobs/basics/hello-model-as-output.yml)|[![jobs/basics/hello-model-as-output](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-model-as-output/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-model-as-output.yml)|*no description* [jobs/basics/hello-notebook.yml](jobs/basics/hello-notebook.yml)|[![jobs/basics/hello-notebook](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-notebook/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-notebook.yml)|*no description* +[jobs/basics/hello-pipeline-abc-serverless.yml](jobs/basics/hello-pipeline-abc-serverless.yml)|[![jobs/basics/hello-pipeline-abc-serverless](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline-abc-serverless/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline-abc-serverless.yml)|*no description* [jobs/basics/hello-pipeline-abc.yml](jobs/basics/hello-pipeline-abc.yml)|[![jobs/basics/hello-pipeline-abc](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline-abc/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline-abc.yml)|*no description* +[jobs/basics/hello-pipeline-customize-output-file-serverless.yml](jobs/basics/hello-pipeline-customize-output-file-serverless.yml)|[![jobs/basics/hello-pipeline-customize-output-file-serverless](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline-customize-output-file-serverless/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline-customize-output-file-serverless.yml)|*no description* [jobs/basics/hello-pipeline-customize-output-file.yml](jobs/basics/hello-pipeline-customize-output-file.yml)|[![jobs/basics/hello-pipeline-customize-output-file](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline-customize-output-file/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline-customize-output-file.yml)|*no description* +[jobs/basics/hello-pipeline-customize-output-folder-serverless.yml](jobs/basics/hello-pipeline-customize-output-folder-serverless.yml)|[![jobs/basics/hello-pipeline-customize-output-folder-serverless](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline-customize-output-folder-serverless/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline-customize-output-folder-serverless.yml)|*no description* [jobs/basics/hello-pipeline-customize-output-folder.yml](jobs/basics/hello-pipeline-customize-output-folder.yml)|[![jobs/basics/hello-pipeline-customize-output-folder](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline-customize-output-folder/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline-customize-output-folder.yml)|*no description* +[jobs/basics/hello-pipeline-default-artifacts-serverless.yml](jobs/basics/hello-pipeline-default-artifacts-serverless.yml)|[![jobs/basics/hello-pipeline-default-artifacts-serverless](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline-default-artifacts-serverless/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline-default-artifacts-serverless.yml)|*no description* [jobs/basics/hello-pipeline-default-artifacts.yml](jobs/basics/hello-pipeline-default-artifacts.yml)|[![jobs/basics/hello-pipeline-default-artifacts](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline-default-artifacts/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline-default-artifacts.yml)|*no description* +[jobs/basics/hello-pipeline-io-serverless.yml](jobs/basics/hello-pipeline-io-serverless.yml)|[![jobs/basics/hello-pipeline-io-serverless](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline-io-serverless/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline-io-serverless.yml)|*no description* [jobs/basics/hello-pipeline-io.yml](jobs/basics/hello-pipeline-io.yml)|[![jobs/basics/hello-pipeline-io](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline-io/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline-io.yml)|*no description* +[jobs/basics/hello-pipeline-serverless.yml](jobs/basics/hello-pipeline-serverless.yml)|[![jobs/basics/hello-pipeline-serverless](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline-serverless/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline-serverless.yml)|*no description* +[jobs/basics/hello-pipeline-settings-serverless.yml](jobs/basics/hello-pipeline-settings-serverless.yml)|[![jobs/basics/hello-pipeline-settings-serverless](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline-settings-serverless/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline-settings-serverless.yml)|*no description* [jobs/basics/hello-pipeline-settings.yml](jobs/basics/hello-pipeline-settings.yml)|[![jobs/basics/hello-pipeline-settings](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline-settings/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline-settings.yml)|*no description* [jobs/basics/hello-pipeline.yml](jobs/basics/hello-pipeline.yml)|[![jobs/basics/hello-pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-pipeline.yml)|*no description* [jobs/basics/hello-sweep.yml](jobs/basics/hello-sweep.yml)|[![jobs/basics/hello-sweep](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-sweep/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-sweep.yml)|Hello sweep job example. @@ -135,14 +136,15 @@ path|status|description [jobs/basics/hello-world-output-data.yml](jobs/basics/hello-world-output-data.yml)|[![jobs/basics/hello-world-output-data](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-world-output-data/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-world-output-data.yml)|*no description* [jobs/basics/hello-world-output.yml](jobs/basics/hello-world-output.yml)|[![jobs/basics/hello-world-output](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-world-output/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-world-output.yml)|*no description* [jobs/basics/hello-world.yml](jobs/basics/hello-world.yml)|[![jobs/basics/hello-world](https://github.com/Azure/azureml-examples/workflows/cli-jobs-basics-hello-world/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-basics-hello-world.yml)|*no description* +[jobs/pipelines/add-column-and-word-count-using-spark/pipeline.yml](jobs/pipelines/add-column-and-word-count-using-spark/pipeline.yml)|[![jobs/pipelines/add-column-and-word-count-using-spark/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-add-column-and-word-count-using-spark-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-add-column-and-word-count-using-spark-pipeline.yml)|'submit a pipeline with spark job' +[jobs/pipelines/automl/cli-aml-demand-forecast-pipeline/cli-aml-demand-forecast-hts/hts_evaluation_pipeline.yml](jobs/pipelines/automl/cli-aml-demand-forecast-pipeline/cli-aml-demand-forecast-hts/hts_evaluation_pipeline.yml)|[![jobs/pipelines/automl/cli-aml-demand-forecast-pipeline/cli-aml-demand-forecast-hts/hts_evaluation_pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-hts-hts_evaluation_pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-hts-hts_evaluation_pipeline.yml)|HTS evaluation pipeline using AutoML +[jobs/pipelines/automl/cli-aml-demand-forecast-pipeline/cli-aml-demand-forecast-mm/many_models_evaluation_pipeline.yml](jobs/pipelines/automl/cli-aml-demand-forecast-pipeline/cli-aml-demand-forecast-mm/many_models_evaluation_pipeline.yml)|[![jobs/pipelines/automl/cli-aml-demand-forecast-pipeline/cli-aml-demand-forecast-mm/many_models_evaluation_pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-mm-many_models_evaluation_pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-cli-aml-demand-forecast-pipeline-cli-aml-demand-forecast-mm-many_models_evaluation_pipeline.yml)|Many models evaluation pipeline using AutoML [jobs/pipelines/automl/cli-automl-classification-task-bankmarketing-pipeline/pipeline.yml](jobs/pipelines/automl/cli-automl-classification-task-bankmarketing-pipeline/pipeline.yml)|[![jobs/pipelines/automl/cli-automl-classification-task-bankmarketing-pipeline/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-cli-automl-classification-task-bankmarketing-pipeline-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-cli-automl-classification-task-bankmarketing-pipeline-pipeline.yml)|Pipeline using AutoML classification task -[jobs/pipelines/automl/cli-automl-forecasting-demand-with-pipeline-components/cli-automl-forecasting-demand-hierarchical-timeseries/hts_evaluation_pipeline.yml](jobs/pipelines/automl/cli-automl-forecasting-demand-with-pipeline-components/cli-automl-forecasting-demand-hierarchical-timeseries/hts_evaluation_pipeline.yml)|[![jobs/pipelines/automl/cli-automl-forecasting-demand-with-pipeline-components/cli-automl-forecasting-demand-hierarchical-timeseries/hts_evaluation_pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-cli-automl-forecasting-demand-with-pipeline-components-cli-automl-forecasting-demand-hierarchical-timeseries-hts_evaluation_pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-cli-automl-forecasting-demand-with-pipeline-components-cli-automl-forecasting-demand-hierarchical-timeseries-hts_evaluation_pipeline.yml)|HTS evaluation pipeline using AutoML -[jobs/pipelines/automl/cli-automl-forecasting-demand-with-pipeline-components/cli-automl-forecasting-demand-many-models/many_models_evaluation_pipeline.yml](jobs/pipelines/automl/cli-automl-forecasting-demand-with-pipeline-components/cli-automl-forecasting-demand-many-models/many_models_evaluation_pipeline.yml)|[![jobs/pipelines/automl/cli-automl-forecasting-demand-with-pipeline-components/cli-automl-forecasting-demand-many-models/many_models_evaluation_pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-cli-automl-forecasting-demand-with-pipeline-components-cli-automl-forecasting-demand-many-models-many_models_evaluation_pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-cli-automl-forecasting-demand-with-pipeline-components-cli-automl-forecasting-demand-many-models-many_models_evaluation_pipeline.yml)|Many models evaluation pipeline using AutoML [jobs/pipelines/automl/cli-automl-regression-housepricing-pipeline/pipeline.yml](jobs/pipelines/automl/cli-automl-regression-housepricing-pipeline/pipeline.yml)|[![jobs/pipelines/automl/cli-automl-regression-housepricing-pipeline/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-cli-automl-regression-housepricing-pipeline-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-cli-automl-regression-housepricing-pipeline-pipeline.yml)|Pipeline using AutoML regression task [jobs/pipelines/automl/cli-automl-text-classification-multilabel-paper-categorization-pipeline/pipeline.yml](jobs/pipelines/automl/cli-automl-text-classification-multilabel-paper-categorization-pipeline/pipeline.yml)|[![jobs/pipelines/automl/cli-automl-text-classification-multilabel-paper-categorization-pipeline/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-cli-automl-text-classification-multilabel-paper-categorization-pipeline-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-cli-automl-text-classification-multilabel-paper-categorization-pipeline-pipeline.yml)|Pipeline using AutoML Text Classification Multilabel task [jobs/pipelines/automl/cli-automl-text-classification-newsgroup-pipeline/pipeline.yml](jobs/pipelines/automl/cli-automl-text-classification-newsgroup-pipeline/pipeline.yml)|[![jobs/pipelines/automl/cli-automl-text-classification-newsgroup-pipeline/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-cli-automl-text-classification-newsgroup-pipeline-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-cli-automl-text-classification-newsgroup-pipeline-pipeline.yml)|Pipeline using AutoML Text Classification task [jobs/pipelines/automl/cli-automl-text-ner-conll-pipeline/pipeline.yml](jobs/pipelines/automl/cli-automl-text-ner-conll-pipeline/pipeline.yml)|[![jobs/pipelines/automl/cli-automl-text-ner-conll-pipeline/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-cli-automl-text-ner-conll-pipeline-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-cli-automl-text-ner-conll-pipeline-pipeline.yml)|Pipeline using AutoML Text Ner task -[jobs/pipelines/automl/forecasting-energy-demand-pipeline/pipeline.yml](jobs/pipelines/automl/forecasting-energy-demand-pipeline/pipeline.yml)|[![jobs/pipelines/automl/forecasting-energy-demand-pipeline/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-forecasting-energy-demand-pipeline-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-forecasting-energy-demand-pipeline-pipeline.yml)|Pipeline using AutoML Forecasting task +[jobs/pipelines/automl/demand-forecast-pipeline/pipeline.yml](jobs/pipelines/automl/demand-forecast-pipeline/pipeline.yml)|[![jobs/pipelines/automl/demand-forecast-pipeline/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-demand-forecast-pipeline-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-demand-forecast-pipeline-pipeline.yml)|Pipeline using AutoML Forecasting task [jobs/pipelines/automl/image-instance-segmentation-task-fridge-items-pipeline/pipeline.yml](jobs/pipelines/automl/image-instance-segmentation-task-fridge-items-pipeline/pipeline.yml)|[![jobs/pipelines/automl/image-instance-segmentation-task-fridge-items-pipeline/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-image-instance-segmentation-task-fridge-items-pipeline-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-image-instance-segmentation-task-fridge-items-pipeline-pipeline.yml)|Pipeline using AutoML Image Instance Segmentation task [jobs/pipelines/automl/image-multiclass-classification-fridge-items-pipeline/pipeline.yml](jobs/pipelines/automl/image-multiclass-classification-fridge-items-pipeline/pipeline.yml)|[![jobs/pipelines/automl/image-multiclass-classification-fridge-items-pipeline/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-image-multiclass-classification-fridge-items-pipeline-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-image-multiclass-classification-fridge-items-pipeline-pipeline.yml)|Pipeline using AutoML Image Multiclass Classification task [jobs/pipelines/automl/image-multilabel-classification-fridge-items-pipeline/pipeline.yml](jobs/pipelines/automl/image-multilabel-classification-fridge-items-pipeline/pipeline.yml)|[![jobs/pipelines/automl/image-multilabel-classification-fridge-items-pipeline/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-automl-image-multilabel-classification-fridge-items-pipeline-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-automl-image-multilabel-classification-fridge-items-pipeline-pipeline.yml)|Pipeline using AutoML Image Multilabel Classification task @@ -152,6 +154,19 @@ path|status|description [jobs/pipelines/mnist-batch-identification-using-parallel/pipeline.yml](jobs/pipelines/mnist-batch-identification-using-parallel/pipeline.yml)|[![jobs/pipelines/mnist-batch-identification-using-parallel/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-mnist-batch-identification-using-parallel-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-mnist-batch-identification-using-parallel-pipeline.yml)|The hello world pipeline job with inline parallel job [jobs/pipelines/nyc-taxi/pipeline.yml](jobs/pipelines/nyc-taxi/pipeline.yml)|[![jobs/pipelines/nyc-taxi/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-nyc-taxi-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-nyc-taxi-pipeline.yml)|Train model with nyc taxi data [jobs/pipelines/tensorflow-image-segmentation/pipeline.yml](jobs/pipelines/tensorflow-image-segmentation/pipeline.yml)|[![jobs/pipelines/tensorflow-image-segmentation/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-tensorflow-image-segmentation-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-tensorflow-image-segmentation-pipeline.yml)|*no description* +[jobs/spark/attached-spark-pipeline-default-identity.yml](jobs/spark/attached-spark-pipeline-default-identity.yml)|[![jobs/spark/attached-spark-pipeline-default-identity](https://github.com/Azure/azureml-examples/workflows/cli-jobs-spark-attached-spark-pipeline-default-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-spark-attached-spark-pipeline-default-identity.yml)|Spark component for Titanic data in Pipeline +[jobs/spark/attached-spark-pipeline-managed-identity.yml](jobs/spark/attached-spark-pipeline-managed-identity.yml)|[![jobs/spark/attached-spark-pipeline-managed-identity](https://github.com/Azure/azureml-examples/workflows/cli-jobs-spark-attached-spark-pipeline-managed-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-spark-attached-spark-pipeline-managed-identity.yml)|Spark component for Titanic data in Pipeline +[jobs/spark/attached-spark-pipeline-user-identity.yml](jobs/spark/attached-spark-pipeline-user-identity.yml)|[![jobs/spark/attached-spark-pipeline-user-identity](https://github.com/Azure/azureml-examples/workflows/cli-jobs-spark-attached-spark-pipeline-user-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-spark-attached-spark-pipeline-user-identity.yml)|Spark component for Titanic data in Pipeline +[jobs/spark/attached-spark-standalone-default-identity.yml](jobs/spark/attached-spark-standalone-default-identity.yml)|[![jobs/spark/attached-spark-standalone-default-identity](https://github.com/Azure/azureml-examples/workflows/cli-jobs-spark-attached-spark-standalone-default-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-spark-attached-spark-standalone-default-identity.yml)|*no description* +[jobs/spark/attached-spark-standalone-managed-identity.yml](jobs/spark/attached-spark-standalone-managed-identity.yml)|[![jobs/spark/attached-spark-standalone-managed-identity](https://github.com/Azure/azureml-examples/workflows/cli-jobs-spark-attached-spark-standalone-managed-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-spark-attached-spark-standalone-managed-identity.yml)|*no description* +[jobs/spark/attached-spark-standalone-user-identity.yml](jobs/spark/attached-spark-standalone-user-identity.yml)|[![jobs/spark/attached-spark-standalone-user-identity](https://github.com/Azure/azureml-examples/workflows/cli-jobs-spark-attached-spark-standalone-user-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-spark-attached-spark-standalone-user-identity.yml)|*no description* +[jobs/spark/serverless-spark-pipeline-default-identity.yml](jobs/spark/serverless-spark-pipeline-default-identity.yml)|[![jobs/spark/serverless-spark-pipeline-default-identity](https://github.com/Azure/azureml-examples/workflows/cli-jobs-spark-serverless-spark-pipeline-default-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-spark-serverless-spark-pipeline-default-identity.yml)|Spark component for Titanic data in Pipeline +[jobs/spark/serverless-spark-pipeline-managed-identity.yml](jobs/spark/serverless-spark-pipeline-managed-identity.yml)|[![jobs/spark/serverless-spark-pipeline-managed-identity](https://github.com/Azure/azureml-examples/workflows/cli-jobs-spark-serverless-spark-pipeline-managed-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-spark-serverless-spark-pipeline-managed-identity.yml)|Spark component for Titanic data in Pipeline +[jobs/spark/serverless-spark-pipeline-user-identity.yml](jobs/spark/serverless-spark-pipeline-user-identity.yml)|[![jobs/spark/serverless-spark-pipeline-user-identity](https://github.com/Azure/azureml-examples/workflows/cli-jobs-spark-serverless-spark-pipeline-user-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-spark-serverless-spark-pipeline-user-identity.yml)|Spark component for Titanic data in Pipeline +[jobs/spark/serverless-spark-standalone-default-identity.yml](jobs/spark/serverless-spark-standalone-default-identity.yml)|[![jobs/spark/serverless-spark-standalone-default-identity](https://github.com/Azure/azureml-examples/workflows/cli-jobs-spark-serverless-spark-standalone-default-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-spark-serverless-spark-standalone-default-identity.yml)|*no description* +[jobs/spark/serverless-spark-standalone-managed-identity.yml](jobs/spark/serverless-spark-standalone-managed-identity.yml)|[![jobs/spark/serverless-spark-standalone-managed-identity](https://github.com/Azure/azureml-examples/workflows/cli-jobs-spark-serverless-spark-standalone-managed-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-spark-serverless-spark-standalone-managed-identity.yml)|*no description* +[jobs/spark/serverless-spark-standalone-user-identity.yml](jobs/spark/serverless-spark-standalone-user-identity.yml)|[![jobs/spark/serverless-spark-standalone-user-identity](https://github.com/Azure/azureml-examples/workflows/cli-jobs-spark-serverless-spark-standalone-user-identity/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-spark-serverless-spark-standalone-user-identity.yml)|*no description* +[jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing/cli-automl-classification-task-bankmarketing-serverless.yml](jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing/cli-automl-classification-task-bankmarketing-serverless.yml)|[![jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing/cli-automl-classification-task-bankmarketing-serverless](https://github.com/Azure/azureml-examples/workflows/cli-jobs-automl-standalone-jobs-cli-automl-classification-task-bankmarketing-cli-automl-classification-task-bankmarketing-serverless/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-automl-standalone-jobs-cli-automl-classification-task-bankmarketing-cli-automl-classification-task-bankmarketing-serverless.yml)|A Classification job using bank marketing [jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing/cli-automl-classification-task-bankmarketing.yml](jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing/cli-automl-classification-task-bankmarketing.yml)|[![jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing/cli-automl-classification-task-bankmarketing](https://github.com/Azure/azureml-examples/workflows/cli-jobs-automl-standalone-jobs-cli-automl-classification-task-bankmarketing-cli-automl-classification-task-bankmarketing/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-automl-standalone-jobs-cli-automl-classification-task-bankmarketing-cli-automl-classification-task-bankmarketing.yml)|A Classification job using bank marketing [jobs/automl-standalone-jobs/cli-automl-forecasting-bike-share/cli-automl-forecasting-task-bike-share.yml](jobs/automl-standalone-jobs/cli-automl-forecasting-bike-share/cli-automl-forecasting-task-bike-share.yml)|[![jobs/automl-standalone-jobs/cli-automl-forecasting-bike-share/cli-automl-forecasting-task-bike-share](https://github.com/Azure/azureml-examples/workflows/cli-jobs-automl-standalone-jobs-cli-automl-forecasting-bike-share-cli-automl-forecasting-task-bike-share/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-automl-standalone-jobs-cli-automl-forecasting-bike-share-cli-automl-forecasting-task-bike-share.yml)|A Time-Series Forecasting job using bike share dataset [jobs/automl-standalone-jobs/cli-automl-forecasting-orange-juice-sales/cli-automl-forecasting-orange-juice-sales.yml](jobs/automl-standalone-jobs/cli-automl-forecasting-orange-juice-sales/cli-automl-forecasting-orange-juice-sales.yml)|[![jobs/automl-standalone-jobs/cli-automl-forecasting-orange-juice-sales/cli-automl-forecasting-orange-juice-sales](https://github.com/Azure/azureml-examples/workflows/cli-jobs-automl-standalone-jobs-cli-automl-forecasting-orange-juice-sales-cli-automl-forecasting-orange-juice-sales/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-automl-standalone-jobs-cli-automl-forecasting-orange-juice-sales-cli-automl-forecasting-orange-juice-sales.yml)|A Time-Series Forecasting job using orange juice sales dataset @@ -171,6 +186,7 @@ path|status|description [jobs/automl-standalone-jobs/cli-automl-text-ner-conll-distributed-sweeping/cli-automl-text-ner-conll2003-distributed-sweeping.yml](jobs/automl-standalone-jobs/cli-automl-text-ner-conll-distributed-sweeping/cli-automl-text-ner-conll2003-distributed-sweeping.yml)|[![jobs/automl-standalone-jobs/cli-automl-text-ner-conll-distributed-sweeping/cli-automl-text-ner-conll2003-distributed-sweeping](https://github.com/Azure/azureml-examples/workflows/cli-jobs-automl-standalone-jobs-cli-automl-text-ner-conll-distributed-sweeping-cli-automl-text-ner-conll2003-distributed-sweeping/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-automl-standalone-jobs-cli-automl-text-ner-conll-distributed-sweeping-cli-automl-text-ner-conll2003-distributed-sweeping.yml)|A text named entity recognition job using CoNLL 2003 data [jobs/automl-standalone-jobs/cli-automl-text-ner-conll/cli-automl-text-ner-conll2003.yml](jobs/automl-standalone-jobs/cli-automl-text-ner-conll/cli-automl-text-ner-conll2003.yml)|[![jobs/automl-standalone-jobs/cli-automl-text-ner-conll/cli-automl-text-ner-conll2003](https://github.com/Azure/azureml-examples/workflows/cli-jobs-automl-standalone-jobs-cli-automl-text-ner-conll-cli-automl-text-ner-conll2003/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-automl-standalone-jobs-cli-automl-text-ner-conll-cli-automl-text-ner-conll2003.yml)|A text named entity recognition job using CoNLL 2003 data [jobs/pipelines-with-components/basics/1a_e2e_local_components/pipeline.yml](jobs/pipelines-with-components/basics/1a_e2e_local_components/pipeline.yml)|[![jobs/pipelines-with-components/basics/1a_e2e_local_components/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-with-components-basics-1a_e2e_local_components-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-with-components-basics-1a_e2e_local_components-pipeline.yml)|Dummy train-score-eval pipeline with local components +[jobs/pipelines-with-components/basics/1b_e2e_registered_components/pipeline-serverless.yml](jobs/pipelines-with-components/basics/1b_e2e_registered_components/pipeline-serverless.yml)|[![jobs/pipelines-with-components/basics/1b_e2e_registered_components/pipeline-serverless](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline-serverless/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline-serverless.yml)|E2E dummy train-score-eval pipeline with registered components [jobs/pipelines-with-components/basics/1b_e2e_registered_components/pipeline.yml](jobs/pipelines-with-components/basics/1b_e2e_registered_components/pipeline.yml)|[![jobs/pipelines-with-components/basics/1b_e2e_registered_components/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-with-components-basics-1b_e2e_registered_components-pipeline.yml)|E2E dummy train-score-eval pipeline with registered components [jobs/pipelines-with-components/basics/2a_basic_component/pipeline.yml](jobs/pipelines-with-components/basics/2a_basic_component/pipeline.yml)|[![jobs/pipelines-with-components/basics/2a_basic_component/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-with-components-basics-2a_basic_component-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-with-components-basics-2a_basic_component-pipeline.yml)|Hello World component example [jobs/pipelines-with-components/basics/2b_component_with_input_output/pipeline.yml](jobs/pipelines-with-components/basics/2b_component_with_input_output/pipeline.yml)|[![jobs/pipelines-with-components/basics/2b_component_with_input_output/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-with-components-basics-2b_component_with_input_output-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-with-components-basics-2b_component_with_input_output-pipeline.yml)|Component with inputs and outputs @@ -196,6 +212,8 @@ path|status|description [jobs/pipelines-with-components/pipeline_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component/train_pipeline/train_pipeline.yml](jobs/pipelines-with-components/pipeline_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component/train_pipeline/train_pipeline.yml)|[![jobs/pipelines-with-components/pipeline_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component/train_pipeline/train_pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-train_pipeline-train_pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-train_pipeline-train_pipeline.yml)|pipeline component with train, predict and score component [jobs/pipelines-with-components/pipeline_with_pipeline_component/pipeline_with_train_eval_pipeline_component/components/train_pipeline_component.yml](jobs/pipelines-with-components/pipeline_with_pipeline_component/pipeline_with_train_eval_pipeline_component/components/train_pipeline_component.yml)|[![jobs/pipelines-with-components/pipeline_with_pipeline_component/pipeline_with_train_eval_pipeline_component/components/train_pipeline_component](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-components-train_pipeline_component/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-components-train_pipeline_component.yml)|Dummy train-score-eval pipeline component with local components [jobs/pipelines-with-components/pipeline_with_pipeline_component/pipeline_with_train_eval_pipeline_component/pipeline.yml](jobs/pipelines-with-components/pipeline_with_pipeline_component/pipeline_with_train_eval_pipeline_component/pipeline.yml)|[![jobs/pipelines-with-components/pipeline_with_pipeline_component/pipeline_with_train_eval_pipeline_component/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-with-components-pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-pipeline.yml)|Select best model trained with different learning rate +[jobs/pipelines-with-components/shakespear_sample_and_word_count_using_spark/pipeline.yml](jobs/pipelines-with-components/shakespear_sample_and_word_count_using_spark/pipeline.yml)|[![jobs/pipelines-with-components/shakespear_sample_and_word_count_using_spark/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-pipelines-with-components-shakespear_sample_and_word_count_using_spark-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-pipelines-with-components-shakespear_sample_and_word_count_using_spark-pipeline.yml)|'submit a pipeline with spark job' +[jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing/cli-automl-classification-task-bankmarketing-serverless.yml](jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing/cli-automl-classification-task-bankmarketing-serverless.yml)|[![jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing/cli-automl-classification-task-bankmarketing-serverless](https://github.com/Azure/azureml-examples/workflows/cli-jobs-automl-standalone-jobs-cli-automl-classification-task-bankmarketing-cli-automl-classification-task-bankmarketing-serverless/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-automl-standalone-jobs-cli-automl-classification-task-bankmarketing-cli-automl-classification-task-bankmarketing-serverless.yml)|A Classification job using bank marketing [jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing/cli-automl-classification-task-bankmarketing.yml](jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing/cli-automl-classification-task-bankmarketing.yml)|[![jobs/automl-standalone-jobs/cli-automl-classification-task-bankmarketing/cli-automl-classification-task-bankmarketing](https://github.com/Azure/azureml-examples/workflows/cli-jobs-automl-standalone-jobs-cli-automl-classification-task-bankmarketing-cli-automl-classification-task-bankmarketing/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-automl-standalone-jobs-cli-automl-classification-task-bankmarketing-cli-automl-classification-task-bankmarketing.yml)|A Classification job using bank marketing [jobs/automl-standalone-jobs/cli-automl-forecasting-bike-share/cli-automl-forecasting-task-bike-share.yml](jobs/automl-standalone-jobs/cli-automl-forecasting-bike-share/cli-automl-forecasting-task-bike-share.yml)|[![jobs/automl-standalone-jobs/cli-automl-forecasting-bike-share/cli-automl-forecasting-task-bike-share](https://github.com/Azure/azureml-examples/workflows/cli-jobs-automl-standalone-jobs-cli-automl-forecasting-bike-share-cli-automl-forecasting-task-bike-share/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-automl-standalone-jobs-cli-automl-forecasting-bike-share-cli-automl-forecasting-task-bike-share.yml)|A Time-Series Forecasting job using bike share dataset [jobs/automl-standalone-jobs/cli-automl-forecasting-orange-juice-sales/cli-automl-forecasting-orange-juice-sales.yml](jobs/automl-standalone-jobs/cli-automl-forecasting-orange-juice-sales/cli-automl-forecasting-orange-juice-sales.yml)|[![jobs/automl-standalone-jobs/cli-automl-forecasting-orange-juice-sales/cli-automl-forecasting-orange-juice-sales](https://github.com/Azure/azureml-examples/workflows/cli-jobs-automl-standalone-jobs-cli-automl-forecasting-orange-juice-sales-cli-automl-forecasting-orange-juice-sales/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-automl-standalone-jobs-cli-automl-forecasting-orange-juice-sales-cli-automl-forecasting-orange-juice-sales.yml)|A Time-Series Forecasting job using orange juice sales dataset @@ -219,13 +237,6 @@ path|status|description [jobs/parallel/1a_oj_sales_prediction/pipeline.yml](jobs/parallel/1a_oj_sales_prediction/pipeline.yml)|[![jobs/parallel/1a_oj_sales_prediction/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-parallel-1a_oj_sales_prediction-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-parallel-1a_oj_sales_prediction-pipeline.yml)|The hello world pipeline job with partition by key [jobs/parallel/2a_iris_batch_prediction/pipeline.yml](jobs/parallel/2a_iris_batch_prediction/pipeline.yml)|[![jobs/parallel/2a_iris_batch_prediction/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-parallel-2a_iris_batch_prediction-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-parallel-2a_iris_batch_prediction-pipeline.yml)|The hello world pipeline job with inline parallel job [jobs/parallel/3a_mnist_batch_identification/pipeline.yml](jobs/parallel/3a_mnist_batch_identification/pipeline.yml)|[![jobs/parallel/3a_mnist_batch_identification/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-parallel-3a_mnist_batch_identification-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-parallel-3a_mnist_batch_identification-pipeline.yml)|The hello world pipeline job with inline parallel job -[finetuning-model-as-a-service-chat-completion](jobs/finetuning/standalone/model-as-a-service/chat-completion/chat-completion-finetuning.yaml)|[![finetuning-model-as-a-service-chat-completion](https://github.com/Azure/azureml-examples/workflows/cli-jobs-finetuning-standalone-model-as-a-service-chat-completion/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-finetuning-standalone-model-as-a-service-chat-completion.yml)|No description -[finetuning-model-as-a-service-text-generation](jobs/finetuning/standalone/model-as-a-service/text-generation/text-generation-finetuning.yaml)|[![finetuning-model-as-a-service-text-generation](https://github.com/Azure/azureml-examples/workflows/cli-jobs-finetuning-standalone-model-as-a-service-text-generation/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-finetuning-standalone-model-as-a-service-text-generation.yml)|No description -[finetuning-model-as-a-platform-chat-completion](jobs/finetuning/standalone/model-as-a-service/chat-completion/chat-completion-finetuning.yaml)|[![finetuning-model-as-a-platform-chat-completion](https://github.com/Azure/azureml-examples/workflows/cli-jobs-finetuning-standalone-model-as-a-platform-chat-completion/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-finetuning-standalone-model-as-a-platform-chat-completion.yml)|No description -[finetuning-model-as-a-platform-text-generation](jobs/finetuning/standalone/model-as-a-service/text-generation/text-generation-finetuning.yaml)|[![finetuning-model-as-a-platform-text-generation](https://github.com/Azure/azureml-examples/workflows/cli-jobs-finetuning-standalone-model-as-a-platform-text-generation/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-finetuning-standalone-model-as-a-platform-text-generation.yml)|No description -[cli\foundation-models\azure_openai\oai-v1\openai_completions_finetune\openai_completions_finetune_spec.yaml](cli\foundation-models\azure_openai\oai-v1\openai_completions_finetune\openai_completions_finetune_spec.yaml)|[![cli\foundation-models\azure_openai\oai-v1\openai_completions_finetune\openai_completions_finetune_spec.yaml](https://github.com/Azure/azureml-examples/workflows/cli-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml)|The hello world pipeline job with inline parallel job -[cli\foundation-models\azure_openai\oai-v2\openai_completions_finetune_pipeline\openai_completions_finetune_pipeline_spec.yaml](cli\foundation-models\azure_openai\oai-v2\openai_completions_finetune_pipeline\openai_completions_finetune_pipeline_spec.yaml)|[![cli\foundation-models\azure_openai\oai-v2\openai_completions_finetune_pipeline\openai_completions_finetune_pipeline_spec.yaml](https://github.com/Azure/azureml-examples/workflows/cli-foundation-models-azure_openai-oai-v2-openai_completions_finetune.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-foundation-models-azure_openai-oai-v2-openai_completions_finetune.yml)|The hello world pipeline job with inline parallel job - **Endpoints** ([endpoints](endpoints)) @@ -236,6 +247,10 @@ path|status|description [endpoints/batch/deploy-models/huggingface-text-summarization/endpoint.yml](endpoints/batch/deploy-models/huggingface-text-summarization/endpoint.yml)|[![endpoints/batch/deploy-models/huggingface-text-summarization/endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-batch-deploy-models-huggingface-text-summarization-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-batch-deploy-models-huggingface-text-summarization-endpoint.yml)|A batch endpoint for summarizing text using a HuggingFace transformer model. [endpoints/batch/deploy-models/imagenet-classifier/endpoint.yml](endpoints/batch/deploy-models/imagenet-classifier/endpoint.yml)|[![endpoints/batch/deploy-models/imagenet-classifier/endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-batch-deploy-models-imagenet-classifier-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-batch-deploy-models-imagenet-classifier-endpoint.yml)|A batch endpoint for performing image classification using a TFHub model ImageNet model. [endpoints/batch/deploy-models/mnist-classifier/endpoint.yml](endpoints/batch/deploy-models/mnist-classifier/endpoint.yml)|[![endpoints/batch/deploy-models/mnist-classifier/endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-batch-deploy-models-mnist-classifier-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-batch-deploy-models-mnist-classifier-endpoint.yml)|A batch endpoint for scoring images from the MNIST dataset. +[endpoints/batch/deploy-models/openai-embeddings/endpoint.yml](endpoints/batch/deploy-models/openai-embeddings/endpoint.yml)|[![endpoints/batch/deploy-models/openai-embeddings/endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-batch-deploy-models-openai-embeddings-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-batch-deploy-models-openai-embeddings-endpoint.yml)|An endpoint to generate embeddings in batch for the ADA-002 model from OpenAI +[endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/endpoint.yml](endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/endpoint.yml)|[![endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-endpoint.yml)|Batch scoring endpoint of the Heart Disease Data Set prediction task. +[endpoints/batch/deploy-pipelines/hello-batch/endpoint.yml](endpoints/batch/deploy-pipelines/hello-batch/endpoint.yml)|[![endpoints/batch/deploy-pipelines/hello-batch/endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-batch-deploy-pipelines-hello-batch-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-batch-deploy-pipelines-hello-batch-endpoint.yml)|A hello world endpoint for component deployments. +[endpoints/batch/deploy-pipelines/training-with-components/endpoint.yml](endpoints/batch/deploy-pipelines/training-with-components/endpoint.yml)|[![endpoints/batch/deploy-pipelines/training-with-components/endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-batch-deploy-pipelines-training-with-components-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-batch-deploy-pipelines-training-with-components-endpoint.yml)|An endpoint to perform training of the Heart Disease Data Set prediction task. [endpoints/online/custom-container/minimal/multimodel/minimal-multimodel-endpoint.yml](endpoints/online/custom-container/minimal/multimodel/minimal-multimodel-endpoint.yml)|[![endpoints/online/custom-container/minimal/multimodel/minimal-multimodel-endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-online-custom-container-minimal-multimodel-minimal-multimodel-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-online-custom-container-minimal-multimodel-minimal-multimodel-endpoint.yml)|*no description* [endpoints/online/custom-container/minimal/single-model/conda-in-dockerfile/minimal-single-model-conda-in-dockerfile-endpoint.yml](endpoints/online/custom-container/minimal/single-model/conda-in-dockerfile/minimal-single-model-conda-in-dockerfile-endpoint.yml)|[![endpoints/online/custom-container/minimal/single-model/conda-in-dockerfile/minimal-single-model-conda-in-dockerfile-endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-online-custom-container-minimal-single-model-conda-in-dockerfile-minimal-single-model-conda-in-dockerfile-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-online-custom-container-minimal-single-model-conda-in-dockerfile-minimal-single-model-conda-in-dockerfile-endpoint.yml)|*no description* [endpoints/online/custom-container/minimal/single-model/minimal-single-model-endpoint.yml](endpoints/online/custom-container/minimal/single-model/minimal-single-model-endpoint.yml)|[![endpoints/online/custom-container/minimal/single-model/minimal-single-model-endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-online-custom-container-minimal-single-model-minimal-single-model-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-online-custom-container-minimal-single-model-minimal-single-model-endpoint.yml)|*no description* @@ -243,6 +258,8 @@ path|status|description [endpoints/online/custom-container/r/multimodel-plumber/r-endpoint.yml](endpoints/online/custom-container/r/multimodel-plumber/r-endpoint.yml)|[![endpoints/online/custom-container/r/multimodel-plumber/r-endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-online-custom-container-r-multimodel-plumber-r-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-online-custom-container-r-multimodel-plumber-r-endpoint.yml)|*no description* [endpoints/online/custom-container/torchserve/densenet/torchserve-endpoint.yml](endpoints/online/custom-container/torchserve/densenet/torchserve-endpoint.yml)|[![endpoints/online/custom-container/torchserve/densenet/torchserve-endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-online-custom-container-torchserve-densenet-torchserve-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-online-custom-container-torchserve-densenet-torchserve-endpoint.yml)|*no description* [endpoints/online/custom-container/triton/single-model/triton-cc-endpoint.yml](endpoints/online/custom-container/triton/single-model/triton-cc-endpoint.yml)|[![endpoints/online/custom-container/triton/single-model/triton-cc-endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-online-custom-container-triton-single-model-triton-cc-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-online-custom-container-triton-single-model-triton-cc-endpoint.yml)|*no description* +[endpoints/online/deploy-with-packages/custom-model/endpoint.yml](endpoints/online/deploy-with-packages/custom-model/endpoint.yml)|[![endpoints/online/deploy-with-packages/custom-model/endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-online-deploy-with-packages-custom-model-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-online-deploy-with-packages-custom-model-endpoint.yml)|*no description* +[endpoints/online/deploy-with-packages/mlflow-model/endpoint.yml](endpoints/online/deploy-with-packages/mlflow-model/endpoint.yml)|[![endpoints/online/deploy-with-packages/mlflow-model/endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-online-deploy-with-packages-mlflow-model-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-online-deploy-with-packages-mlflow-model-endpoint.yml)|*no description* [endpoints/online/kubernetes/kubernetes-endpoint.yml](endpoints/online/kubernetes/kubernetes-endpoint.yml)|[![endpoints/online/kubernetes/kubernetes-endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-online-kubernetes-kubernetes-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-online-kubernetes-kubernetes-endpoint.yml)|*no description* [endpoints/online/managed/sample/endpoint.yml](endpoints/online/managed/sample/endpoint.yml)|[![endpoints/online/managed/sample/endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-online-managed-sample-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-online-managed-sample-endpoint.yml)|*no description* [endpoints/online/managed/vnet/mlflow/endpoint.yml](endpoints/online/managed/vnet/mlflow/endpoint.yml)|[![endpoints/online/managed/vnet/mlflow/endpoint](https://github.com/Azure/azureml-examples/workflows/cli-endpoints-online-managed-vnet-mlflow-endpoint/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-endpoints-online-managed-vnet-mlflow-endpoint.yml)|*no description* @@ -263,8 +280,10 @@ path|status|description path|status|description -|-|- +[assets/component/component_additional_includes.yml](assets/component/component_additional_includes.yml)|[![assets/component/component_additional_includes](https://github.com/Azure/azureml-examples/workflows/cli-assets-component-component_additional_includes/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-component-component_additional_includes.yml)|A example of component using additional includes [assets/component/pipeline.yml](assets/component/pipeline.yml)|[![assets/component/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-assets-component-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-component-pipeline.yml)|pipeline component with inline job [assets/component/train.yml](assets/component/train.yml)|[![assets/component/train](https://github.com/Azure/azureml-examples/workflows/cli-assets-component-train/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-component-train.yml)|A example train component +[assets/data/azure-sql-import.yml](assets/data/azure-sql-import.yml)|[![assets/data/azure-sql-import](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-azure-sql-import/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-azure-sql-import.yml)|*no description* [assets/data/cloud-file-https.yml](assets/data/cloud-file-https.yml)|[![assets/data/cloud-file-https](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-cloud-file-https/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-cloud-file-https.yml)|Data asset created from a file in cloud using https URL. [assets/data/cloud-file-wasbs.yml](assets/data/cloud-file-wasbs.yml)|[![assets/data/cloud-file-wasbs](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-cloud-file-wasbs/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-cloud-file-wasbs.yml)|Data asset created from a file in cloud using wasbs URL. [assets/data/cloud-file.yml](assets/data/cloud-file.yml)|[![assets/data/cloud-file](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-cloud-file/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-cloud-file.yml)|Data asset created from file in cloud. @@ -272,14 +291,18 @@ path|status|description [assets/data/cloud-folder-wasbs.yml](assets/data/cloud-folder-wasbs.yml)|[![assets/data/cloud-folder-wasbs](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-cloud-folder-wasbs/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-cloud-folder-wasbs.yml)|Data asset created from folder in cloud using wasbs URL. [assets/data/cloud-folder.yml](assets/data/cloud-folder.yml)|[![assets/data/cloud-folder](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-cloud-folder/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-cloud-folder.yml)|Data asset created from folder in cloud. [assets/data/cloud-mltable.yml](assets/data/cloud-mltable.yml)|[![assets/data/cloud-mltable](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-cloud-mltable/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-cloud-mltable.yml)|Titanic data +[assets/data/data_import_schedule_database_inline.yml](assets/data/data_import_schedule_database_inline.yml)|[![assets/data/data_import_schedule_database_inline](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-data_import_schedule_database_inline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-data_import_schedule_database_inline.yml)|Schedule data import inline [assets/data/iris-csv-example.yml](assets/data/iris-csv-example.yml)|[![assets/data/iris-csv-example](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-iris-csv-example/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-iris-csv-example.yml)|*no description* [assets/data/local-file.yml](assets/data/local-file.yml)|[![assets/data/local-file](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-local-file/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-local-file.yml)|Data asset created from local file. [assets/data/local-folder-sampledata.yml](assets/data/local-folder-sampledata.yml)|[![assets/data/local-folder-sampledata](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-local-folder-sampledata/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-local-folder-sampledata.yml)|Dataset created from local folder. [assets/data/local-folder.yml](assets/data/local-folder.yml)|[![assets/data/local-folder](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-local-folder/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-local-folder.yml)|Dataset created from local folder. [assets/data/local-mltable.yml](assets/data/local-mltable.yml)|[![assets/data/local-mltable](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-local-mltable/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-local-mltable.yml)|Data asset created from local MLTable. [assets/data/public-file-https.yml](assets/data/public-file-https.yml)|[![assets/data/public-file-https](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-public-file-https/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-public-file-https.yml)|Data asset created from a publicly available file using https URL. +[assets/data/s3-import.yml](assets/data/s3-import.yml)|[![assets/data/s3-import](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-s3-import/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-s3-import.yml)|*no description* +[assets/data/simple_import_schedule.yml](assets/data/simple_import_schedule.yml)|[![assets/data/simple_import_schedule](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-simple_import_schedule/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-simple_import_schedule.yml)|Schedule data import +[assets/data/snowflake-import-managed.yml](assets/data/snowflake-import-managed.yml)|[![assets/data/snowflake-import-managed](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-snowflake-import-managed/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-snowflake-import-managed.yml)|*no description* +[assets/data/snowflake-import.yml](assets/data/snowflake-import.yml)|[![assets/data/snowflake-import](https://github.com/Azure/azureml-examples/workflows/cli-assets-data-snowflake-import/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-data-snowflake-import.yml)|*no description* [assets/environment/docker-context.yml](assets/environment/docker-context.yml)|[![assets/environment/docker-context](https://github.com/Azure/azureml-examples/workflows/cli-assets-environment-docker-context/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-environment-docker-context.yml)|*no description* -[assets/environment/docker-image-plus-conda.yaml](assets/environment/docker-image-plus-conda.yaml)|[![assets/environment/docker-image-plus-conda](https://github.com/Azure/azureml-examples/workflows/cli-assets-environment-docker-image-plus-conda/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-environment-docker-image-plus-conda.yaml)|Environment created from a Docker image plus Conda environment. [assets/environment/docker-image.yml](assets/environment/docker-image.yml)|[![assets/environment/docker-image](https://github.com/Azure/azureml-examples/workflows/cli-assets-environment-docker-image/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-environment-docker-image.yml)|Environment created from a Docker image. [assets/model/local-file.yml](assets/model/local-file.yml)|[![assets/model/local-file](https://github.com/Azure/azureml-examples/workflows/cli-assets-model-local-file/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-model-local-file.yml)|Model created from local file. [assets/model/local-mlflow.yml](assets/model/local-mlflow.yml)|[![assets/model/local-mlflow](https://github.com/Azure/azureml-examples/workflows/cli-assets-model-local-mlflow/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-model-local-mlflow.yml)|Model created from local MLflow model directory. diff --git a/cli/jobs/basics/src/hello-notebook.ipynb b/cli/jobs/basics/src/hello-notebook.ipynb index 9acb14369b..f10d1d6fce 100644 --- a/cli/jobs/basics/src/hello-notebook.ipynb +++ b/cli/jobs/basics/src/hello-notebook.ipynb @@ -21,9 +21,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.8 - AzureML", + "display_name": "Python 3.11 - AzureML", "language": "python", - "name": "python38-azureml" + "name": "python311-azureml" }, "language_info": { "codemirror_mode": { diff --git a/cli/jobs/single-step/scikit-learn/iris-notebook/src/main.ipynb b/cli/jobs/single-step/scikit-learn/iris-notebook/src/main.ipynb index 85c2d731b9..5ba2d9a87b 100644 --- a/cli/jobs/single-step/scikit-learn/iris-notebook/src/main.ipynb +++ b/cli/jobs/single-step/scikit-learn/iris-notebook/src/main.ipynb @@ -91,9 +91,9 @@ "name": "plaintext" }, "kernelspec": { - "display_name": "Python 3.8 - AzureML", + "display_name": "Python 3.11 - AzureML", "language": "python", - "name": "python38-azureml" + "name": "python311-azureml" } }, "nbformat": 4, diff --git a/cli/monitoring/azureml-e2e-model-monitoring/notebooks/model-monitoring-e2e.ipynb b/cli/monitoring/azureml-e2e-model-monitoring/notebooks/model-monitoring-e2e.ipynb index d8d92425bd..c23a9e1457 100644 --- a/cli/monitoring/azureml-e2e-model-monitoring/notebooks/model-monitoring-e2e.ipynb +++ b/cli/monitoring/azureml-e2e-model-monitoring/notebooks/model-monitoring-e2e.ipynb @@ -313,7 +313,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32mUploading code (0.01 MBs): 100%|██████████| 7445/7445 [00:00<00:00, 82192.33it/s]\n", + "\u001b[32mUploading code (0.01 MBs): 100%|\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6| 7445/7445 [00:00<00:00, 82192.33it/s]\n", "\u001b[39m\n", "\n" ] @@ -479,7 +479,7 @@ "output_type": "stream", "text": [ "Check: endpoint credit-default exists\n", - "\u001b[32mUploading code (0.01 MBs): 100%|██████████| 7431/7431 [00:00<00:00, 79754.43it/s]\n", + "\u001b[32mUploading code (0.01 MBs): 100%|\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6\u00e2\u2013\u02c6| 7431/7431 [00:00<00:00, 79754.43it/s]\n", "\u001b[39m\n", "\n" ] @@ -1000,9 +1000,9 @@ "name": "python310-sdkv2" }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.11 - AzureML", "language": "python", - "name": "python3" + "name": "python311-azureml" }, "language_info": { "codemirror_mode": { @@ -1032,4 +1032,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cli/readme.py b/cli/readme.py index 8efc239e4a..093cd9bf81 100644 --- a/cli/readme.py +++ b/cli/readme.py @@ -192,9 +192,9 @@ def main(args): def modify_notebooks(notebooks): # setup variables kernelspec = { - "display_name": "Python 3.8 - AzureML", + "display_name": "Python 3.11 - AzureML", "language": "python", - "name": "python38-azureml", + "name": "python311-azureml", } # for each notebooks @@ -567,7 +567,7 @@ def write_job_using_registry_components_workflow(job): run: | echo '{GITHUB_CONCURRENCY_GROUP}'; bash bootstrap.sh - working-directory: infra + working-directory: infra/bootstrapping continue-on-error: false - name: setup-cli run: | @@ -850,7 +850,7 @@ def write_script_workflow(script): - name: bootstrap resources run: | bash bootstrap.sh - working-directory: infra + working-directory: infra/bootstrapping continue-on-error: false - name: setup-cli run: | @@ -924,7 +924,7 @@ def write_schedule_workflow(schedule): - name: bootstrap resources run: | bash bootstrap.sh - working-directory: infra + working-directory: infra/bootstrapping continue-on-error: false - name: setup-cli run: | diff --git a/sdk/python/README.md b/sdk/python/README.md index f87953757e..0580aeab38 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -39,11 +39,13 @@ Test Status is for branch - **_main_** |assets|assets-in-registry|[share-models-components-environments](assets/assets-in-registry/share-models-components-environments.ipynb)|*no description*|[![share-models-components-environments](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-assets-in-registry-share-models-components-environments.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-assets-in-registry-share-models-components-environments.yml)| |assets|component|[component](assets/component/component.ipynb)|Create a component asset|[![component](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-component-component.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-component-component.yml)| |assets|data|[data](assets/data/data.ipynb)|Read, write and register a data asset|[![data](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-data-data.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-data-data.yml)| -|assets|data|[versioning](assets/data/versioning.ipynb)|Compute and check dataset hash in Azure ML; register if unique for efficient versioning.|[![versioning](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-data-versioning.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-data-versioning.yml)| +|assets|data|[versioning](assets/data/versioning.ipynb)|*no description*|[![versioning](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-data-versioning.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-data-versioning.yml)| |assets|data|[working_with_mltable](assets/data/working_with_mltable.ipynb)|Read, write and register a data asset|[![working_with_mltable](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-data-working_with_mltable.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-data-working_with_mltable.yml)| +|assets|environment|[environment-with-private-package-conda-spec](assets/environment/environment-with-private-packages/environment-with-private-package-conda-spec.ipynb)|Create custom environments from docker and/or conda YAML|[![environment-with-private-package-conda-spec](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-environment-environment-with-private-packages-environment-with-private-package-conda-spec.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-environment-environment-with-private-packages-environment-with-private-package-conda-spec.yml)| +|assets|environment|[environment-with-private-package-docker-image](assets/environment/environment-with-private-packages/environment-with-private-package-docker-image.ipynb)|Create custom environments from docker and/or conda YAML|[![environment-with-private-package-docker-image](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-environment-environment-with-private-packages-environment-with-private-package-docker-image.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-environment-environment-with-private-packages-environment-with-private-package-docker-image.yml)| |assets|environment|[environment](assets/environment/environment.ipynb)|Create custom environments from docker and/or conda YAML|[![environment](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-environment-environment.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-environment-environment.yml)| |assets|model|[model](assets/model/model.ipynb)|Create model from local files, cloud files, Runs|[![model](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-model-model.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-assets-model-model.yml)| -|basic-installation|basic-installation|[basic-installation](basic-installation/basic-installation.ipynb)|Install and do a basic sanity of azure-ai-ml by listing the workspaces available in the resource group.|| +|basic-installation|basic-installation.ipynb|[basic-installation](basic-installation/basic-installation.ipynb)|Create Azure ML workspace|[![basic-installation](https://github.com/Azure/azureml-examples/actions/workflows/sdk-basic-installation-basic-installation.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-basic-installation-basic-installation.yml)| |data-wrangling|interactive_data_wrangling.ipynb|[interactive_data_wrangling](data-wrangling/interactive_data_wrangling.ipynb)|*no description* - _This sample is excluded from automated tests_|[![interactive_data_wrangling](https://github.com/Azure/azureml-examples/actions/workflows/sdk-data-wrangling-interactive_data_wrangling.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-data-wrangling-interactive_data_wrangling.yml)| |endpoints|batch|[custom-output-batch](endpoints/batch/deploy-models/custom-outputs-parquet/custom-output-batch.ipynb)|*no description*|[![custom-output-batch](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-batch-deploy-models-custom-outputs-parquet-custom-output-batch.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-batch-deploy-models-custom-outputs-parquet-custom-output-batch.yml)| |endpoints|batch|[mlflow-for-batch-tabular](endpoints/batch/deploy-models/heart-classifier-mlflow/mlflow-for-batch-tabular.ipynb)|*no description*|[![mlflow-for-batch-tabular](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-batch-deploy-models-heart-classifier-mlflow-mlflow-for-batch-tabular.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-batch-deploy-models-heart-classifier-mlflow-mlflow-for-batch-tabular.yml)| @@ -76,12 +78,32 @@ Test Status is for branch - **_main_** |endpoints|online|[online-endpoints-deploy-mlflow-model-with-script](endpoints/online/mlflow/online-endpoints-deploy-mlflow-model-with-script.ipynb)|Deploy an mlflow model to an online endpoint. This will be a no-code-deployment. It doesn't require scoring script and environment.|[![online-endpoints-deploy-mlflow-model-with-script](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model-with-script.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model-with-script.yml)| |endpoints|online|[online-endpoints-deploy-mlflow-model](endpoints/online/mlflow/online-endpoints-deploy-mlflow-model.ipynb)|Deploy an mlflow model to an online endpoint. This will be a no-code-deployment. It doesn't require scoring script and environment.|[![online-endpoints-deploy-mlflow-model](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model.yml)| |endpoints|online|[online-endpoints-triton](endpoints/online/triton/single-model/online-endpoints-triton.ipynb)|Deploy a custom container as an online endpoint. Use web servers other than the default Python Flask server used by Azure ML without losing the benefits of Azure ML's built-in monitoring, scaling, alerting, and authentication.|[![online-endpoints-triton](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-online-triton-single-model-online-endpoints-triton.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-online-triton-single-model-online-endpoints-triton.yml)| -|environments|environment|[environment](assets/environment/environment.ipynb)|Environments in AML SDK v2 ensure reproducibility and consistency by encapsulating dependencies, packages, and runtime settings for training and inference workflows.| NA | -|environments|environment with private package - conda file|[environment-with-private-package-conda-spec](assets/environment/environment-with-private-packages/environment-with-private-package-conda-spec.ipynb)|Environments in AML SDK v2 ensure reproducibility and consistency by encapsulating dependencies, packages, and runtime settings for training and inference workflows. This example shows how to use private python packages with environments using conda file.| NA | -|environments|environment with private packages - docker image|[environment-with-private-package-docker-image](assets/environment/environment-with-private-packages/environment-with-private-package-docker-image.ipynb)|Environments in AML SDK v2 ensure reproducibility and consistency by encapsulating dependencies, packages, and runtime settings for training and inference workflows.This example shows how to use private python packages with environments using docker image.| NA | |featurestore_sample|automation-test|[test_featurestore_cli_samples](featurestore_sample/automation-test/test_featurestore_cli_samples.ipynb)|*no description*|[![test_featurestore_cli_samples](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-automation-test-test_featurestore_cli_samples.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-automation-test-test_featurestore_cli_samples.yml)| |featurestore_sample|automation-test|[test_featurestore_sdk_samples](featurestore_sample/automation-test/test_featurestore_sdk_samples.ipynb)|*no description*|[![test_featurestore_sdk_samples](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-automation-test-test_featurestore_sdk_samples.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-automation-test-test_featurestore_sdk_samples.yml)| |featurestore_sample|automation-test|[test_featurestore_vnet_samples](featurestore_sample/automation-test/test_featurestore_vnet_samples.ipynb)|*no description*|[![test_featurestore_vnet_samples](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-automation-test-test_featurestore_vnet_samples.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-automation-test-test_featurestore_vnet_samples.yml)| +|featurestore_sample|notebooks|[1.Develop-feature-set-and-register](featurestore_sample/notebooks/sdk_and_cli/1.Develop-feature-set-and-register.ipynb)|*no description*|[![1.Develop-feature-set-and-register](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-1.Develop-feature-set-and-register.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-1.Develop-feature-set-and-register.yml)| +|featurestore_sample|notebooks|[2.Experiment-train-models-using-features](featurestore_sample/notebooks/sdk_and_cli/2.Experiment-train-models-using-features.ipynb)|*no description*|[![2.Experiment-train-models-using-features](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-2.Experiment-train-models-using-features.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-2.Experiment-train-models-using-features.yml)| +|featurestore_sample|notebooks|[3.Enable-recurrent-materialization-run-batch-inference](featurestore_sample/notebooks/sdk_and_cli/3.Enable-recurrent-materialization-run-batch-inference.ipynb)|*no description*|[![3.Enable-recurrent-materialization-run-batch-inference](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-3.Enable-recurrent-materialization-run-batch-inference.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-3.Enable-recurrent-materialization-run-batch-inference.yml)| +|featurestore_sample|notebooks|[4.Provision-feature-store](featurestore_sample/notebooks/sdk_and_cli/4.Provision-feature-store.ipynb)|*no description* - _This sample is excluded from automated tests_|[![4.Provision-feature-store](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-4.Provision-feature-store.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-4.Provision-feature-store.yml)| +|featurestore_sample|notebooks|[Network-isolation-feature-store](featurestore_sample/notebooks/sdk_and_cli/network_isolation/Network-isolation-feature-store.ipynb)|*no description*|[![Network-isolation-feature-store](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-network_isolation-Network-isolation-feature-store.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_and_cli-network_isolation-Network-isolation-feature-store.yml)| +|featurestore_sample|notebooks|[1.Develop-feature-set-and-register](featurestore_sample/notebooks/sdk_only/1.Develop-feature-set-and-register.ipynb)|*no description*|[![1.Develop-feature-set-and-register](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-1.Develop-feature-set-and-register.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-1.Develop-feature-set-and-register.yml)| +|featurestore_sample|notebooks|[2.Experiment-train-models-using-features](featurestore_sample/notebooks/sdk_only/2.Experiment-train-models-using-features.ipynb)|*no description*|[![2.Experiment-train-models-using-features](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-2.Experiment-train-models-using-features.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-2.Experiment-train-models-using-features.yml)| +|featurestore_sample|notebooks|[3.Enable-recurrent-materialization-run-batch-inference](featurestore_sample/notebooks/sdk_only/3.Enable-recurrent-materialization-run-batch-inference.ipynb)|*no description*|[![3.Enable-recurrent-materialization-run-batch-inference](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-3.Enable-recurrent-materialization-run-batch-inference.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-3.Enable-recurrent-materialization-run-batch-inference.yml)| +|featurestore_sample|notebooks|[4.Enable-online-store-run-inference](featurestore_sample/notebooks/sdk_only/4.Enable-online-store-run-inference.ipynb)|*no description*|[![4.Enable-online-store-run-inference](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-4.Enable-online-store-run-inference.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-4.Enable-online-store-run-inference.yml)| +|featurestore_sample|notebooks|[5.Develop-feature-set-custom-source](featurestore_sample/notebooks/sdk_only/5.Develop-feature-set-custom-source.ipynb)|*no description*|[![5.Develop-feature-set-custom-source](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-5.Develop-feature-set-custom-source.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-5.Develop-feature-set-custom-source.yml)| +|featurestore_sample|notebooks|[7.Develop-feature-set-domain-specific-language-dsl](featurestore_sample/notebooks/sdk_only/7.Develop-feature-set-domain-specific-language-dsl.ipynb)|*no description*|[![7.Develop-feature-set-domain-specific-language-dsl](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-7.Develop-feature-set-domain-specific-language-dsl.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-7.Develop-feature-set-domain-specific-language-dsl.yml)| +|featurestore_sample|notebooks|[ADB Example - Develop a feature set and register with managed feature store](featurestore_sample/notebooks/sdk_only/ADB Example - Develop a feature set and register with managed feature store.ipynb)|*no description*|[![ADB Example - Develop a feature set and register with managed feature store](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-ADB Example - Develop a feature set and register with managed feature store.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-ADB Example - Develop a feature set and register with managed feature store.yml)| +|featurestore_sample|notebooks|[Synapse example - Develop feature set and register](featurestore_sample/notebooks/sdk_only/Synapse example - Develop feature set and register.ipynb)|*no description*|[![Synapse example - Develop feature set and register](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-Synapse example - Develop feature set and register.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-featurestore_sample-notebooks-sdk_only-Synapse example - Develop feature set and register.yml)| +|foundation-models|ai21-labs|[ai21_azure_client](foundation-models/ai21-labs/jamba-1-5/ai21_azure_client.ipynb)|*no description*|[![ai21_azure_client](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-ai21_azure_client.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-ai21_azure_client.yml)| +|foundation-models|ai21-labs|[langchain](foundation-models/ai21-labs/jamba-1-5/langchain.ipynb)|*no description*|[![langchain](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-langchain.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-langchain.yml)| +|foundation-models|ai21-labs|[litellm](foundation-models/ai21-labs/jamba-1-5/litellm.ipynb)|*no description*|[![litellm](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-litellm.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-litellm.yml)| +|foundation-models|ai21-labs|[openaisdk](foundation-models/ai21-labs/jamba-1-5/openaisdk.ipynb)|*no description*|[![openaisdk](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-openaisdk.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-openaisdk.yml)| +|foundation-models|ai21-labs|[webrequests](foundation-models/ai21-labs/jamba-1-5/webrequests.ipynb)|*no description*|[![webrequests](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-webrequests.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-1-5-webrequests.yml)| +|foundation-models|ai21-labs|[ai21_client](foundation-models/ai21-labs/jamba-instruct/ai21_client.ipynb)|*no description*|[![ai21_client](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-ai21_client.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-ai21_client.yml)| +|foundation-models|ai21-labs|[langchain](foundation-models/ai21-labs/jamba-instruct/langchain.ipynb)|*no description*|[![langchain](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-langchain.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-langchain.yml)| +|foundation-models|ai21-labs|[litellm](foundation-models/ai21-labs/jamba-instruct/litellm.ipynb)|*no description*|[![litellm](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-litellm.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-litellm.yml)| +|foundation-models|ai21-labs|[openaisdk](foundation-models/ai21-labs/jamba-instruct/openaisdk.ipynb)|*no description*|[![openaisdk](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-openaisdk.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-openaisdk.yml)| +|foundation-models|ai21-labs|[webrequests](foundation-models/ai21-labs/jamba-instruct/webrequests.ipynb)|*no description*|[![webrequests](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-webrequests.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-ai21-labs-jamba-instruct-webrequests.yml)| |foundation-models|azure_openai|[openai_completions_finetune](foundation-models/azure_openai/oai-v1/openai_completions_finetune.ipynb)|*no description*|[![openai_completions_finetune](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml)| |foundation-models|benchmarking|[evaluating_claude_models](foundation-models/benchmarking/evaluating_claude_models.ipynb)|*no description*|[![evaluating_claude_models](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-benchmarking-evaluating_claude_models.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-benchmarking-evaluating_claude_models.yml)| |foundation-models|benchmarking|[evaluating_oai_models](foundation-models/benchmarking/evaluating_oai_models.ipynb)|*no description*|[![evaluating_oai_models](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-benchmarking-evaluating_oai_models.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-benchmarking-evaluating_oai_models.yml)| @@ -95,9 +117,67 @@ Test Status is for branch - **_main_** |foundation-models|cohere|[langchain](foundation-models/cohere/langchain.ipynb)|*no description*|[![langchain](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-langchain.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-langchain.yml)| |foundation-models|cohere|[litellm](foundation-models/cohere/litellm.ipynb)|*no description*|[![litellm](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-litellm.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-litellm.yml)| |foundation-models|cohere|[openaisdk](foundation-models/cohere/openaisdk.ipynb)|*no description*|[![openaisdk](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-openaisdk.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-openaisdk.yml)| +|foundation-models|cohere|[rerank-3-langchain](foundation-models/cohere/rerank-3-langchain.ipynb)|*no description*|[![rerank-3-langchain](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-rerank-3-langchain.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-rerank-3-langchain.yml)| +|foundation-models|cohere|[rerank-cohere-client](foundation-models/cohere/rerank-cohere-client.ipynb)|*no description*|[![rerank-cohere-client](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-rerank-cohere-client.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-rerank-cohere-client.yml)| +|foundation-models|cohere|[rerank-webrequests](foundation-models/cohere/rerank-webrequests.ipynb)|*no description*|[![rerank-webrequests](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-rerank-webrequests.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-rerank-webrequests.yml)| |foundation-models|cohere|[webrequests](foundation-models/cohere/webrequests.ipynb)|*no description*|[![webrequests](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-webrequests.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-cohere-webrequests.yml)| -|foundation-models|azure_openai|[openai_chat_finetune_pipeline](foundation-models/azure_openai/oai-v2/openai_chat_finetune_pipeline.ipynb)|*no description*|[![openai_chat_finetune_pipeline](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-azure_openai-oai-v2-openai_chat_finetune_pipeline.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-azure_openai-oai-v2-openai_chat_finetune_pipeline.yml)| +|foundation-models|healthcare-ai|[cxr-deploy](foundation-models/healthcare-ai/cxrreportgen/cxr-deploy.ipynb)|*no description*|[![cxr-deploy](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-cxrreportgen-cxr-deploy.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-cxrreportgen-cxr-deploy.yml)| +|foundation-models|healthcare-ai|[adapter-training](foundation-models/healthcare-ai/medimageinsight/adapter-training.ipynb)|*no description*|[![adapter-training](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-adapter-training.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-adapter-training.yml)| +|foundation-models|healthcare-ai|[advanced-call-example](foundation-models/healthcare-ai/medimageinsight/advanced-call-example.ipynb)|*no description*|[![advanced-call-example](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-advanced-call-example.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-advanced-call-example.yml)| +|foundation-models|healthcare-ai|[exam-parameter-detection](foundation-models/healthcare-ai/medimageinsight/exam-parameter-demo/exam-parameter-detection.ipynb)|*no description*|[![exam-parameter-detection](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-exam-parameter-demo-exam-parameter-detection.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-exam-parameter-demo-exam-parameter-detection.yml)| +|foundation-models|healthcare-ai|[mi2-deploy](foundation-models/healthcare-ai/medimageinsight/mi2-deploy.ipynb)|*no description*|[![mi2-deploy](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-mi2-deploy.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-mi2-deploy.yml)| +|foundation-models|healthcare-ai|[zero-shot-classification](foundation-models/healthcare-ai/medimageinsight/zero-shot-classification.ipynb)|*no description*|[![zero-shot-classification](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-zero-shot-classification.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageinsight-zero-shot-classification.yml)| +|foundation-models|healthcare-ai|[mip-deploy](foundation-models/healthcare-ai/medimageparse/mip-deploy.ipynb)|*no description*|[![mip-deploy](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageparse-mip-deploy.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageparse-mip-deploy.yml)| +|foundation-models|healthcare-ai|[medimageparse_segmentation_demo](foundation-models/healthcare-ai/medimageparse/sematic-segmentation-demo/medimageparse_segmentation_demo.ipynb)|*no description*|[![medimageparse_segmentation_demo](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageparse-sematic-segmentation-demo-medimageparse_segmentation_demo.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-healthcare-ai-medimageparse-sematic-segmentation-demo-medimageparse_segmentation_demo.yml)| |foundation-models|huggingface|[text-generation-streaming-online-endpoint](foundation-models/huggingface/inference/text-generation-streaming/text-generation-streaming-online-endpoint.ipynb)|*no description*|[![text-generation-streaming-online-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-huggingface-inference-text-generation-streaming-text-generation-streaming-online-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-huggingface-inference-text-generation-streaming-text-generation-streaming-online-endpoint.yml)| +|foundation-models|jais|[litellm](foundation-models/jais/litellm.ipynb)|*no description*|[![litellm](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-jais-litellm.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-jais-litellm.yml)| +|foundation-models|jais|[openaisdk](foundation-models/jais/openaisdk.ipynb)|*no description*|[![openaisdk](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-jais-openaisdk.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-jais-openaisdk.yml)| +|foundation-models|jais|[webrequests](foundation-models/jais/webrequests.ipynb)|*no description*|[![webrequests](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-jais-webrequests.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-jais-webrequests.yml)| +|foundation-models|llama2|[langchain](foundation-models/llama2/langchain.ipynb)|*no description*|[![langchain](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-llama2-langchain.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-llama2-langchain.yml)| +|foundation-models|llama2|[litellm](foundation-models/llama2/litellm.ipynb)|*no description*|[![litellm](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-llama2-litellm.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-llama2-litellm.yml)| +|foundation-models|llama2|[openaisdk](foundation-models/llama2/openaisdk.ipynb)|*no description*|[![openaisdk](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-llama2-openaisdk.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-llama2-openaisdk.yml)| +|foundation-models|llama2|[webrequests](foundation-models/llama2/webrequests.ipynb)|*no description*|[![webrequests](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-llama2-webrequests.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-llama2-webrequests.yml)| +|foundation-models|meta-llama-3.1|[langchain](foundation-models/meta-llama-3.1/langchain.ipynb)|*no description*|[![langchain](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama-3.1-langchain.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama-3.1-langchain.yml)| +|foundation-models|meta-llama-3.1|[litellm](foundation-models/meta-llama-3.1/litellm.ipynb)|*no description*|[![litellm](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama-3.1-litellm.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama-3.1-litellm.yml)| +|foundation-models|meta-llama-3.1|[openaisdk](foundation-models/meta-llama-3.1/openaisdk.ipynb)|*no description*|[![openaisdk](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama-3.1-openaisdk.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama-3.1-openaisdk.yml)| +|foundation-models|meta-llama-3.1|[webrequests](foundation-models/meta-llama-3.1/webrequests.ipynb)|*no description*|[![webrequests](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama-3.1-webrequests.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama-3.1-webrequests.yml)| +|foundation-models|meta-llama3|[langchain](foundation-models/meta-llama3/langchain.ipynb)|*no description*|[![langchain](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama3-langchain.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama3-langchain.yml)| +|foundation-models|meta-llama3|[litellm](foundation-models/meta-llama3/litellm.ipynb)|*no description*|[![litellm](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama3-litellm.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama3-litellm.yml)| +|foundation-models|meta-llama3|[openaisdk](foundation-models/meta-llama3/openaisdk.ipynb)|*no description*|[![openaisdk](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama3-openaisdk.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama3-openaisdk.yml)| +|foundation-models|meta-llama3|[webrequests](foundation-models/meta-llama3/webrequests.ipynb)|*no description*|[![webrequests](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama3-webrequests.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-meta-llama3-webrequests.yml)| +|foundation-models|mistral|[langchain](foundation-models/mistral/langchain.ipynb)|*no description*|[![langchain](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-mistral-langchain.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-mistral-langchain.yml)| +|foundation-models|mistral|[litellm](foundation-models/mistral/litellm.ipynb)|*no description*|[![litellm](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-mistral-litellm.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-mistral-litellm.yml)| +|foundation-models|mistral|[mistralai](foundation-models/mistral/mistralai.ipynb)|*no description*|[![mistralai](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-mistral-mistralai.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-mistral-mistralai.yml)| +|foundation-models|mistral|[openaisdk](foundation-models/mistral/openaisdk.ipynb)|*no description*|[![openaisdk](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-mistral-openaisdk.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-mistral-openaisdk.yml)| +|foundation-models|mistral|[webrequests](foundation-models/mistral/webrequests.ipynb)|*no description*|[![webrequests](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-mistral-webrequests.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-mistral-webrequests.yml)| +|foundation-models|nixtla|[01_quickstart_forecast](foundation-models/nixtla/01_quickstart_forecast.ipynb)|*no description*|[![01_quickstart_forecast](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nixtla-01_quickstart_forecast.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nixtla-01_quickstart_forecast.yml)| +|foundation-models|nixtla|[02_finetuning](foundation-models/nixtla/02_finetuning.ipynb)|*no description*|[![02_finetuning](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nixtla-02_finetuning.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nixtla-02_finetuning.yml)| +|foundation-models|nixtla|[03_anomaly_detection](foundation-models/nixtla/03_anomaly_detection.ipynb)|*no description*|[![03_anomaly_detection](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nixtla-03_anomaly_detection.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nixtla-03_anomaly_detection.yml)| +|foundation-models|nixtla|[04_exogenous_variables](foundation-models/nixtla/04_exogenous_variables.ipynb)|*no description*|[![04_exogenous_variables](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nixtla-04_exogenous_variables.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nixtla-04_exogenous_variables.yml)| +|foundation-models|nixtla|[05_demand_forecasting](foundation-models/nixtla/05_demand_forecasting.ipynb)|*no description*|[![05_demand_forecasting](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nixtla-05_demand_forecasting.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nixtla-05_demand_forecasting.yml)| +|foundation-models|nvidia-nemo|[deploy_infer](foundation-models/nvidia-nemo/deploy_infer.ipynb)|*no description*|[![deploy_infer](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-deploy_infer.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-deploy_infer.yml)| +|foundation-models|nvidia-nemo|[nemo_ptuning_summarization_with_text_gen](foundation-models/nvidia-nemo/evaluate/nemo_ptuning_summarization_with_text_gen.ipynb)|*no description*|[![nemo_ptuning_summarization_with_text_gen](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nemo_ptuning_summarization_with_text_gen.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nemo_ptuning_summarization_with_text_gen.yml)| +|foundation-models|nvidia-nemo|[nvidia-text-classification-evaluation](foundation-models/nvidia-nemo/evaluate/nvidia-text-classification-evaluation.ipynb)|*no description*|[![nvidia-text-classification-evaluation](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-classification-evaluation.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-classification-evaluation.yml)| +|foundation-models|nvidia-nemo|[nvidia-text-gen-evaluation](foundation-models/nvidia-nemo/evaluate/nvidia-text-gen-evaluation.ipynb)|*no description*|[![nvidia-text-gen-evaluation](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-gen-evaluation.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-gen-evaluation.yml)| +|foundation-models|nvidia-nemo|[nvidia-text-qna-evaluation](foundation-models/nvidia-nemo/evaluate/nvidia-text-qna-evaluation.ipynb)|*no description*|[![nvidia-text-qna-evaluation](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-qna-evaluation.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-qna-evaluation.yml)| +|foundation-models|nvidia-nemo|[nvidia-text-translation-evaluation](foundation-models/nvidia-nemo/evaluate/nvidia-text-translation-evaluation.ipynb)|*no description*|[![nvidia-text-translation-evaluation](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-translation-evaluation.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-evaluate-nvidia-text-translation-evaluation.yml)| +|foundation-models|nvidia-nemo|[ptune_evaluate](foundation-models/nvidia-nemo/ptune_evaluate.ipynb)|*no description*|[![ptune_evaluate](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-ptune_evaluate.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nemo-ptune_evaluate.yml)| +|foundation-models|nvidia-nim-llama3-8b|[aiinference](foundation-models/nvidia-nim-llama3-8b/aiinference.ipynb)|*no description*|[![aiinference](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-aiinference.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-aiinference.yml)| +|foundation-models|nvidia-nim-llama3-8b|[openaisdk](foundation-models/nvidia-nim-llama3-8b/openaisdk.ipynb)|*no description*|[![openaisdk](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-openaisdk.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-openaisdk.yml)| +|foundation-models|nvidia-nim-llama3-8b|[webrequests](foundation-models/nvidia-nim-llama3-8b/webrequests.ipynb)|*no description*|[![webrequests](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-webrequests.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-nvidia-nim-llama3-8b-webrequests.yml)| +|foundation-models|phi-3|[litellm](foundation-models/phi-3/litellm.ipynb)|*no description*|[![litellm](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-phi-3-litellm.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-phi-3-litellm.yml)| +|foundation-models|phi-3|[openaisdk](foundation-models/phi-3/openaisdk.ipynb)|*no description*|[![openaisdk](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-phi-3-openaisdk.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-phi-3-openaisdk.yml)| +|foundation-models|phi-3|[webrequests](foundation-models/phi-3/webrequests.ipynb)|*no description*|[![webrequests](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-phi-3-webrequests.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-phi-3-webrequests.yml)| +|foundation-models|stabilityai|[Image_to_Image](foundation-models/stabilityai/Image_to_Image.ipynb)|*no description*|[![Image_to_Image](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-stabilityai-Image_to_Image.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-stabilityai-Image_to_Image.yml)| +|foundation-models|stabilityai|[Text_to_Image_openai_library](foundation-models/stabilityai/Text_to_Image_openai_library.ipynb)|*no description*|[![Text_to_Image_openai_library](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-stabilityai-Text_to_Image_openai_library.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-stabilityai-Text_to_Image_openai_library.yml)| +|foundation-models|stabilityai|[Text_to_Image_requests_library](foundation-models/stabilityai/Text_to_Image_requests_library.ipynb)|*no description*|[![Text_to_Image_requests_library](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-stabilityai-Text_to_Image_requests_library.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-stabilityai-Text_to_Image_requests_library.yml)| +|foundation-models|system|[distillation_conversational_task](foundation-models/system/distillation/conversation/distillation_conversational_task.ipynb)|*no description*|[![distillation_conversational_task](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-distillation-conversation-distillation_conversational_task.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-distillation-conversation-distillation_conversational_task.yml)| +|foundation-models|system|[distillation_math](foundation-models/system/distillation/math/distillation_math.ipynb)|*no description*|[![distillation_math](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-distillation-math-distillation_math.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-distillation-math-distillation_math.yml)| +|foundation-models|system|[distillation_chat_completion](foundation-models/system/distillation/nli/distillation_chat_completion.ipynb)|*no description*|[![distillation_chat_completion](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-distillation-nli-distillation_chat_completion.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-distillation-nli-distillation_chat_completion.yml)| +|foundation-models|system|[distillation_nlu_qa_task](foundation-models/system/distillation/nlu_qa/distillation_nlu_qa_task.ipynb)|*no description*|[![distillation_nlu_qa_task](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-distillation-nlu_qa-distillation_nlu_qa_task.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-distillation-nlu_qa-distillation_nlu_qa_task.yml)| +|foundation-models|system|[distillation_qa_math](foundation-models/system/distillation/nlu_qa/distillation_qa_math.ipynb)|*no description*|[![distillation_qa_math](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-distillation-nlu_qa-distillation_qa_math.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-distillation-nlu_qa-distillation_qa_math.yml)| +|foundation-models|system|[distillation_summarization](foundation-models/system/distillation/summarization/distillation_summarization.ipynb)|*no description*|[![distillation_summarization](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-distillation-summarization-distillation_summarization.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-distillation-summarization-distillation_summarization.yml)| +|foundation-models|system|[chat-completion](foundation-models/system/evaluation/chat-completion/chat-completion.ipynb)|*no description*|[![chat-completion](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-chat-completion-chat-completion.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-chat-completion-chat-completion.yml)| |foundation-models|system|[fill-mask](foundation-models/system/evaluation/fill-mask/fill-mask.ipynb)|*no description*|[![fill-mask](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-fill-mask-fill-mask.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-fill-mask-fill-mask.yml)| |foundation-models|system|[image-multiclass-classification](foundation-models/system/evaluation/image-classification/multiclass-classification/image-multiclass-classification.ipynb)|*no description*|[![image-multiclass-classification](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-image-classification-multiclass-classification-image-multiclass-classification.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-image-classification-multiclass-classification-image-multiclass-classification.yml)| |foundation-models|system|[image-multilabel-classification](foundation-models/system/evaluation/image-classification/multilabel-classification/image-multilabel-classification.ipynb)|*no description*|[![image-multilabel-classification](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-image-classification-multilabel-classification-image-multilabel-classification.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-image-classification-multilabel-classification-image-multilabel-classification.yml)| @@ -110,12 +190,14 @@ Test Status is for branch - **_main_** |foundation-models|system|[text-generation](foundation-models/system/evaluation/text-generation/text-generation.ipynb)|*no description*|[![text-generation](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-text-generation-text-generation.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-text-generation-text-generation.yml)| |foundation-models|system|[news-articles-entity-recognition](foundation-models/system/evaluation/token-classification/news-articles-entity-recognition.ipynb)|*no description*|[![news-articles-entity-recognition](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-token-classification-news-articles-entity-recognition.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-token-classification-news-articles-entity-recognition.yml)| |foundation-models|system|[translation-romanian-to-english](foundation-models/system/evaluation/translation/translation-romanian-to-english.ipynb)|*no description*|[![translation-romanian-to-english](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-translation-translation-romanian-to-english.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-evaluation-translation-translation-romanian-to-english.yml)| +|foundation-models|system|[synthetic-data-generation](foundation-models/system/finetune/Llama-notebooks/datagen/synthetic-data-generation.ipynb)|*no description*|[![synthetic-data-generation](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-datagen-synthetic-data-generation.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-datagen-synthetic-data-generation.yml)| |foundation-models|system|[emotion-detection-llama-multinode-serverless](foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/emotion-detection-llama-multinode-serverless.ipynb)|*no description*|[![emotion-detection-llama-multinode-serverless](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode-serverless.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode-serverless.yml)| |foundation-models|system|[emotion-detection-llama-multinode](foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/emotion-detection-llama-multinode.ipynb)|*no description*|[![emotion-detection-llama-multinode](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-multinode-text-classification-emotion-detection-llama-multinode.yml)| |foundation-models|system|[emotion-detection-llama-serverless-compute](foundation-models/system/finetune/Llama-notebooks/text-classification/emotion-detection-llama-serverless-compute.ipynb)|*no description*|[![emotion-detection-llama-serverless-compute](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama-serverless-compute.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama-serverless-compute.yml)| |foundation-models|system|[emotion-detection-llama](foundation-models/system/finetune/Llama-notebooks/text-classification/emotion-detection-llama.ipynb)|*no description*|[![emotion-detection-llama](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-classification-emotion-detection-llama.yml)| |foundation-models|system|[abstractive_qna_with_text_gen](foundation-models/system/finetune/Llama-notebooks/text-generation/abstractive_qna_with_text_gen.ipynb)|*no description*|[![abstractive_qna_with_text_gen](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-abstractive_qna_with_text_gen.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-abstractive_qna_with_text_gen.yml)| |foundation-models|system|[summarization_with_text_gen](foundation-models/system/finetune/Llama-notebooks/text-generation/summarization_with_text_gen.ipynb)|*no description*|[![summarization_with_text_gen](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-summarization_with_text_gen.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-Llama-notebooks-text-generation-summarization_with_text_gen.yml)| +|foundation-models|system|[chat-completion](foundation-models/system/finetune/chat-completion/chat-completion.ipynb)|*no description*|[![chat-completion](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-chat-completion-chat-completion.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-chat-completion-chat-completion.yml)| |foundation-models|system|[hftransformers-fridgeobjects-multiclass-classification](foundation-models/system/finetune/image-classification/multiclass-classification/hftransformers-fridgeobjects-multiclass-classification.ipynb)|*no description*|[![hftransformers-fridgeobjects-multiclass-classification](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-image-classification-multiclass-classification-hftransformers-fridgeobjects-multiclass-classification.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-image-classification-multiclass-classification-hftransformers-fridgeobjects-multiclass-classification.yml)| |foundation-models|system|[hftransformers-fridgeobjects-multilabel-classification](foundation-models/system/finetune/image-classification/multilabel-classification/hftransformers-fridgeobjects-multilabel-classification.ipynb)|*no description*|[![hftransformers-fridgeobjects-multilabel-classification](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-image-classification-multilabel-classification-hftransformers-fridgeobjects-multilabel-classification.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-image-classification-multilabel-classification-hftransformers-fridgeobjects-multilabel-classification.yml)| |foundation-models|system|[mmdetection-fridgeobjects-instance-segmentation](foundation-models/system/finetune/image-instance-segmentation/mmdetection-fridgeobjects-instance-segmentation.ipynb)|*no description*|[![mmdetection-fridgeobjects-instance-segmentation](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-image-instance-segmentation-mmdetection-fridgeobjects-instance-segmentation.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-image-instance-segmentation-mmdetection-fridgeobjects-instance-segmentation.yml)| @@ -125,6 +207,9 @@ Test Status is for branch - **_main_** |foundation-models|system|[summarization_with_text_gen](foundation-models/system/finetune/scenarios/very-large-model-deepspeed-stage3-finetuning/summarization_with_text_gen.ipynb)|*no description*|[![summarization_with_text_gen](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-scenarios-very-large-model-deepspeed-stage3-finetuning-summarization_with_text_gen.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-scenarios-very-large-model-deepspeed-stage3-finetuning-summarization_with_text_gen.yml)| |foundation-models|system|[news-summary](foundation-models/system/finetune/summarization/news-summary.ipynb)|*no description*|[![news-summary](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-summarization-news-summary.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-summarization-news-summary.yml)| |foundation-models|system|[emotion-detection](foundation-models/system/finetune/text-classification/emotion-detection.ipynb)|*no description*|[![emotion-detection](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-text-classification-emotion-detection.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-text-classification-emotion-detection.yml)| +|foundation-models|system|[abstractive_qna_with_text_gen](foundation-models/system/finetune/text-generation/abstractive_qna_with_text_gen.ipynb)|*no description*|[![abstractive_qna_with_text_gen](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-text-generation-abstractive_qna_with_text_gen.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-text-generation-abstractive_qna_with_text_gen.yml)| +|foundation-models|system|[summarization_with_text_gen](foundation-models/system/finetune/text-generation/summarization_with_text_gen.ipynb)|*no description*|[![summarization_with_text_gen](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-text-generation-summarization_with_text_gen.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-text-generation-summarization_with_text_gen.yml)| +|foundation-models|system|[diffusers-dreambooth-dog-text-to-image](foundation-models/system/finetune/text-to-image/diffusers-dreambooth-dog-text-to-image.ipynb)|*no description*|[![diffusers-dreambooth-dog-text-to-image](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-text-to-image-diffusers-dreambooth-dog-text-to-image.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-text-to-image-diffusers-dreambooth-dog-text-to-image.yml)| |foundation-models|system|[token-classification](foundation-models/system/finetune/token-classification/token-classification.ipynb)|*no description*|[![token-classification](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-token-classification-token-classification.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-token-classification-token-classification.yml)| |foundation-models|system|[translation](foundation-models/system/finetune/translation/translation.ipynb)|*no description*|[![translation](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-translation-translation.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-translation-translation.yml)| |foundation-models|system|[mmtracking-video-multi-object-tracking](foundation-models/system/finetune/video-multi-object-tracking/mmtracking-video-multi-object-tracking.ipynb)|*no description*|[![mmtracking-video-multi-object-tracking](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-video-multi-object-tracking-mmtracking-video-multi-object-tracking.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-finetune-video-multi-object-tracking-mmtracking-video-multi-object-tracking.yml)| @@ -137,6 +222,8 @@ Test Status is for branch - **_main_** |foundation-models|system|[fill-mask-online-endpoint](foundation-models/system/inference/fill-mask/fill-mask-online-endpoint.ipynb)|*no description*|[![fill-mask-online-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-online-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-fill-mask-fill-mask-online-endpoint.yml)| |foundation-models|system|[image-classification-batch-endpoint](foundation-models/system/inference/image-classification/image-classification-batch-endpoint.ipynb)|*no description*|[![image-classification-batch-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-classification-image-classification-batch-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-classification-image-classification-batch-endpoint.yml)| |foundation-models|system|[image-classification-online-endpoint](foundation-models/system/inference/image-classification/image-classification-online-endpoint.ipynb)|*no description*|[![image-classification-online-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-classification-image-classification-online-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-classification-image-classification-online-endpoint.yml)| +|foundation-models|system|[image-embeddings-batch-endpoint](foundation-models/system/inference/image-embeddings/image-embeddings-batch-endpoint.ipynb)|*no description*|[![image-embeddings-batch-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-embeddings-image-embeddings-batch-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-embeddings-image-embeddings-batch-endpoint.yml)| +|foundation-models|system|[image-embeddings-online-endpoint](foundation-models/system/inference/image-embeddings/image-embeddings-online-endpoint.ipynb)|*no description*|[![image-embeddings-online-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-embeddings-image-embeddings-online-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-embeddings-image-embeddings-online-endpoint.yml)| |foundation-models|system|[image-instance-segmentation-batch-endpoint](foundation-models/system/inference/image-instance-segmentation/image-instance-segmentation-batch-endpoint.ipynb)|*no description*|[![image-instance-segmentation-batch-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-batch-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-batch-endpoint.yml)| |foundation-models|system|[image-instance-segmentation-online-endpoint](foundation-models/system/inference/image-instance-segmentation/image-instance-segmentation-online-endpoint.ipynb)|*no description*|[![image-instance-segmentation-online-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-online-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-instance-segmentation-image-instance-segmentation-online-endpoint.yml)| |foundation-models|system|[image-object-detection-batch-endpoint](foundation-models/system/inference/image-object-detection/image-object-detection-batch-endpoint.ipynb)|*no description*|[![image-object-detection-batch-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-object-detection-image-object-detection-batch-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-object-detection-image-object-detection-batch-endpoint.yml)| @@ -148,6 +235,10 @@ Test Status is for branch - **_main_** |foundation-models|system|[image-to-text-online-endpoint](foundation-models/system/inference/image-to-text/image-to-text-online-endpoint.ipynb)|*no description*|[![image-to-text-online-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-to-text-image-to-text-online-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-image-to-text-image-to-text-online-endpoint.yml)| |foundation-models|system|[mask-generation-batch-endpoint](foundation-models/system/inference/mask-generation/mask-generation-batch-endpoint.ipynb)|*no description*|[![mask-generation-batch-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-mask-generation-mask-generation-batch-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-mask-generation-mask-generation-batch-endpoint.yml)| |foundation-models|system|[mask-generation-online-endpoint](foundation-models/system/inference/mask-generation/mask-generation-online-endpoint.ipynb)|*no description*|[![mask-generation-online-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-mask-generation-mask-generation-online-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-mask-generation-mask-generation-online-endpoint.yml)| +|foundation-models|system|[online-endpoint-chat-completions-inference](foundation-models/system/inference/meta-llama-3/meta-llama-3-70b-instruct/online-endpoint-chat-completions-inference.ipynb)|*no description*|[![online-endpoint-chat-completions-inference](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-instruct-online-endpoint-chat-completions-inference.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-instruct-online-endpoint-chat-completions-inference.yml)| +|foundation-models|system|[online-endpoint-completions-inference](foundation-models/system/inference/meta-llama-3/meta-llama-3-70b/online-endpoint-completions-inference.ipynb)|*no description*|[![online-endpoint-completions-inference](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-online-endpoint-completions-inference.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-70b-online-endpoint-completions-inference.yml)| +|foundation-models|system|[online-endpoint-chat-completions-inference](foundation-models/system/inference/meta-llama-3/meta-llama-3-8b-instruct/online-endpoint-chat-completions-inference.ipynb)|*no description*|[![online-endpoint-chat-completions-inference](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-instruct-online-endpoint-chat-completions-inference.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-instruct-online-endpoint-chat-completions-inference.yml)| +|foundation-models|system|[online-endpoint-completions-inference](foundation-models/system/inference/meta-llama-3/meta-llama-3-8b/online-endpoint-completions-inference.ipynb)|*no description*|[![online-endpoint-completions-inference](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-online-endpoint-completions-inference.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-meta-llama-3-meta-llama-3-8b-online-endpoint-completions-inference.yml)| |foundation-models|system|[question-answering-batch-endpoint](foundation-models/system/inference/question-answering/question-answering-batch-endpoint.ipynb)|*no description*|[![question-answering-batch-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-batch-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-batch-endpoint.yml)| |foundation-models|system|[question-answering-online-endpoint-oss](foundation-models/system/inference/question-answering/question-answering-online-endpoint-oss.ipynb)|*no description*|[![question-answering-online-endpoint-oss](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint-oss.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint-oss.yml)| |foundation-models|system|[question-answering-online-endpoint](foundation-models/system/inference/question-answering/question-answering-online-endpoint.ipynb)|*no description*|[![question-answering-online-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-question-answering-question-answering-online-endpoint.yml)| @@ -184,15 +275,22 @@ Test Status is for branch - **_main_** |foundation-models|system|[translation-online-endpoint-oss](foundation-models/system/inference/translation/translation-online-endpoint-oss.ipynb)|*no description*|[![translation-online-endpoint-oss](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-translation-translation-online-endpoint-oss.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-translation-translation-online-endpoint-oss.yml)| |foundation-models|system|[translation-online-endpoint](foundation-models/system/inference/translation/translation-online-endpoint.ipynb)|*no description*|[![translation-online-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-translation-translation-online-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-translation-translation-online-endpoint.yml)| |foundation-models|system|[video-multi-object-tracking-online-endpoint](foundation-models/system/inference/video-multi-object-tracking/video-multi-object-tracking-online-endpoint.ipynb)|*no description*|[![video-multi-object-tracking-online-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-video-multi-object-tracking-video-multi-object-tracking-online-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-video-multi-object-tracking-video-multi-object-tracking-online-endpoint.yml)| +|foundation-models|system|[image-text-to-text-online-endpoint](foundation-models/system/inference/visual-chat-completion/image-text-to-text-online-endpoint.ipynb)|*no description*|[![image-text-to-text-online-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-visual-chat-completion-image-text-to-text-online-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-visual-chat-completion-image-text-to-text-online-endpoint.yml)| |foundation-models|system|[visual-question-answering-batch-endpoint](foundation-models/system/inference/visual-question-answering/visual-question-answering-batch-endpoint.ipynb)|*no description*|[![visual-question-answering-batch-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-batch-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-batch-endpoint.yml)| |foundation-models|system|[visual-question-answering-online-endpoint](foundation-models/system/inference/visual-question-answering/visual-question-answering-online-endpoint.ipynb)|*no description*|[![visual-question-answering-online-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-online-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-visual-question-answering-visual-question-answering-online-endpoint.yml)| |foundation-models|system|[zero-shot-image-classification-batch-endpoint](foundation-models/system/inference/zero-shot-image-classification/zero-shot-image-classification-batch-endpoint.ipynb)|*no description*|[![zero-shot-image-classification-batch-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-batch-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-batch-endpoint.yml)| |foundation-models|system|[zero-shot-image-classification-online-endpoint](foundation-models/system/inference/zero-shot-image-classification/zero-shot-image-classification-online-endpoint.ipynb)|*no description*|[![zero-shot-image-classification-online-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-online-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-zero-shot-image-classification-zero-shot-image-classification-online-endpoint.yml)| |generative-ai|promptflow|[create_faiss_index](generative-ai/promptflow/create_faiss_index.ipynb)|*no description*|[![create_faiss_index](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-promptflow-create_faiss_index.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-promptflow-create_faiss_index.yml)| +|generative-ai|promptflow|[deploy-flow](generative-ai/promptflow/deploy-flow/deploy-flow.ipynb)|*no description*|[![deploy-flow](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-promptflow-deploy-flow-deploy-flow.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-promptflow-deploy-flow-deploy-flow.yml)| |generative-ai|rag|[test](generative-ai/rag/code_first/flows/chat-with-index/src/test.ipynb)|*no description*|[![test](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-code_first-flows-chat-with-index-src-test.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-code_first-flows-chat-with-index-src-test.yml)| |generative-ai|rag|[acs_mlindex_with_langchain](generative-ai/rag/notebooks/azure_cognitive_search/acs_mlindex_with_langchain.ipynb)|*no description*|[![acs_mlindex_with_langchain](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-azure_cognitive_search-acs_mlindex_with_langchain.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-azure_cognitive_search-acs_mlindex_with_langchain.yml)| |generative-ai|rag|[s3_to_acs_mlindex_with_langchain](generative-ai/rag/notebooks/azure_cognitive_search/s3_to_acs_mlindex_with_langchain.ipynb)|*no description*|[![s3_to_acs_mlindex_with_langchain](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-azure_cognitive_search-s3_to_acs_mlindex_with_langchain.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-azure_cognitive_search-s3_to_acs_mlindex_with_langchain.yml)| |generative-ai|rag|[crack_pdfs_with_azure_document_intelligence](generative-ai/rag/notebooks/custom_crack_and_chunk/crack_pdfs_with_azure_document_intelligence.ipynb)|*no description*|[![crack_pdfs_with_azure_document_intelligence](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk-crack_pdfs_with_azure_document_intelligence.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk-crack_pdfs_with_azure_document_intelligence.yml)| +|generative-ai|rag|[crack_and_chunk_with_doc_intel_parallel_component](generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/crack_and_chunk_with_doc_intel_parallel_component.ipynb)|*no description*|[![crack_and_chunk_with_doc_intel_parallel_component](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-crack_and_chunk_with_doc_intel_parallel_component.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-crack_and_chunk_with_doc_intel_parallel_component.yml)| +|generative-ai|rag|[d_parallel_crack_pdfs_with_azure_document_intelligence](generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/d_parallel_crack_pdfs_with_azure_document_intelligence.ipynb)|*no description*|[![d_parallel_crack_pdfs_with_azure_document_intelligence](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-d_parallel_crack_pdfs_with_azure_document_intelligence.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-d_parallel_crack_pdfs_with_azure_document_intelligence.yml)| +|generative-ai|rag|[e_parallel_crack_pdfs_with_azure_document_intelligence](generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/e_parallel_crack_pdfs_with_azure_document_intelligence.ipynb)|*no description*|[![e_parallel_crack_pdfs_with_azure_document_intelligence](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-e_parallel_crack_pdfs_with_azure_document_intelligence.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-e_parallel_crack_pdfs_with_azure_document_intelligence.yml)| +|generative-ai|rag|[f_parallel_crack_pdfs_with_azure_document_intelligence](generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/f_parallel_crack_pdfs_with_azure_document_intelligence.ipynb)|*no description*|[![f_parallel_crack_pdfs_with_azure_document_intelligence](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-f_parallel_crack_pdfs_with_azure_document_intelligence.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-f_parallel_crack_pdfs_with_azure_document_intelligence.yml)| +|generative-ai|rag|[local_parallel_crack_pdfs_with_azure_document_intelligence](generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/local_parallel_crack_pdfs_with_azure_document_intelligence.ipynb)|*no description*|[![local_parallel_crack_pdfs_with_azure_document_intelligence](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-local_parallel_crack_pdfs_with_azure_document_intelligence.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-custom_crack_and_chunk_parallel-local_parallel_crack_pdfs_with_azure_document_intelligence.yml)| |generative-ai|rag|[db_copilot_with_rag](generative-ai/rag/notebooks/db_copilot_with_rag.ipynb)|*no description*|[![db_copilot_with_rag](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-db_copilot_with_rag.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-db_copilot_with_rag.yml)| |generative-ai|rag|[faiss_mlindex_with_langchain](generative-ai/rag/notebooks/faiss/faiss_mlindex_with_langchain.ipynb)|*no description*|[![faiss_mlindex_with_langchain](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-faiss-faiss_mlindex_with_langchain.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-faiss-faiss_mlindex_with_langchain.yml)| |generative-ai|rag|[scheduled_update_faiss_index](generative-ai/rag/notebooks/faiss/scheduled_update_faiss_index.ipynb)|*no description*|[![scheduled_update_faiss_index](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-faiss-scheduled_update_faiss_index.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-faiss-scheduled_update_faiss_index.yml)| @@ -202,11 +300,13 @@ Test Status is for branch - **_main_** |generative-ai|rag|[pinecone_mlindex_with_langchain](generative-ai/rag/notebooks/pinecone/pinecone_mlindex_with_langchain.ipynb)|*no description*|[![pinecone_mlindex_with_langchain](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-pinecone-pinecone_mlindex_with_langchain.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-pinecone-pinecone_mlindex_with_langchain.yml)| |generative-ai|rag|[s3_to_pinecone_mlindex_with_langchain](generative-ai/rag/notebooks/pinecone/s3_to_pinecone_mlindex_with_langchain.ipynb)|*no description*|[![s3_to_pinecone_mlindex_with_langchain](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-pinecone-s3_to_pinecone_mlindex_with_langchain.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-pinecone-s3_to_pinecone_mlindex_with_langchain.yml)| |generative-ai|rag|[qa_data_generation](generative-ai/rag/notebooks/qa_data_generation.ipynb)|*no description*|[![qa_data_generation](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-qa_data_generation.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-generative-ai-rag-notebooks-qa_data_generation.yml)| -|jobs|automl-standalone-jobs|[script_run_notebook](jobs/automl-standalone-jobs/automl-classification-task-bankmarketing/artifact_downloads/outputs/generated_code/script_run_notebook.ipynb)|*no description*|[![script_run_notebook](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-artifact_downloads-outputs-generated_code-script_run_notebook.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-artifact_downloads-outputs-generated_code-script_run_notebook.yml)| |jobs|automl-standalone-jobs|[automl-classification-task-bankmarketing-serverless](jobs/automl-standalone-jobs/automl-classification-task-bankmarketing/automl-classification-task-bankmarketing-serverless.ipynb)|*no description*|[![automl-classification-task-bankmarketing-serverless](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing-serverless.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing-serverless.yml)| |jobs|automl-standalone-jobs|[automl-classification-task-bankmarketing](jobs/automl-standalone-jobs/automl-classification-task-bankmarketing/automl-classification-task-bankmarketing.ipynb)|*no description*|[![automl-classification-task-bankmarketing](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing.yml)| |jobs|automl-standalone-jobs|[mlflow-model-local-inference-test](jobs/automl-standalone-jobs/automl-classification-task-bankmarketing/mlflow-model-local-inference-test.ipynb)|*no description* - _This sample is excluded from automated tests_|[![mlflow-model-local-inference-test](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-mlflow-model-local-inference-test.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-mlflow-model-local-inference-test.yml)| |jobs|automl-standalone-jobs|[automl-forecasting-distributed-tcn](jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/automl-forecasting-distributed-tcn.ipynb)|*no description*|[![automl-forecasting-distributed-tcn](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-distributed-tcn-automl-forecasting-distributed-tcn.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-distributed-tcn-automl-forecasting-distributed-tcn.yml)| +|jobs|automl-standalone-jobs|[auto-ml-forecasting-function-gap-batch-inference](jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-batch-inference.ipynb)|*no description*|[![auto-ml-forecasting-function-gap-batch-inference](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-batch-inference.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-batch-inference.yml)| +|jobs|automl-standalone-jobs|[auto-ml-forecasting-function-gap-local-inference](jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-local-inference.ipynb)|*no description*|[![auto-ml-forecasting-function-gap-local-inference](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-local-inference.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-local-inference.yml)| +|jobs|automl-standalone-jobs|[auto-ml-forecasting-function-gap-training](jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-training.ipynb)|*no description*|[![auto-ml-forecasting-function-gap-training](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-training.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-forecast-function-auto-ml-forecasting-function-gap-training.yml)| |jobs|automl-standalone-jobs|[auto-ml-forecasting-github-dau](jobs/automl-standalone-jobs/automl-forecasting-github-dau/auto-ml-forecasting-github-dau.ipynb)|*no description*|[![auto-ml-forecasting-github-dau](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-github-dau-auto-ml-forecasting-github-dau.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-github-dau-auto-ml-forecasting-github-dau.yml)| |jobs|automl-standalone-jobs|[automl-forecasting-orange-juice-sales-mlflow](jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales/automl-forecasting-orange-juice-sales-mlflow.ipynb)|*no description*|[![automl-forecasting-orange-juice-sales-mlflow](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-orange-juice-sales-automl-forecasting-orange-juice-sales-mlflow.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-orange-juice-sales-automl-forecasting-orange-juice-sales-mlflow.yml)| |jobs|automl-standalone-jobs|[automl-forecasting-recipe-univariate-experiment-settings](jobs/automl-standalone-jobs/automl-forecasting-recipes-univariate/automl-forecasting-recipe-univariate-experiment-settings.ipynb)|*no description*|[![automl-forecasting-recipe-univariate-experiment-settings](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-recipes-univariate-automl-forecasting-recipe-univariate-experiment-settings.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-recipes-univariate-automl-forecasting-recipe-univariate-experiment-settings.yml)| @@ -233,6 +333,13 @@ Test Status is for branch - **_main_** |jobs|automl-standalone-jobs|[automl-nlp-text-ner-task](jobs/automl-standalone-jobs/automl-nlp-text-named-entity-recognition-task/automl-nlp-text-ner-task.ipynb)|*no description*|[![automl-nlp-text-ner-task](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-named-entity-recognition-task-automl-nlp-text-ner-task.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-nlp-text-named-entity-recognition-task-automl-nlp-text-ner-task.yml)| |jobs|automl-standalone-jobs|[automl-regression-task-hardware-performance](jobs/automl-standalone-jobs/automl-regression-task-hardware-performance/automl-regression-task-hardware-performance.ipynb)|*no description*|[![automl-regression-task-hardware-performance](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-regression-task-hardware-performance-automl-regression-task-hardware-performance.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-regression-task-hardware-performance-automl-regression-task-hardware-performance.yml)| |jobs|configuration.ipynb|[configuration](jobs/configuration.ipynb)|Setting up your Azure Machine Learning services workspace and configuring needed resources|[![configuration](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-configuration.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-configuration.yml)| +|jobs|finetuning|[chat_completion_with_model_as_platform](jobs/finetuning/standalone/model-as-a-platform/chat-completion/chat_completion_with_model_as_platform.ipynb)|*no description*|[![chat_completion_with_model_as_platform](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-chat_completion_with_model_as_platform.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-chat_completion_with_model_as_platform.yml)| +|jobs|finetuning|[gpt-oss-20b-chat-completion](jobs/finetuning/standalone/model-as-a-platform/chat-completion/gpt-oss-20b/gpt-oss-20b-chat-completion.ipynb)|*no description*|[![gpt-oss-20b-chat-completion](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-gpt-oss-20b-gpt-oss-20b-chat-completion.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-chat-completion-gpt-oss-20b-gpt-oss-20b-chat-completion.yml)| +|jobs|finetuning|[text_generation_with_model_as_a_platform](jobs/finetuning/standalone/model-as-a-platform/text-generation/text_generation_with_model_as_a_platform.ipynb)|*no description*|[![text_generation_with_model_as_a_platform](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-text-generation-text_generation_with_model_as_a_platform.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-model-as-a-platform-text-generation-text_generation_with_model_as_a_platform.yml)| +|jobs|finetuning|[chat_completion_with_model_as_service](jobs/finetuning/standalone/model-as-a-service/chat-completion/chat_completion_with_model_as_service.ipynb)|*no description*|[![chat_completion_with_model_as_service](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-model-as-a-service-chat-completion-chat_completion_with_model_as_service.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-model-as-a-service-chat-completion-chat_completion_with_model_as_service.yml)| +|jobs|finetuning|[text_generation_with_model_as_a_service](jobs/finetuning/standalone/model-as-a-service/text-generation/text_generation_with_model_as_a_service.ipynb)|*no description*|[![text_generation_with_model_as_a_service](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-model-as-a-service-text-generation-text_generation_with_model_as_a_service.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-model-as-a-service-text-generation-text_generation_with_model_as_a_service.yml)| +|jobs|grpo|[grpo_from_aml_pipeline](jobs/grpo/grpo_from_aml_pipeline.ipynb)|*no description*|[![grpo_from_aml_pipeline](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-grpo-grpo_from_aml_pipeline.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-grpo-grpo_from_aml_pipeline.yml)| +|jobs|grpo|[launch_grpo_command_job-med-mcqa-commented](jobs/grpo/launch_grpo_command_job-med-mcqa-commented.ipynb)|*no description*|[![launch_grpo_command_job-med-mcqa-commented](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-grpo-launch_grpo_command_job-med-mcqa-commented.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-grpo-launch_grpo_command_job-med-mcqa-commented.yml)| |jobs|multicloud-configuration.ipynb|[multicloud-configuration](jobs/multicloud-configuration.ipynb)|Setting up your Azure Machine Learning services workspace and configuring needed resources - _This sample is excluded from automated tests_|[![multicloud-configuration](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-multicloud-configuration.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-multicloud-configuration.yml)| |jobs|parallel|[oj_sales_prediction](jobs/parallel/1a_oj_sales_prediction/oj_sales_prediction.ipynb)|Create pipeline with parallel node to do batch inference|[![oj_sales_prediction](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-parallel-1a_oj_sales_prediction-oj_sales_prediction.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-parallel-1a_oj_sales_prediction-oj_sales_prediction.yml)| |jobs|parallel|[iris_batch_prediction](jobs/parallel/2a_iris_batch_prediction/iris_batch_prediction.ipynb)|Create pipeline with parallel node to do batch inference|[![iris_batch_prediction](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-parallel-2a_iris_batch_prediction-iris_batch_prediction.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-parallel-2a_iris_batch_prediction-iris_batch_prediction.yml)| @@ -257,17 +364,16 @@ Test Status is for branch - **_main_** |jobs|pipelines|[pipeline_with_spark_nodes](jobs/pipelines/1i_pipeline_with_spark_nodes/pipeline_with_spark_nodes.ipynb)|Create pipeline with spark node|[![pipeline_with_spark_nodes](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1i_pipeline_with_spark_nodes-pipeline_with_spark_nodes.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1i_pipeline_with_spark_nodes-pipeline_with_spark_nodes.yml)| |jobs|pipelines|[nyc_taxi_data_regression_with_pipeline_component](jobs/pipelines/1j_pipeline_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component.ipynb)|Create pipeline with CommandComponents from local YAML file|[![nyc_taxi_data_regression_with_pipeline_component](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component.yml)| |jobs|pipelines|[pipeline_with_train_eval_pipeline_component](jobs/pipelines/1j_pipeline_with_pipeline_component/pipeline_with_train_eval_pipeline_component/pipeline_with_train_eval_pipeline_component.ipynb)|Create pipeline with CommandComponents from local YAML file|[![pipeline_with_train_eval_pipeline_component](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-pipeline_with_train_eval_pipeline_component.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-pipeline_with_train_eval_pipeline_component.yml)| -|jobs|pipelines|[automl-forecasting-demand-hts](jobs/pipelines/1k_demand_forecasting_with_pipeline_components/automl-forecasting-demand-hierarchical-timeseries-in-pipeline/automl-forecasting-demand-hts.ipynb)|*no description*|[![automl-forecasting-demand-hts](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecasting_with_pipeline_components-automl-forecasting-demand-hierarchical-timeseries-in-pipeline-automl-forecasting-demand-hts.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecasting_with_pipeline_components-automl-forecasting-demand-hierarchical-timeseries-in-pipeline-automl-forecasting-demand-hts.yml)| -|jobs|pipelines|[automl-forecasting-demand-many-models-in-pipeline](jobs/pipelines/1k_demand_forecasting_with_pipeline_components/automl-forecasting-demand-many-models-in-pipeline/automl-forecasting-demand-many-models-in-pipeline.ipynb)|*no description*|[![automl-forecasting-demand-many-models-in-pipeline](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecasting_with_pipeline_components-automl-forecasting-demand-many-models-in-pipeline-automl-forecasting-demand-many-models-in-pipeline.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecasting_with_pipeline_components-automl-forecasting-demand-many-models-in-pipeline-automl-forecasting-demand-many-models-in-pipeline.yml)| +|jobs|pipelines|[aml-demand-forecast-hts](jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-hts-pipeline/aml-demand-forecast-hts.ipynb)|*no description*|[![aml-demand-forecast-hts](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-hts-pipeline-aml-demand-forecast-hts.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-hts-pipeline-aml-demand-forecast-hts.yml)| +|jobs|pipelines|[aml-demand-forecast-mm-pipeline](jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-mm-pipeline/aml-demand-forecast-mm-pipeline.ipynb)|*no description*|[![aml-demand-forecast-mm-pipeline](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-mm-pipeline-aml-demand-forecast-mm-pipeline.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-mm-pipeline-aml-demand-forecast-mm-pipeline.yml)| |jobs|pipelines|[flow_in_pipeline](jobs/pipelines/1l_flow_in_pipeline/flow_in_pipeline.ipynb)|Create pipeline using components to run a distributed job with tensorflow|[![flow_in_pipeline](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1l_flow_in_pipeline-flow_in_pipeline.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1l_flow_in_pipeline-flow_in_pipeline.yml)| |jobs|pipelines|[train_mnist_with_tensorflow](jobs/pipelines/2a_train_mnist_with_tensorflow/train_mnist_with_tensorflow.ipynb)|Create pipeline using components to run a distributed job with tensorflow|[![train_mnist_with_tensorflow](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2a_train_mnist_with_tensorflow-train_mnist_with_tensorflow.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2a_train_mnist_with_tensorflow-train_mnist_with_tensorflow.yml)| |jobs|pipelines|[train_cifar_10_with_pytorch](jobs/pipelines/2b_train_cifar_10_with_pytorch/train_cifar_10_with_pytorch.ipynb)|Get data, train and evaluate a model in pipeline with Components|[![train_cifar_10_with_pytorch](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2b_train_cifar_10_with_pytorch-train_cifar_10_with_pytorch.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2b_train_cifar_10_with_pytorch-train_cifar_10_with_pytorch.yml)| |jobs|pipelines|[nyc_taxi_data_regression](jobs/pipelines/2c_nyc_taxi_data_regression/nyc_taxi_data_regression.ipynb)|Build pipeline with components for 5 jobs - prep data, transform data, train model, predict results and evaluate model performance|[![nyc_taxi_data_regression](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2c_nyc_taxi_data_regression-nyc_taxi_data_regression.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2c_nyc_taxi_data_regression-nyc_taxi_data_regression.yml)| |jobs|pipelines|[image_classification_with_densenet](jobs/pipelines/2d_image_classification_with_densenet/image_classification_with_densenet.ipynb)|Create pipeline to train cnn image classification model|[![image_classification_with_densenet](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2d_image_classification_with_densenet-image_classification_with_densenet.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2d_image_classification_with_densenet-image_classification_with_densenet.yml)| |jobs|pipelines|[image_classification_keras_minist_convnet](jobs/pipelines/2e_image_classification_keras_minist_convnet/image_classification_keras_minist_convnet.ipynb)|Create pipeline to train cnn image classification model with keras|[![image_classification_keras_minist_convnet](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2e_image_classification_keras_minist_convnet-image_classification_keras_minist_convnet.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2e_image_classification_keras_minist_convnet-image_classification_keras_minist_convnet.yml)| -|jobs|pipelines|[stepsequence](jobs/pipelines/stepsequence/pipeline_with_step_sequence_dummy_dependencies.ipynb)|Create pipeline demonstrating multi-step execution with dummy inputs/outputs, replicating AzureML SDK v1 step sequence behavior|[![stepsequence](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-stepsequence-pipeline_with_step_sequence_dummy_dependencies.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-stepsequence-pipeline_with_step_sequence_dummy_dependencies.yml)| +|jobs|pipelines|[pipeline_with_step_sequence_dummy_dependencies](jobs/pipelines/stepsequence/pipeline_with_step_sequence_dummy_dependencies.ipynb)|*no description*|[![pipeline_with_step_sequence_dummy_dependencies](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-stepsequence-pipeline_with_step_sequence_dummy_dependencies.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-stepsequence-pipeline_with_step_sequence_dummy_dependencies.yml)| |jobs|single-step|[lightgbm-iris-sweep](jobs/single-step/lightgbm/iris/lightgbm-iris-sweep.ipynb)|Run **hyperparameter sweep** on a Command or CommandComponent|[![lightgbm-iris-sweep](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-single-step-lightgbm-iris-lightgbm-iris-sweep.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-single-step-lightgbm-iris-lightgbm-iris-sweep.yml)| -|jobs|single-step|[tutorial](jobs/single-step/pytorch/distributed-training-yolov5/yolov5/tutorial.ipynb)|*no description*|[![tutorial](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-single-step-pytorch-distributed-training-yolov5-yolov5-tutorial.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-single-step-pytorch-distributed-training-yolov5-yolov5-tutorial.yml)| |jobs|single-step|[distributed-cifar10](jobs/single-step/pytorch/distributed-training/distributed-cifar10.ipynb)|*no description*|[![distributed-cifar10](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-single-step-pytorch-distributed-training-distributed-cifar10.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-single-step-pytorch-distributed-training-distributed-cifar10.yml)| |jobs|single-step|[train-hyperparameter-tune-deploy-with-pytorch](jobs/single-step/pytorch/train-hyperparameter-tune-deploy-with-pytorch/train-hyperparameter-tune-deploy-with-pytorch.ipynb)|Train, hyperparameter tune, and deploy a PyTorch model to classify chicken and turkey images to build a deep learning neural network (DNN) based on PyTorch's transfer learning tutorial.|[![train-hyperparameter-tune-deploy-with-pytorch](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-single-step-pytorch-train-hyperparameter-tune-deploy-with-pytorch-train-hyperparameter-tune-deploy-with-pytorch.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-single-step-pytorch-train-hyperparameter-tune-deploy-with-pytorch-train-hyperparameter-tune-deploy-with-pytorch.yml)| |jobs|single-step|[train-hyperparameter-tune-with-sklearn](jobs/single-step/scikit-learn/train-hyperparameter-tune-deploy-with-sklearn/train-hyperparameter-tune-with-sklearn.ipynb)|Train and tune a machine learning model using scikit-learn training scripts to build a to classify iris flower images. - _This sample is excluded from automated tests_|[![train-hyperparameter-tune-with-sklearn](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-single-step-scikit-learn-train-hyperparameter-tune-deploy-with-sklearn-train-hyperparameter-tune-with-sklearn.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-single-step-scikit-learn-train-hyperparameter-tune-deploy-with-sklearn-train-hyperparameter-tune-with-sklearn.yml)| @@ -278,20 +384,17 @@ Test Status is for branch - **_main_** |jobs|spark|[submit_spark_pipeline_jobs](jobs/spark/submit_spark_pipeline_jobs.ipynb)|*no description*|[![submit_spark_pipeline_jobs](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-spark-submit_spark_pipeline_jobs.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-spark-submit_spark_pipeline_jobs.yml)| |jobs|spark|[submit_spark_standalone_jobs](jobs/spark/submit_spark_standalone_jobs.ipynb)|*no description*|[![submit_spark_standalone_jobs](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-spark-submit_spark_standalone_jobs.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-spark-submit_spark_standalone_jobs.yml)| |jobs|spark|[submit_spark_standalone_jobs_managed_vnet](jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb)|*no description*|[![submit_spark_standalone_jobs_managed_vnet](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-spark-submit_spark_standalone_jobs_managed_vnet.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-spark-submit_spark_standalone_jobs_managed_vnet.yml)| -|jobs|finetuning-model-as-a-service|[chat-completion](jobs/finetuning/standalone/model-as-a-service/chat-completion/chat_completion_with_model_as_service.ipynb)|*no description*|[![finetuning-chat-completion](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-chat-completion-maas.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-chat-completion-maas.yml)| -|jobs|finetuning-model-as-a-service|[text-generation](jobs/finetuning/standalone/model-as-a-service/text-generation/text_generation_with_model_as_a_service.ipynb)|*no description*|[![finetuning-text-generation](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-text-generation-maas.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-text-generation-maas.yml)| -|jobs|finetuning-model-as-a-platform|[chat-completion](jobs/finetuning/standalone/model-as-a-platform/chat-completion/chat_completion_with_model_as_platform.ipynb)|*no description*|[![finetuning-chat-completion](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-chat-completion-maap.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-chat-completion-maap.yml)| -|jobs|finetuning-model-as-a-platform|[text-generation](jobs/finetuning/standalone/model-as-a-platform//text-generation/text_generation_with_model_as_a_platform.ipynb.ipynb)|*no description*|[![finetuning-text-generation](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-text-generation-maap.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-finetuning-standalone-text-generation-maap.yml)| |resources|compute|[attach_manage_spark_pools](resources/compute/attach_manage_spark_pools.ipynb)|*no description*|[![attach_manage_spark_pools](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-compute-attach_manage_spark_pools.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-compute-attach_manage_spark_pools.yml)| |resources|compute|[compute](resources/compute/compute.ipynb)|Create compute in Azure ML workspace|[![compute](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-compute-compute.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-compute-compute.yml)| |resources|connections|[connections](resources/connections/connections.ipynb)|*no description*|[![connections](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-connections-connections.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-connections-connections.yml)| |resources|datastores|[datastore](resources/datastores/datastore.ipynb)|Create datastores and use in a Command - _This sample is excluded from automated tests_|[![datastore](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-datastores-datastore.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-datastores-datastore.yml)| |resources|registry|[registry-create](resources/registry/registry-create.ipynb)|*no description*|[![registry-create](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-registry-registry-create.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-registry-registry-create.yml)| +|resources|workspace|[workspace-managed-network](resources/workspace/workspace-managed-network.ipynb)|Create Azure ML workspace|[![workspace-managed-network](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-workspace-workspace-managed-network.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-workspace-workspace-managed-network.yml)| |resources|workspace|[workspace](resources/workspace/workspace.ipynb)|Create Azure ML workspace|[![workspace](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-workspace-workspace.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-resources-workspace-workspace.yml)| |responsible-ai|mlflow-deployment-with-explanations|[mlflow-deployment-with-explanations](responsible-ai/mlflow-deployment-with-explanations/mlflow-deployment-with-explanations.ipynb)|*no description*|[![mlflow-deployment-with-explanations](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-mlflow-deployment-with-explanations-mlflow-deployment-with-explanations.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-mlflow-deployment-with-explanations-mlflow-deployment-with-explanations.yml)| |responsible-ai|tabular|[responsibleaidashboard-diabetes-decision-making](responsible-ai/tabular/responsibleaidashboard-diabetes-decision-making/responsibleaidashboard-diabetes-decision-making.ipynb)|*no description*|[![responsibleaidashboard-diabetes-decision-making](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-diabetes-decision-making-responsibleaidashboard-diabetes-decision-making.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-diabetes-decision-making-responsibleaidashboard-diabetes-decision-making.yml)| |responsible-ai|tabular|[responsibleaidashboard-diabetes-regression-model-debugging](responsible-ai/tabular/responsibleaidashboard-diabetes-regression-model-debugging/responsibleaidashboard-diabetes-regression-model-debugging.ipynb)|*no description*|[![responsibleaidashboard-diabetes-regression-model-debugging](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-diabetes-regression-model-debugging-responsibleaidashboard-diabetes-regression-model-debugging.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-diabetes-regression-model-debugging-responsibleaidashboard-diabetes-regression-model-debugging.yml)| -|responsible-ai|tabular|[responsibleaidashboard-education-student-attrition-classificaton](responsible-ai/tabular/responsibleaidashboard-education-student-attrition-classificaton/responsibleaidashboard-education-student-attrition-classificaton.ipynb)|*no description*|[![responsibleaidashboard-education-student-attrition-classificaton](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-education-student-attrition-classificaton-responsibleaidashboard-education-student-attrition-classificaton.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-education-student-attrition-classificaton-responsibleaidashboard-education-student-attrition-classificaton.yml)| +|responsible-ai|tabular|[responsibleaidashboard-ed-student-attrition-clf](responsible-ai/tabular/responsibleaidashboard-ed-student-attrition-clf/responsibleaidashboard-ed-student-attrition-clf.ipynb)|*no description*|[![responsibleaidashboard-ed-student-attrition-clf](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-ed-student-attrition-clf-responsibleaidashboard-ed-student-attrition-clf.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-ed-student-attrition-clf-responsibleaidashboard-ed-student-attrition-clf.yml)| |responsible-ai|tabular|[responsibleaidashboard-finance-loan-classification](responsible-ai/tabular/responsibleaidashboard-finance-loan-classification/responsibleaidashboard-finance-loan-classification.ipynb)|*no description*|[![responsibleaidashboard-finance-loan-classification](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-finance-loan-classification-responsibleaidashboard-finance-loan-classification.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-finance-loan-classification-responsibleaidashboard-finance-loan-classification.yml)| |responsible-ai|tabular|[responsibleaidashboard-healthcare-covid-classification](responsible-ai/tabular/responsibleaidashboard-healthcare-covid-classification/responsibleaidashboard-healthcare-covid-classification.ipynb)|*no description*|[![responsibleaidashboard-healthcare-covid-classification](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-healthcare-covid-classification-responsibleaidashboard-healthcare-covid-classification.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-healthcare-covid-classification-responsibleaidashboard-healthcare-covid-classification.yml)| |responsible-ai|tabular|[responsibleaidashboard-housing-classification-model-debugging](responsible-ai/tabular/responsibleaidashboard-housing-classification-model-debugging/responsibleaidashboard-housing-classification-model-debugging.ipynb)|*no description*|[![responsibleaidashboard-housing-classification-model-debugging](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-housing-classification-model-debugging-responsibleaidashboard-housing-classification-model-debugging.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-responsible-ai-tabular-responsibleaidashboard-housing-classification-model-debugging-responsibleaidashboard-housing-classification-model-debugging.yml)| @@ -327,4 +430,4 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope ## Reference -* [Documentation](https://docs.microsoft.com/azure/machine-learning) +* [Documentation](https://docs.microsoft.com/azure/machine-learning) \ No newline at end of file diff --git a/sdk/python/assets/data/versioning.ipynb b/sdk/python/assets/data/versioning.ipynb index e3acb0ce0e..006e688fba 100644 --- a/sdk/python/assets/data/versioning.ipynb +++ b/sdk/python/assets/data/versioning.ipynb @@ -292,9 +292,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/assets/environment/environment-with-private-packages/environment-with-private-package-conda-spec.ipynb b/sdk/python/assets/environment/environment-with-private-packages/environment-with-private-package-conda-spec.ipynb index aef1e96182..e78ed587ca 100644 --- a/sdk/python/assets/environment/environment-with-private-packages/environment-with-private-package-conda-spec.ipynb +++ b/sdk/python/assets/environment/environment-with-private-packages/environment-with-private-package-conda-spec.ipynb @@ -193,7 +193,7 @@ "hash": "66962d4c952b5ba37638a017d6cc83bab37d76f69b13c17d86b9f71233a0aa71" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" } diff --git a/sdk/python/assets/environment/environment-with-private-packages/environment-with-private-package-docker-image.ipynb b/sdk/python/assets/environment/environment-with-private-packages/environment-with-private-package-docker-image.ipynb index 8ee5d37a4a..31f9ce9236 100644 --- a/sdk/python/assets/environment/environment-with-private-packages/environment-with-private-package-docker-image.ipynb +++ b/sdk/python/assets/environment/environment-with-private-packages/environment-with-private-package-docker-image.ipynb @@ -198,9 +198,9 @@ "description": "Create custom environments from docker and/or conda YAML" }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "name": "python", diff --git a/sdk/python/basic-installation/basic-installation.ipynb b/sdk/python/basic-installation/basic-installation.ipynb index a610a096bc..a7fe717489 100644 --- a/sdk/python/basic-installation/basic-installation.ipynb +++ b/sdk/python/basic-installation/basic-installation.ipynb @@ -114,7 +114,7 @@ "hash": "66962d4c952b5ba37638a017d6cc83bab37d76f69b13c17d86b9f71233a0aa71" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" } diff --git a/sdk/python/endpoints/batch/deploy-models/custom-outputs-parquet/custom-output-batch.ipynb b/sdk/python/endpoints/batch/deploy-models/custom-outputs-parquet/custom-output-batch.ipynb index 3520178dec..dbaa3e2e5b 100644 --- a/sdk/python/endpoints/batch/deploy-models/custom-outputs-parquet/custom-output-batch.ipynb +++ b/sdk/python/endpoints/batch/deploy-models/custom-outputs-parquet/custom-output-batch.ipynb @@ -865,9 +865,9 @@ "name": "amlv2" }, "kernelspec": { - "display_name": "previews", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "previews" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/endpoints/batch/deploy-models/heart-classifier-mlflow/mlflow-for-batch-tabular.ipynb b/sdk/python/endpoints/batch/deploy-models/heart-classifier-mlflow/mlflow-for-batch-tabular.ipynb index 9b77710ae3..04ef298938 100644 --- a/sdk/python/endpoints/batch/deploy-models/heart-classifier-mlflow/mlflow-for-batch-tabular.ipynb +++ b/sdk/python/endpoints/batch/deploy-models/heart-classifier-mlflow/mlflow-for-batch-tabular.ipynb @@ -1043,9 +1043,9 @@ "name": "python310-sdkv2" }, "kernelspec": { - "display_name": "env", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/endpoints/batch/deploy-models/huggingface-text-summarization/text-summarization-batch.ipynb b/sdk/python/endpoints/batch/deploy-models/huggingface-text-summarization/text-summarization-batch.ipynb index 3f1eaced44..f707dae7bf 100644 --- a/sdk/python/endpoints/batch/deploy-models/huggingface-text-summarization/text-summarization-batch.ipynb +++ b/sdk/python/endpoints/batch/deploy-models/huggingface-text-summarization/text-summarization-batch.ipynb @@ -1080,9 +1080,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.8 - AzureML", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python38-azureml" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/endpoints/batch/deploy-models/mnist-classifier/mnist-batch.ipynb b/sdk/python/endpoints/batch/deploy-models/mnist-classifier/mnist-batch.ipynb index 85b533074d..cf35f19d43 100644 --- a/sdk/python/endpoints/batch/deploy-models/mnist-classifier/mnist-batch.ipynb +++ b/sdk/python/endpoints/batch/deploy-models/mnist-classifier/mnist-batch.ipynb @@ -1325,9 +1325,9 @@ "name": "python3-azureml" }, "kernelspec": { - "display_name": "previews", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "previews" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/endpoints/batch/deploy-models/openai-embeddings/deploy-and-test.ipynb b/sdk/python/endpoints/batch/deploy-models/openai-embeddings/deploy-and-test.ipynb index 6d1a705e3b..c8b79e602a 100644 --- a/sdk/python/endpoints/batch/deploy-models/openai-embeddings/deploy-and-test.ipynb +++ b/sdk/python/endpoints/batch/deploy-models/openai-embeddings/deploy-and-test.ipynb @@ -799,6 +799,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/sdk-deploy-and-test.ipynb b/sdk/python/endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/sdk-deploy-and-test.ipynb index 783a9e91e1..2b26e3a8c9 100644 --- a/sdk/python/endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/sdk-deploy-and-test.ipynb +++ b/sdk/python/endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/sdk-deploy-and-test.ipynb @@ -950,9 +950,9 @@ "name": "amlv2" }, "kernelspec": { - "display_name": "previews", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "previews" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/endpoints/batch/deploy-pipelines/hello-batch/sdk-deploy-and-test.ipynb b/sdk/python/endpoints/batch/deploy-pipelines/hello-batch/sdk-deploy-and-test.ipynb index 0f106d8b24..ecf6424ed6 100644 --- a/sdk/python/endpoints/batch/deploy-pipelines/hello-batch/sdk-deploy-and-test.ipynb +++ b/sdk/python/endpoints/batch/deploy-pipelines/hello-batch/sdk-deploy-and-test.ipynb @@ -574,7 +574,7 @@ "name": "amlv2" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/endpoints/batch/deploy-pipelines/training-with-components/sdk-deploy-and-test.ipynb b/sdk/python/endpoints/batch/deploy-pipelines/training-with-components/sdk-deploy-and-test.ipynb index dcdd915e42..9998191b18 100644 --- a/sdk/python/endpoints/batch/deploy-pipelines/training-with-components/sdk-deploy-and-test.ipynb +++ b/sdk/python/endpoints/batch/deploy-pipelines/training-with-components/sdk-deploy-and-test.ipynb @@ -1143,7 +1143,7 @@ "name": "amlv2" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/endpoints/online/custom-container/online-endpoints-custom-container.ipynb b/sdk/python/endpoints/online/custom-container/online-endpoints-custom-container.ipynb index e4f0fb39ee..19c0a4f535 100644 --- a/sdk/python/endpoints/online/custom-container/online-endpoints-custom-container.ipynb +++ b/sdk/python/endpoints/online/custom-container/online-endpoints-custom-container.ipynb @@ -406,4 +406,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/sdk/python/endpoints/online/custom-container/triton/online-endpoints-triton-cc.ipynb b/sdk/python/endpoints/online/custom-container/triton/online-endpoints-triton-cc.ipynb index c919bc3ec2..b00ff4e55b 100644 --- a/sdk/python/endpoints/online/custom-container/triton/online-endpoints-triton-cc.ipynb +++ b/sdk/python/endpoints/online/custom-container/triton/online-endpoints-triton-cc.ipynb @@ -579,4 +579,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/sdk/python/endpoints/online/deploy-with-packages/custom-model/sdk-deploy-and-test.ipynb b/sdk/python/endpoints/online/deploy-with-packages/custom-model/sdk-deploy-and-test.ipynb index d98bcbea0f..dd8fc62b66 100644 --- a/sdk/python/endpoints/online/deploy-with-packages/custom-model/sdk-deploy-and-test.ipynb +++ b/sdk/python/endpoints/online/deploy-with-packages/custom-model/sdk-deploy-and-test.ipynb @@ -697,7 +697,7 @@ "name": "previews" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/endpoints/online/deploy-with-packages/mlflow-model/sdk-deploy-and-test.ipynb b/sdk/python/endpoints/online/deploy-with-packages/mlflow-model/sdk-deploy-and-test.ipynb index 0fbac1713f..afbd941936 100644 --- a/sdk/python/endpoints/online/deploy-with-packages/mlflow-model/sdk-deploy-and-test.ipynb +++ b/sdk/python/endpoints/online/deploy-with-packages/mlflow-model/sdk-deploy-and-test.ipynb @@ -648,7 +648,7 @@ "name": "previews" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/endpoints/online/deploy-with-packages/registry-model/sdk-deploy-and-test.ipynb b/sdk/python/endpoints/online/deploy-with-packages/registry-model/sdk-deploy-and-test.ipynb index 511b82e490..07e11a6098 100644 --- a/sdk/python/endpoints/online/deploy-with-packages/registry-model/sdk-deploy-and-test.ipynb +++ b/sdk/python/endpoints/online/deploy-with-packages/registry-model/sdk-deploy-and-test.ipynb @@ -623,7 +623,7 @@ "name": "previews" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/endpoints/online/mlflow/online-endpoints-deploy-mlflow-model-with-script.ipynb b/sdk/python/endpoints/online/mlflow/online-endpoints-deploy-mlflow-model-with-script.ipynb index c57ae721db..af43024bef 100644 --- a/sdk/python/endpoints/online/mlflow/online-endpoints-deploy-mlflow-model-with-script.ipynb +++ b/sdk/python/endpoints/online/mlflow/online-endpoints-deploy-mlflow-model-with-script.ipynb @@ -404,7 +404,7 @@ "name": "python310-sdkv2" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/endpoints/online/mlflow/online-endpoints-deploy-mlflow-model.ipynb b/sdk/python/endpoints/online/mlflow/online-endpoints-deploy-mlflow-model.ipynb index 035a97671e..e28a093b10 100644 --- a/sdk/python/endpoints/online/mlflow/online-endpoints-deploy-mlflow-model.ipynb +++ b/sdk/python/endpoints/online/mlflow/online-endpoints-deploy-mlflow-model.ipynb @@ -340,7 +340,7 @@ "name": "python310-sdkv2" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/featurestore_sample/notebooks/sdk_and_cli/1.Develop-feature-set-and-register.ipynb b/sdk/python/featurestore_sample/notebooks/sdk_and_cli/1.Develop-feature-set-and-register.ipynb index e749e166f4..9940085c1c 100644 --- a/sdk/python/featurestore_sample/notebooks/sdk_and_cli/1.Develop-feature-set-and-register.ipynb +++ b/sdk/python/featurestore_sample/notebooks/sdk_and_cli/1.Develop-feature-set-and-register.ipynb @@ -1,1304 +1,1304 @@ { - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# Tutorial #1: Develop a feature set and register with managed feature store" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Azure ML managed feature store lets you discover, create and operationalize features. Features are the connective tissue in ML lifecycle, starting from prototyping where you experiment with various features to operationalization where models are deployed and feature data is looked up during inference. For information on basics concept of feature store, see [feature store concepts](https://learn.microsoft.com/azure/machine-learning/concept-what-is-managed-feature-store).\n", - "\n", - "In this tutorial series you will experience how features seamlessly integrates all the phases of ML lifecycle:\n", - "\n", - "This tutorial is the first part of a three part series. In this tutorial you will:\n", - "- Create a new minimal feature store resource\n", - "- Develop and test featureset locally with feature transformation capability\n", - "- Register a feature-store entity with the feature store\n", - "- Register the featureset that you developed with the feature store\n", - "- Generate sample training data dataframe using the features you created\n", - "- Enable offline materialization on the feature sets, and backfill the feature data\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Prerequisites\n", - "\n", - "> [!NOTE]\n", - "> This tutorial uses Azure Machine Learning notebook with **Serverless Spark Compute**.\n", - "\n", - "Before following the steps in this article, make sure you have the following prerequisites:\n", - "\n", - "* An Azure Machine Learning workspace. If you don't have one, use the steps in the [Quickstart: Create workspace resources](https://learn.microsoft.com/en-us/azure/machine-learning/quickstart-create-resources?view=azureml-api-2) article to create one.\n", - "* To perform the steps in this article, your user account must be assigned the owner or contributor role to a resource group where the feature store will be created" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Note\n", - "This tutorial series has two tracks:\n", - "1. SDK only track: Uses only Python SDKs. This is suitable if pure Python-based development and deployment is preferred.\n", - "1. SDK & CLI track: Uses CLI for CRUD (create, read, update, and delete) operations and Python SDK for feature set development and testing only. This is useful in CI/CD or GitOps scenarios where CLI/YAML is preferred." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Set up\n", - "\n", - "This tutorial uses the Python feature store core SDK (`azureml-featurestore`) for feature set development and testing. The CLI is used for create, read, update, and delete (CRUD) operations, on feature stores, feature sets, and feature store entities.\n", - "\n", - "You don't need to explicitly install these resources for this tutorial, because in the set-up instructions shown here, the `conda.yml` file covers them.\n", - "\n", - "To prepare the notebook environment for development:\n", - "\n", - "1. In the Azure Machine Learning studio environment, select Notebooks on the left pane, and then select the Samples tab.\n", - "\n", - "1. Browse to the featurestore_sample directory (select **Samples** > **SDK v2** > **sdk** > **python** > **featurestore_sample**), and then select **Clone**.\n", - "\n", - "1. The **Select target directory** panel opens. Select the **Users** directory and then select _your user name_, and then select **Clone**.\n", - "\n", - "1. Run the tutorial\n", - "\n", - " * Option 1: Create a new notebook, and execute the instructions in this document, step by step.\n", - " * Option 2: Open existing notebook `featurestore_sample/notebooks/sdk_and_cli/1. Develop a feature set and register with managed feature store.ipynb`. You may keep this document open and refer to it for more explanation and documentation links.\n", - "\n", - "1. To configure the notebook environment, you must upload the conda.yml file\n", - "\n", - " 1. Select **Notebooks** on the left pane, and then select the **Files** tab.\n", - " 1. Browse to the *env* directory (select **Users** > *your_user_name* > **featurestore_sample** > **project** > **env**), and then select the conda.yml file.\n", - " 1. Select **Download**\n", - " 1. Select **Serverless Spark Compute** in the top navigation **Compute** dropdown. This operation might take one to two minutes. Wait for a status bar in the top to display **Configure session**.\n", - " 1. Select **Configure session** in the top status bar.\n", - " 1. Select **Python packages**.\n", - " 1. Select **Upload conda file**.\n", - " 1. Select the `conda.yml` you downloaded on your local device.\n", - " 1. (Optional) Increase the session time-out (idle time in minutes) to reduce the serverless spark cluster startup time.\n", - "\n", - "__Important:__ Except for this step, you need to run all the other steps every time you have a new spark session/session time out.\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Start spark session\n", - "Execute the following code cell to start the Spark session. It wil take approximately 10 minutes to install all dependencies and start the Spark session." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697155816751 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "start-spark-session", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# Run this cell to start the spark session (any code block will start the session). This can take around 10 mins.\n", - "print(\"start spark session\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Setup root directory for the samples\n", - "This code cell sets up the root directory for the samples." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697155915577 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "root-dir", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# Please update your alias belpw (or any custom directory you uploaded the samples to).\n", - "# You can find the name from the directory structure in the left nav\n", - "root_dir = \"./Users//featurestore_sample\"\n", - "\n", - "if os.path.isdir(root_dir):\n", - " print(\"The folder exists.\")\n", - "else:\n", - " print(\"The folder does not exist. Please create or fix the path\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Setup CLI\n", - "\n", - "1. Install AzureML CLI extention\n", - "1. Authenticate\n", - "1. Set the default subscription" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697155966146 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "install-ml-ext-cli", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "# Install AzureML CLI extension\n", - "!az extension add --name ml" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697156018290 - }, - "jupyter": { - "outputs_hidden": true, - "source_hidden": false - }, - "name": "auth-cli", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "# Authenticate\n", - "!az login" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697156029362 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "set-default-subs-cli", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# Set default subscription\n", - "import os\n", - "\n", - "subscription_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", - "\n", - "!az account set -s $subscription_id" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Note\n", - "Feature store Vs Project workspace: You will use a feature store to reuse features across projects. You will use a project workspace (the current workspace) to train and inference models, by leveraging features from feature stores. Many project workspaces can share and reuse a same feature store.\n", - "\n", - "## Note\n", - "In this tutorial you will be using CLI and feature store core SDK:\n", - "\n", - "1. CLI: You will use CLI for CRUD (create, read, update, and delete) operations on feature store, feature set and feature store entities.\n", - "2. Feature store core SDK: This SDK (`azureml-featurestore`) is meant to be used for feature set development and consumption (you will learn more about these operations later):\n", - " - List/Get registered feature set\n", - " - Generate/resolve feature retrieval specification\n", - " - Execute featureset definition to generate Spark dataframe\n", - " - Generate training data using a point-in-time join\n", - "\n", - "For this tutorial so you do not need to install any of these explicitly, since the instructions already cover them (`conda.yml` in the above step include these)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 1: Create a minimal feature store" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 1a: Set feature store parameters\n", - "Set name, location and other values for the feature store." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697156060740 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "fs-params", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# We use the subscription, resource group, region of this active project workspace.\n", - "# You can optionally replace them to create the resources in a different subsciprtion/resourceGroup, or use existing resources\n", - "import os\n", - "\n", - "featurestore_name = \"\"\n", - "featurestore_location = \"eastus\"\n", - "featurestore_subscription_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", - "featurestore_resource_group_name = os.environ[\"AZUREML_ARM_RESOURCEGROUP\"]\n", - "\n", - "feature_store_arm_id = \"/subscriptions/{sub_id}/resourceGroups/{rg}/providers/Microsoft.MachineLearningServices/workspaces/{ws_name}\".format(\n", - " sub_id=featurestore_subscription_id,\n", - " rg=featurestore_resource_group_name,\n", - " ws_name=featurestore_name,\n", - ")\n", - "\n", - "print(feature_store_arm_id)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 1b: Create the feature store" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1683415482376 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "create-fs-cli", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "!az ml feature-store create --subscription $featurestore_subscription_id --resource-group $featurestore_resource_group_name --location $featurestore_location --name $featurestore_name" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 1c: Initialize AzureML feature store core SDK client\n", - "As explained above, this is used to develop and consume features." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697156242248 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "init-fs-core-sdk", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# feature store client\n", - "from azureml.featurestore import FeatureStoreClient\n", - "from azure.ai.ml.identity import AzureMLOnBehalfOfCredential\n", - "\n", - "featurestore = FeatureStoreClient(\n", - " credential=AzureMLOnBehalfOfCredential(),\n", - " subscription_id=featurestore_subscription_id,\n", - " resource_group_name=featurestore_resource_group_name,\n", - " name=featurestore_name,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Step 1d. Grant \"AzureML Data Scientist\" role on the feature store to your user identity" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- Get your AAD object id from Azure portal following this instruction: https://learn.microsoft.com/en-us/partner-center/find-ids-and-domain-names#find-the-user-object-id\\\n", - "\n", - "Assign **AzureML Data Scientist** role to your user identity so that it can create resources in feature store workspace. Please note that it may take some time for the permissions to propagate." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "assign-aad-ds-role-cli" - }, - "outputs": [], - "source": [ - "your_aad_objectid = \"\"\n", - "\n", - "!az role assignment create --role \"AzureML Data Scientist\" --assignee-object-id $your_aad_objectid --assignee-principal-type User --scope $feature_store_arm_id" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 2: Prototype and develop a transaction rolling aggregation feature set in this notebook" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 2a: Explore the transactions source data\n", - "\n", - "#### Note\n", - "The sample data used in this notebook is hosted in a public accessible blob container. It can only be read in Spark via `wasbs` driver. When you create feature sets using your own source data, please host them in ADLS Gen2 account and use `abfss` driver in the data path. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697156282781 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "explore-txn-src-data", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# Remove the \".\" in the roor directory path as we need to generate absolute path to read from Spark.\n", - "transactions_source_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/transactions-source/*.parquet\"\n", - "transactions_src_df = spark.read.parquet(transactions_source_data_path)\n", - "\n", - "display(transactions_src_df.head(5))\n", - "# Note: display(training_df.head(5)) displays the timestamp column in a different format. You can can call transactions_src_df.show() to see correctly formatted value" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 2b: Develop a transactions feature set locally\n", - "\n", - "Feature set specification is a self-contained definition of feature set that can be developed and tested locally.\n", - "\n", - "Lets create the following rolling window aggregate features:\n", - "- transactions 3-day count\n", - "- transactions amount 3-day sum\n", - "- transactions amount 3-day avg\n", - "- transactions 7-day count\n", - "- transactions amount 7-day sum\n", - "- transactions amount 7-day avg\n", - "\n", - "__Action__:\n", - "- Inspect the feature transformation code file: `featurestore/featuresets/transactions/spec/transformation_code/transaction_transform.py`. You will see how is the rolling aggregation defined for the features. This is a Spark transformer.\n", - "\n", - "To understand the feature set and transformations in more detail, see [feature store concepts](https://learn.microsoft.com/azure/machine-learning/concept-what-is-managed-feature-store)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697156302360 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "develop-txn-fset-locally", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from azureml.featurestore import create_feature_set_spec, FeatureSetSpec\n", - "from azureml.featurestore.contracts import (\n", - " DateTimeOffset,\n", - " FeatureSource,\n", - " TransformationCode,\n", - " Column,\n", - " ColumnType,\n", - " SourceType,\n", - " TimestampColumn,\n", - ")\n", - "\n", - "\n", - "transactions_featureset_code_path = (\n", - " root_dir + \"/featurestore/featuresets/transactions/transformation_code\"\n", - ")\n", - "\n", - "transactions_featureset_spec = create_feature_set_spec(\n", - " source=FeatureSource(\n", - " type=SourceType.parquet,\n", - " path=\"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/transactions-source/*.parquet\",\n", - " timestamp_column=TimestampColumn(name=\"timestamp\"),\n", - " source_delay=DateTimeOffset(days=0, hours=0, minutes=20),\n", - " ),\n", - " transformation_code=TransformationCode(\n", - " path=transactions_featureset_code_path,\n", - " transformer_class=\"transaction_transform.TransactionFeatureTransformer\",\n", - " ),\n", - " index_columns=[Column(name=\"accountID\", type=ColumnType.string)],\n", - " source_lookback=DateTimeOffset(days=7, hours=0, minutes=0),\n", - " temporal_join_lookback=DateTimeOffset(days=1, hours=0, minutes=0),\n", - " infer_schema=True,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "# Generate a spark dataframe from the feature set specification.\n", - "transactions_fset_df = transactions_featureset_spec.to_spark_dataframe()\n", - "# Display few records.\n", - "display(transactions_fset_df.head(5))" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 2c: Export as feature set specification\n", - "In order to register the feature set specification with the feature store, it needs to be saved in a specific format. \n", - "Action: Please inspect the generated `transactions` FeaturesetSpec: Open this file from the file tree to see the specification: `featurestore/featuresets/accounts/spec/FeaturesetSpec.yaml`\n", - "\n", - "Specification contains these important elements:\n", - "\n", - "1. `source`: Reference to a storage. In this case a parquet file in a blob storage.\n", - "1. `features`: List of features and their datatypes. If you provide transformation code (see Day 2 section), the code has to return a dataframe that maps to the features and datatypes.\n", - "1. `index_columns`: The join keys required to access values from the feature set\n", - "\n", - "Learn more about it in the [top level feature store entities document](https://learn.microsoft.com/azure/machine-learning/concept-top-level-entities-in-managed-feature-store) and the [feature set specification YAML reference](https://learn.microsoft.com/azure/machine-learning/reference-yaml-featureset-spec).\n", - "\n", - "The additional benefit of persisting the feature set specification is that it can be source controlled." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697156318907 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "dump-transactions-fs-spec", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# Create a new folder to dump the feature set specification.\n", - "transactions_featureset_spec_folder = (\n", - " root_dir + \"/featurestore/featuresets/transactions/spec\"\n", - ")\n", - "\n", - "# Check if the folder exists, create one if it does not exist.\n", - "if not os.path.exists(transactions_featureset_spec_folder):\n", - " os.makedirs(transactions_featureset_spec_folder)\n", - "\n", - "transactions_featureset_spec.dump(transactions_featureset_spec_folder, overwrite=True)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 3: Register a feature store entity\n", - "Entity helps enforce best practice that same join key definitions are used across feature sets which uses the same logical entities. Examples of entities are account entity, customer entity etc. Entities are typically created once and reused across feature-sets. For information on basics concept of feature store, see [feature store concepts](https://learn.microsoft.com/azure/machine-learning/concept-what-is-managed-feature-store)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697157524651 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "register-acct-entity-cli", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "account_entity_path = root_dir + \"/featurestore/entities/account.yaml\"\n", - "!az ml feature-store-entity create --file $account_entity_path --resource-group $featurestore_resource_group_name --feature-store-name $featurestore_name" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 4: Register the transaction feature set with the feature store\n", - "You register a feature set asset with the feature store so that you can share and reuse with others. You also get managed capabilities like versioning and materialization (we will learn in this tutorial series).\n", - "\n", - "The feature set asset has reference to the feature set spec that you created earlier and additional properties like version and materialization settings." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697157498133 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "register-txn-fset-cli", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "account_featureset_path = (\n", - " root_dir + \"/featurestore/featuresets/transactions/featureset_asset.yaml\"\n", - ")\n", - "!az ml feature-set create --file $account_featureset_path --resource-group $featurestore_resource_group_name --feature-store-name $featurestore_name" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Explore the feature store UI\n", - "* Goto the [Azure ML global landing page](https://ml.azure.com/home).\n", - "* Click on **Feature stores** in the left navigation.\n", - "* You will see the list of feature stores that you have access to. Click on the feature store that you have created above.\n", - "\n", - "You can see the feature sets and entities that you have created.\n", - "\n", - "Note: Creating and updating feature store assets (feature sets and entities) is possible only through SDK and CLI. You can use the UI to search/browse the feature store." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Grant \"Storage Blob Data Reader\" role on the offline store to your user identity\n", - "If feature data is materialized, then you need this role to read feature data from offline materialization store.\n", - "- Get information about the offline materialization store from the Feature Store **Overview** page in the Feature Store UI. The storage account subscription ID, storage account resource group name, and storage account name for offline materialization store can be found on **Offline materialization store** card. \n", - "![OFFLINE_STORE_INFO](./images/offline-store-information.png) \n", - "\n", - "To learn more about access control, see access control document in the docs.\n", - "\n", - "Execute the following code cell for role assignment. Please note that it may take some time for permissions to propagate. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "grant-rbac-to-user-identity-cli" - }, - "outputs": [], - "source": [ - "storage_subscription_id = \"\"\n", - "storage_resource_group_name = \"\"\n", - "storage_account_name = \"\"\n", - "\n", - "# Set the ADLS Gen2 storage account ARM ID\n", - "gen2_storage_arm_id = \"/subscriptions/{sub_id}/resourceGroups/{rg}/providers/Microsoft.Storage/storageAccounts/{account}\".format(\n", - " sub_id=storage_subscription_id,\n", - " rg=storage_resource_group_name,\n", - " account=storage_account_name,\n", - ")\n", - "\n", - "print(gen2_storage_arm_id)\n", - "\n", - "!az role assignment create --role \"Storage Blob Data Reader\" --assignee-object-id $your_aad_objectid --assignee-principal-type User --scope $gen2_storage_arm_id" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 5: Generate a training data dataframe using the registered features" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 5a: Load observation data\n", - "\n", - "We start by exploring the observation data. Observation data is typically the core data used in training and inference data. This is then joined with feature data to create the full training data. Observation data is the data captured during the time of the event: in this case it has core transaction data including transaction ID, account ID, transaction amount. In this case, since it is for training, it also has the target variable appended (`is_fraud`).\n", - "\n", - "To learn more core concepts including observation data, refer to the feature store documentation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697156393056 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "load-obs-data", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "observation_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/observation_data/train/*.parquet\"\n", - "observation_data_df = spark.read.parquet(observation_data_path)\n", - "obs_data_timestamp_column = \"timestamp\"\n", - "\n", - "display(observation_data_df)\n", - "# Note: the timestamp column is displayed in a different format. Optionally, you can can call training_df.show() to see correctly formatted value" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 5b: Get the registered feature set and list its features" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697156408060 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "get-txn-fset", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# Look up the featureset by providing a name and a version.\n", - "transactions_featureset = featurestore.feature_sets.get(\"transactions\", \"1\")\n", - "# List its features.\n", - "transactions_featureset.features" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697156423972 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "print-txn-fset-sample-values", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "# Print sample values.\n", - "display(transactions_featureset.to_spark_dataframe().head(5))" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 5c: Select features and generate training data\n", - "In this step we will select features that we would like to be part of training data and use the feature store SDK to generate the training data." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697156450018 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "select-features-and-gen-training-data", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from azureml.featurestore import get_offline_features\n", - "\n", - "# You can select features in pythonic way.\n", - "features = [\n", - " transactions_featureset.get_feature(\"transaction_amount_7d_sum\"),\n", - " transactions_featureset.get_feature(\"transaction_amount_7d_avg\"),\n", - "]\n", - "\n", - "# You can also specify features in string form: featureset:version:feature.\n", - "more_features = [\n", - " \"transactions:1:transaction_3d_count\",\n", - " \"transactions:1:transaction_amount_3d_avg\",\n", - "]\n", - "\n", - "more_features = featurestore.resolve_feature_uri(more_features)\n", - "features.extend(more_features)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "# Generate training dataframe by using feature data and observation data.\n", - "training_df = get_offline_features(\n", - " features=features,\n", - " observation_data=observation_data_df,\n", - " timestamp_column=obs_data_timestamp_column,\n", - ")\n", - "\n", - "# Ignore the message that says feature set is not materialized (materialization is optional). We will enable materialization in the next part of the tutorial.\n", - "display(training_df)\n", - "# Note: the timestamp column is displayed in a different format. Optionally, you can can call training_df.show() to see correctly formatted value" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "You can see how the features are appended to the training data using a point-in-time join." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 6: Enable offline materialization on transactions feature set\n", - "Once materialization is enabled on a feature set, you can perform backfill (this tutorial) or schedule recurrent materialization jobs (shown in a later tutorial).\n", - "\n", - "#### Set spark.sql.shuffle.partitions in the yaml file according to the feature data size\n", - "\n", - "The spark configuration `spark.sql.shuffle.partitions` is an OPTIONAL parameter that can affect the number of parquet files generated (per day) when the feature set is materialized into the offline store. The default value of this parameter is 200. The best practice is to avoid generating many small parquet files. If offline feature retrieval turns out to become slow after the feature set is materialized, please go to the corresponding folder in the offline store to check whether it is the issue of having too many small parquet files (per day), and adjust the value of this parameter accordingly.\n", - "\n", - "*Note: The sample data used in this notebook is small. So this parameter is set to 1 in the featureset_asset_offline_enabled.yaml file.*" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697157176939 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "enable-offline-mat-txns-fset-cli", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "transaction_asset_mat_yaml = (\n", - " root_dir\n", - " + \"/featurestore/featuresets/transactions/featureset_asset_offline_enabled.yaml\"\n", - ")\n", - "\n", - "!az ml feature-set update --file $transaction_asset_mat_yaml --resource-group $featurestore_resource_group_name --feature-store-name $featurestore_name" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 7: Backfill data for transactions feature set\n", - "Materialization is the process of computing the feature values for a given feature window and storing this in an materialization store. Materializing the features will increase its reliability and availability. All feature queries will use the materialized values from the materialization store. In this step you perform a one-time backfill for a feature window of __18 months__.\n", - "\n", - "The following code cell will materialize data by current status *None* or *Incomplete* for the defined feature window. \n", - "\n", - "#### Note\n", - "How to determine the window of backfill data needed? It has to match with the window of your training data. For e.g. if you want to train with two years of data, then you will want to be able to retrieve features for the same window, so you will backfill for a two year window." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697157219809 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "backfill-txns-fset-cli", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "feature_window_start_time = \"2022-01-01T00:00.000Z\"\n", - "feature_window_end_time = \"2023-06-30T00:00.000Z\"\n", - "\n", - "!az ml feature-set backfill --name transactions --version 1 --by-data-status \"['None', 'Incomplete']\" --feature-window-start-time $feature_window_start_time --feature-window-end-time $feature_window_end_time --feature-store-name $featurestore_name --resource-group $featurestore_resource_group_name" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Let's print sample data from the feature set. You can notice from the output information that the data was retrieved from the materilization store. `get_offline_features()` method that is used to retrieve training/inference data will also use the materialization store by default." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697157361591 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "sample-txns-fset-data-cli", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "# Look up the feature set by providing a name and a version and display few records.\n", - "transactions_featureset = featurestore.feature_sets.get(\"transactions\", \"1\")\n", - "display(transactions_featureset.to_spark_dataframe().head(5))" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Cleanup\n", - "\n", - "Tutorial `3. Enable recurrent materialization and run batch inference` has instructions deleting the resources." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Next steps\n", - "* Experiment and train models using features" - ] - } - ], - "metadata": { - "celltoolbar": "Edit Metadata", - "kernel_info": { - "name": "synapse_pyspark" - }, - "kernelspec": { - "display_name": "Synapse PySpark", - "language": "Python", - "name": "synapse_pyspark" - }, - "language_info": { - "codemirror_mode": "ipython", - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython", - "version": "3.8.0" - }, - "microsoft": { - "host": { - "AzureML": { - "notebookHasBeenCompleted": true - } - }, - "ms_spell_check": { - "ms_ignore_dictionary": [ - "dataframe", - "featureset", - "operationalization", - "operationalize" - ], - "ms_spell_check_language": "en" - } - }, - "nteract": { - "version": "nteract-front-end@1.0.0" - } + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "# Tutorial #1: Develop a feature set and register with managed feature store" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "Azure ML managed feature store lets you discover, create and operationalize features. Features are the connective tissue in ML lifecycle, starting from prototyping where you experiment with various features to operationalization where models are deployed and feature data is looked up during inference. For information on basics concept of feature store, see [feature store concepts](https://learn.microsoft.com/azure/machine-learning/concept-what-is-managed-feature-store).\n", + "\n", + "In this tutorial series you will experience how features seamlessly integrates all the phases of ML lifecycle:\n", + "\n", + "This tutorial is the first part of a three part series. In this tutorial you will:\n", + "- Create a new minimal feature store resource\n", + "- Develop and test featureset locally with feature transformation capability\n", + "- Register a feature-store entity with the feature store\n", + "- Register the featureset that you developed with the feature store\n", + "- Generate sample training data dataframe using the features you created\n", + "- Enable offline materialization on the feature sets, and backfill the feature data\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Prerequisites\n", + "\n", + "> [!NOTE]\n", + "> This tutorial uses Azure Machine Learning notebook with **Serverless Spark Compute**.\n", + "\n", + "Before following the steps in this article, make sure you have the following prerequisites:\n", + "\n", + "* An Azure Machine Learning workspace. If you don't have one, use the steps in the [Quickstart: Create workspace resources](https://learn.microsoft.com/en-us/azure/machine-learning/quickstart-create-resources?view=azureml-api-2) article to create one.\n", + "* To perform the steps in this article, your user account must be assigned the owner or contributor role to a resource group where the feature store will be created" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Note\n", + "This tutorial series has two tracks:\n", + "1. SDK only track: Uses only Python SDKs. This is suitable if pure Python-based development and deployment is preferred.\n", + "1. SDK & CLI track: Uses CLI for CRUD (create, read, update, and delete) operations and Python SDK for feature set development and testing only. This is useful in CI/CD or GitOps scenarios where CLI/YAML is preferred." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Set up\n", + "\n", + "This tutorial uses the Python feature store core SDK (`azureml-featurestore`) for feature set development and testing. The CLI is used for create, read, update, and delete (CRUD) operations, on feature stores, feature sets, and feature store entities.\n", + "\n", + "You don't need to explicitly install these resources for this tutorial, because in the set-up instructions shown here, the `conda.yml` file covers them.\n", + "\n", + "To prepare the notebook environment for development:\n", + "\n", + "1. In the Azure Machine Learning studio environment, select Notebooks on the left pane, and then select the Samples tab.\n", + "\n", + "1. Browse to the featurestore_sample directory (select **Samples** > **SDK v2** > **sdk** > **python** > **featurestore_sample**), and then select **Clone**.\n", + "\n", + "1. The **Select target directory** panel opens. Select the **Users** directory and then select _your user name_, and then select **Clone**.\n", + "\n", + "1. Run the tutorial\n", + "\n", + " * Option 1: Create a new notebook, and execute the instructions in this document, step by step.\n", + " * Option 2: Open existing notebook `featurestore_sample/notebooks/sdk_and_cli/1. Develop a feature set and register with managed feature store.ipynb`. You may keep this document open and refer to it for more explanation and documentation links.\n", + "\n", + "1. To configure the notebook environment, you must upload the conda.yml file\n", + "\n", + " 1. Select **Notebooks** on the left pane, and then select the **Files** tab.\n", + " 1. Browse to the *env* directory (select **Users** > *your_user_name* > **featurestore_sample** > **project** > **env**), and then select the conda.yml file.\n", + " 1. Select **Download**\n", + " 1. Select **Serverless Spark Compute** in the top navigation **Compute** dropdown. This operation might take one to two minutes. Wait for a status bar in the top to display **Configure session**.\n", + " 1. Select **Configure session** in the top status bar.\n", + " 1. Select **Python packages**.\n", + " 1. Select **Upload conda file**.\n", + " 1. Select the `conda.yml` you downloaded on your local device.\n", + " 1. (Optional) Increase the session time-out (idle time in minutes) to reduce the serverless spark cluster startup time.\n", + "\n", + "__Important:__ Except for this step, you need to run all the other steps every time you have a new spark session/session time out.\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Start spark session\n", + "Execute the following code cell to start the Spark session. It wil take approximately 10 minutes to install all dependencies and start the Spark session." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697155816751 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "start-spark-session", + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# Run this cell to start the spark session (any code block will start the session). This can take around 10 mins.\n", + "print(\"start spark session\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Setup root directory for the samples\n", + "This code cell sets up the root directory for the samples." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697155915577 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "root-dir", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Please update your alias belpw (or any custom directory you uploaded the samples to).\n", + "# You can find the name from the directory structure in the left nav\n", + "root_dir = \"./Users//featurestore_sample\"\n", + "\n", + "if os.path.isdir(root_dir):\n", + " print(\"The folder exists.\")\n", + "else:\n", + " print(\"The folder does not exist. Please create or fix the path\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Setup CLI\n", + "\n", + "1. Install AzureML CLI extention\n", + "1. Authenticate\n", + "1. Set the default subscription" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697155966146 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "install-ml-ext-cli", + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "# Install AzureML CLI extension\n", + "!az extension add --name ml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697156018290 + }, + "jupyter": { + "outputs_hidden": true, + "source_hidden": false + }, + "name": "auth-cli", + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "# Authenticate\n", + "!az login" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697156029362 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "set-default-subs-cli", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# Set default subscription\n", + "import os\n", + "\n", + "subscription_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", + "\n", + "!az account set -s $subscription_id" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Note\n", + "Feature store Vs Project workspace: You will use a feature store to reuse features across projects. You will use a project workspace (the current workspace) to train and inference models, by leveraging features from feature stores. Many project workspaces can share and reuse a same feature store.\n", + "\n", + "## Note\n", + "In this tutorial you will be using CLI and feature store core SDK:\n", + "\n", + "1. CLI: You will use CLI for CRUD (create, read, update, and delete) operations on feature store, feature set and feature store entities.\n", + "2. Feature store core SDK: This SDK (`azureml-featurestore`) is meant to be used for feature set development and consumption (you will learn more about these operations later):\n", + " - List/Get registered feature set\n", + " - Generate/resolve feature retrieval specification\n", + " - Execute featureset definition to generate Spark dataframe\n", + " - Generate training data using a point-in-time join\n", + "\n", + "For this tutorial so you do not need to install any of these explicitly, since the instructions already cover them (`conda.yml` in the above step include these)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 1: Create a minimal feature store" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 1a: Set feature store parameters\n", + "Set name, location and other values for the feature store." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697156060740 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "fs-params", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# We use the subscription, resource group, region of this active project workspace.\n", + "# You can optionally replace them to create the resources in a different subsciprtion/resourceGroup, or use existing resources\n", + "import os\n", + "\n", + "featurestore_name = \"\"\n", + "featurestore_location = \"eastus\"\n", + "featurestore_subscription_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", + "featurestore_resource_group_name = os.environ[\"AZUREML_ARM_RESOURCEGROUP\"]\n", + "\n", + "feature_store_arm_id = \"/subscriptions/{sub_id}/resourceGroups/{rg}/providers/Microsoft.MachineLearningServices/workspaces/{ws_name}\".format(\n", + " sub_id=featurestore_subscription_id,\n", + " rg=featurestore_resource_group_name,\n", + " ws_name=featurestore_name,\n", + ")\n", + "\n", + "print(feature_store_arm_id)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 1b: Create the feature store" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1683415482376 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "create-fs-cli", + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "!az ml feature-store create --subscription $featurestore_subscription_id --resource-group $featurestore_resource_group_name --location $featurestore_location --name $featurestore_name" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 1c: Initialize AzureML feature store core SDK client\n", + "As explained above, this is used to develop and consume features." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697156242248 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "init-fs-core-sdk", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# feature store client\n", + "from azureml.featurestore import FeatureStoreClient\n", + "from azure.ai.ml.identity import AzureMLOnBehalfOfCredential\n", + "\n", + "featurestore = FeatureStoreClient(\n", + " credential=AzureMLOnBehalfOfCredential(),\n", + " subscription_id=featurestore_subscription_id,\n", + " resource_group_name=featurestore_resource_group_name,\n", + " name=featurestore_name,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Step 1d. Grant \"AzureML Data Scientist\" role on the feature store to your user identity" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Get your AAD object id from Azure portal following this instruction: https://learn.microsoft.com/en-us/partner-center/find-ids-and-domain-names#find-the-user-object-id\\\n", + "\n", + "Assign **AzureML Data Scientist** role to your user identity so that it can create resources in feature store workspace. Please note that it may take some time for the permissions to propagate." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "assign-aad-ds-role-cli" + }, + "outputs": [], + "source": [ + "your_aad_objectid = \"\"\n", + "\n", + "!az role assignment create --role \"AzureML Data Scientist\" --assignee-object-id $your_aad_objectid --assignee-principal-type User --scope $feature_store_arm_id" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 2: Prototype and develop a transaction rolling aggregation feature set in this notebook" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 2a: Explore the transactions source data\n", + "\n", + "#### Note\n", + "The sample data used in this notebook is hosted in a public accessible blob container. It can only be read in Spark via `wasbs` driver. When you create feature sets using your own source data, please host them in ADLS Gen2 account and use `abfss` driver in the data path. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697156282781 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "explore-txn-src-data", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# Remove the \".\" in the roor directory path as we need to generate absolute path to read from Spark.\n", + "transactions_source_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/transactions-source/*.parquet\"\n", + "transactions_src_df = spark.read.parquet(transactions_source_data_path)\n", + "\n", + "display(transactions_src_df.head(5))\n", + "# Note: display(training_df.head(5)) displays the timestamp column in a different format. You can can call transactions_src_df.show() to see correctly formatted value" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 2b: Develop a transactions feature set locally\n", + "\n", + "Feature set specification is a self-contained definition of feature set that can be developed and tested locally.\n", + "\n", + "Lets create the following rolling window aggregate features:\n", + "- transactions 3-day count\n", + "- transactions amount 3-day sum\n", + "- transactions amount 3-day avg\n", + "- transactions 7-day count\n", + "- transactions amount 7-day sum\n", + "- transactions amount 7-day avg\n", + "\n", + "__Action__:\n", + "- Inspect the feature transformation code file: `featurestore/featuresets/transactions/spec/transformation_code/transaction_transform.py`. You will see how is the rolling aggregation defined for the features. This is a Spark transformer.\n", + "\n", + "To understand the feature set and transformations in more detail, see [feature store concepts](https://learn.microsoft.com/azure/machine-learning/concept-what-is-managed-feature-store)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697156302360 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "develop-txn-fset-locally", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "from azureml.featurestore import create_feature_set_spec, FeatureSetSpec\n", + "from azureml.featurestore.contracts import (\n", + " DateTimeOffset,\n", + " FeatureSource,\n", + " TransformationCode,\n", + " Column,\n", + " ColumnType,\n", + " SourceType,\n", + " TimestampColumn,\n", + ")\n", + "\n", + "\n", + "transactions_featureset_code_path = (\n", + " root_dir + \"/featurestore/featuresets/transactions/transformation_code\"\n", + ")\n", + "\n", + "transactions_featureset_spec = create_feature_set_spec(\n", + " source=FeatureSource(\n", + " type=SourceType.parquet,\n", + " path=\"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/transactions-source/*.parquet\",\n", + " timestamp_column=TimestampColumn(name=\"timestamp\"),\n", + " source_delay=DateTimeOffset(days=0, hours=0, minutes=20),\n", + " ),\n", + " transformation_code=TransformationCode(\n", + " path=transactions_featureset_code_path,\n", + " transformer_class=\"transaction_transform.TransactionFeatureTransformer\",\n", + " ),\n", + " index_columns=[Column(name=\"accountID\", type=ColumnType.string)],\n", + " source_lookback=DateTimeOffset(days=7, hours=0, minutes=0),\n", + " temporal_join_lookback=DateTimeOffset(days=1, hours=0, minutes=0),\n", + " infer_schema=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "# Generate a spark dataframe from the feature set specification.\n", + "transactions_fset_df = transactions_featureset_spec.to_spark_dataframe()\n", + "# Display few records.\n", + "display(transactions_fset_df.head(5))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 2c: Export as feature set specification\n", + "In order to register the feature set specification with the feature store, it needs to be saved in a specific format. \n", + "Action: Please inspect the generated `transactions` FeaturesetSpec: Open this file from the file tree to see the specification: `featurestore/featuresets/accounts/spec/FeaturesetSpec.yaml`\n", + "\n", + "Specification contains these important elements:\n", + "\n", + "1. `source`: Reference to a storage. In this case a parquet file in a blob storage.\n", + "1. `features`: List of features and their datatypes. If you provide transformation code (see Day 2 section), the code has to return a dataframe that maps to the features and datatypes.\n", + "1. `index_columns`: The join keys required to access values from the feature set\n", + "\n", + "Learn more about it in the [top level feature store entities document](https://learn.microsoft.com/azure/machine-learning/concept-top-level-entities-in-managed-feature-store) and the [feature set specification YAML reference](https://learn.microsoft.com/azure/machine-learning/reference-yaml-featureset-spec).\n", + "\n", + "The additional benefit of persisting the feature set specification is that it can be source controlled." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697156318907 }, - "nbformat": 4, - "nbformat_minor": 2 + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "dump-transactions-fs-spec", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Create a new folder to dump the feature set specification.\n", + "transactions_featureset_spec_folder = (\n", + " root_dir + \"/featurestore/featuresets/transactions/spec\"\n", + ")\n", + "\n", + "# Check if the folder exists, create one if it does not exist.\n", + "if not os.path.exists(transactions_featureset_spec_folder):\n", + " os.makedirs(transactions_featureset_spec_folder)\n", + "\n", + "transactions_featureset_spec.dump(transactions_featureset_spec_folder, overwrite=True)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 3: Register a feature store entity\n", + "Entity helps enforce best practice that same join key definitions are used across feature sets which uses the same logical entities. Examples of entities are account entity, customer entity etc. Entities are typically created once and reused across feature-sets. For information on basics concept of feature store, see [feature store concepts](https://learn.microsoft.com/azure/machine-learning/concept-what-is-managed-feature-store)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697157524651 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "register-acct-entity-cli", + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "account_entity_path = root_dir + \"/featurestore/entities/account.yaml\"\n", + "!az ml feature-store-entity create --file $account_entity_path --resource-group $featurestore_resource_group_name --feature-store-name $featurestore_name" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 4: Register the transaction feature set with the feature store\n", + "You register a feature set asset with the feature store so that you can share and reuse with others. You also get managed capabilities like versioning and materialization (we will learn in this tutorial series).\n", + "\n", + "The feature set asset has reference to the feature set spec that you created earlier and additional properties like version and materialization settings." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697157498133 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "register-txn-fset-cli", + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "account_featureset_path = (\n", + " root_dir + \"/featurestore/featuresets/transactions/featureset_asset.yaml\"\n", + ")\n", + "!az ml feature-set create --file $account_featureset_path --resource-group $featurestore_resource_group_name --feature-store-name $featurestore_name" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Explore the feature store UI\n", + "* Goto the [Azure ML global landing page](https://ml.azure.com/home).\n", + "* Click on **Feature stores** in the left navigation.\n", + "* You will see the list of feature stores that you have access to. Click on the feature store that you have created above.\n", + "\n", + "You can see the feature sets and entities that you have created.\n", + "\n", + "Note: Creating and updating feature store assets (feature sets and entities) is possible only through SDK and CLI. You can use the UI to search/browse the feature store." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Grant \"Storage Blob Data Reader\" role on the offline store to your user identity\n", + "If feature data is materialized, then you need this role to read feature data from offline materialization store.\n", + "- Get information about the offline materialization store from the Feature Store **Overview** page in the Feature Store UI. The storage account subscription ID, storage account resource group name, and storage account name for offline materialization store can be found on **Offline materialization store** card. \n", + "![OFFLINE_STORE_INFO](./images/offline-store-information.png) \n", + "\n", + "To learn more about access control, see access control document in the docs.\n", + "\n", + "Execute the following code cell for role assignment. Please note that it may take some time for permissions to propagate. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "grant-rbac-to-user-identity-cli" + }, + "outputs": [], + "source": [ + "storage_subscription_id = \"\"\n", + "storage_resource_group_name = \"\"\n", + "storage_account_name = \"\"\n", + "\n", + "# Set the ADLS Gen2 storage account ARM ID\n", + "gen2_storage_arm_id = \"/subscriptions/{sub_id}/resourceGroups/{rg}/providers/Microsoft.Storage/storageAccounts/{account}\".format(\n", + " sub_id=storage_subscription_id,\n", + " rg=storage_resource_group_name,\n", + " account=storage_account_name,\n", + ")\n", + "\n", + "print(gen2_storage_arm_id)\n", + "\n", + "!az role assignment create --role \"Storage Blob Data Reader\" --assignee-object-id $your_aad_objectid --assignee-principal-type User --scope $gen2_storage_arm_id" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 5: Generate a training data dataframe using the registered features" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 5a: Load observation data\n", + "\n", + "We start by exploring the observation data. Observation data is typically the core data used in training and inference data. This is then joined with feature data to create the full training data. Observation data is the data captured during the time of the event: in this case it has core transaction data including transaction ID, account ID, transaction amount. In this case, since it is for training, it also has the target variable appended (`is_fraud`).\n", + "\n", + "To learn more core concepts including observation data, refer to the feature store documentation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697156393056 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "load-obs-data", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "observation_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/observation_data/train/*.parquet\"\n", + "observation_data_df = spark.read.parquet(observation_data_path)\n", + "obs_data_timestamp_column = \"timestamp\"\n", + "\n", + "display(observation_data_df)\n", + "# Note: the timestamp column is displayed in a different format. Optionally, you can can call training_df.show() to see correctly formatted value" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 5b: Get the registered feature set and list its features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697156408060 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "get-txn-fset", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# Look up the featureset by providing a name and a version.\n", + "transactions_featureset = featurestore.feature_sets.get(\"transactions\", \"1\")\n", + "# List its features.\n", + "transactions_featureset.features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697156423972 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "print-txn-fset-sample-values", + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "# Print sample values.\n", + "display(transactions_featureset.to_spark_dataframe().head(5))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 5c: Select features and generate training data\n", + "In this step we will select features that we would like to be part of training data and use the feature store SDK to generate the training data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697156450018 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "select-features-and-gen-training-data", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "from azureml.featurestore import get_offline_features\n", + "\n", + "# You can select features in pythonic way.\n", + "features = [\n", + " transactions_featureset.get_feature(\"transaction_amount_7d_sum\"),\n", + " transactions_featureset.get_feature(\"transaction_amount_7d_avg\"),\n", + "]\n", + "\n", + "# You can also specify features in string form: featureset:version:feature.\n", + "more_features = [\n", + " \"transactions:1:transaction_3d_count\",\n", + " \"transactions:1:transaction_amount_3d_avg\",\n", + "]\n", + "\n", + "more_features = featurestore.resolve_feature_uri(more_features)\n", + "features.extend(more_features)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "# Generate training dataframe by using feature data and observation data.\n", + "training_df = get_offline_features(\n", + " features=features,\n", + " observation_data=observation_data_df,\n", + " timestamp_column=obs_data_timestamp_column,\n", + ")\n", + "\n", + "# Ignore the message that says feature set is not materialized (materialization is optional). We will enable materialization in the next part of the tutorial.\n", + "display(training_df)\n", + "# Note: the timestamp column is displayed in a different format. Optionally, you can can call training_df.show() to see correctly formatted value" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "You can see how the features are appended to the training data using a point-in-time join." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 6: Enable offline materialization on transactions feature set\n", + "Once materialization is enabled on a feature set, you can perform backfill (this tutorial) or schedule recurrent materialization jobs (shown in a later tutorial).\n", + "\n", + "#### Set spark.sql.shuffle.partitions in the yaml file according to the feature data size\n", + "\n", + "The spark configuration `spark.sql.shuffle.partitions` is an OPTIONAL parameter that can affect the number of parquet files generated (per day) when the feature set is materialized into the offline store. The default value of this parameter is 200. The best practice is to avoid generating many small parquet files. If offline feature retrieval turns out to become slow after the feature set is materialized, please go to the corresponding folder in the offline store to check whether it is the issue of having too many small parquet files (per day), and adjust the value of this parameter accordingly.\n", + "\n", + "*Note: The sample data used in this notebook is small. So this parameter is set to 1 in the featureset_asset_offline_enabled.yaml file.*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697157176939 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "enable-offline-mat-txns-fset-cli", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "transaction_asset_mat_yaml = (\n", + " root_dir\n", + " + \"/featurestore/featuresets/transactions/featureset_asset_offline_enabled.yaml\"\n", + ")\n", + "\n", + "!az ml feature-set update --file $transaction_asset_mat_yaml --resource-group $featurestore_resource_group_name --feature-store-name $featurestore_name" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 7: Backfill data for transactions feature set\n", + "Materialization is the process of computing the feature values for a given feature window and storing this in an materialization store. Materializing the features will increase its reliability and availability. All feature queries will use the materialized values from the materialization store. In this step you perform a one-time backfill for a feature window of __18 months__.\n", + "\n", + "The following code cell will materialize data by current status *None* or *Incomplete* for the defined feature window. \n", + "\n", + "#### Note\n", + "How to determine the window of backfill data needed? It has to match with the window of your training data. For e.g. if you want to train with two years of data, then you will want to be able to retrieve features for the same window, so you will backfill for a two year window." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697157219809 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "backfill-txns-fset-cli", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "feature_window_start_time = \"2022-01-01T00:00.000Z\"\n", + "feature_window_end_time = \"2023-06-30T00:00.000Z\"\n", + "\n", + "!az ml feature-set backfill --name transactions --version 1 --by-data-status \"['None', 'Incomplete']\" --feature-window-start-time $feature_window_start_time --feature-window-end-time $feature_window_end_time --feature-store-name $featurestore_name --resource-group $featurestore_resource_group_name" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "Let's print sample data from the feature set. You can notice from the output information that the data was retrieved from the materilization store. `get_offline_features()` method that is used to retrieve training/inference data will also use the materialization store by default." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697157361591 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "sample-txns-fset-data-cli", + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "# Look up the feature set by providing a name and a version and display few records.\n", + "transactions_featureset = featurestore.feature_sets.get(\"transactions\", \"1\")\n", + "display(transactions_featureset.to_spark_dataframe().head(5))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Cleanup\n", + "\n", + "Tutorial `3. Enable recurrent materialization and run batch inference` has instructions deleting the resources." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Next steps\n", + "* Experiment and train models using features" + ] + } + ], + "metadata": { + "celltoolbar": "Edit Metadata", + "kernel_info": { + "name": "synapse_pyspark" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": "ipython", + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython", + "version": "3.8.0" + }, + "microsoft": { + "host": { + "AzureML": { + "notebookHasBeenCompleted": true + } + }, + "ms_spell_check": { + "ms_ignore_dictionary": [ + "dataframe", + "featureset", + "operationalization", + "operationalize" + ], + "ms_spell_check_language": "en" + } + }, + "nteract": { + "version": "nteract-front-end@1.0.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/sdk/python/featurestore_sample/notebooks/sdk_and_cli/2.Experiment-train-models-using-features.ipynb b/sdk/python/featurestore_sample/notebooks/sdk_and_cli/2.Experiment-train-models-using-features.ipynb index 997e412b6f..2d74f2f8f4 100644 --- a/sdk/python/featurestore_sample/notebooks/sdk_and_cli/2.Experiment-train-models-using-features.ipynb +++ b/sdk/python/featurestore_sample/notebooks/sdk_and_cli/2.Experiment-train-models-using-features.ipynb @@ -1,1228 +1,1228 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# Tutorial #2: Experiment and train models using features" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "In this tutorial series you will experience how features seamlessly integrates all the phases of ML lifecycle: Prototyping features, training and operationalizing.\n", - "\n", - "In part 1 of the tutorial you learnt how to create a feature set spec with custom transformations, enable materialization and perform backfill. In this tutorial you will will learn how to experiment with features to improve model performance. You will see how feature store increasing agility in the experimentation and training flows. \n", - "\n", - "You will perform the following:\n", - "- Prototype a create new `acccounts` feature set spec using existing precomputed values as features, unlike part 1 of the tutorial where we created feature set that had custom transformations. You will then Register the local feature set spec as a feature set in the feature store\n", - "- Select features for the model: You will select features from the `transactions` and `accounts` feature sets and save them as a feature-retrieval spec\n", - "- Run training pipeline that uses the Feature retrieval spec to train a new model. This pipeline will use the built in feature-retrieval component to generate the training data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# Prerequisites\n", - "1. Please ensure you have executed `1. Develop a feature set and register with managed feature store` tutorial" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# Setup" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### (updated) Configure Azure ML spark notebook\n", - "\n", - "1. Running the tutorial: You can either create a new notebook, and execute the instructions in this document step by step or open the existing notebook named `Experiment and train models using features`, and run it. The notebooks are available in `featurestore_sample/notebooks` directory. You can select from `sdk_only` or `sdk_and_cli`. You may keep this document open and refer to it for additional explanation and documentation links.\n", - "1. In the \"Compute\" dropdown in the top nav, select \"Serverless Spark Compute\". \n", - "1. Click on \"configure session\" in top status bar -> click on \"Python packages\" -> click on \"upload conda file\" -> select the file azureml-examples/sdk/python/featurestore-sample/project/env/conda.yml from your local machine; Also increase the session time out (idle time) if you want to avoid running the prerequisites frequently\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Start spark session" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697158542512 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "start-spark-session", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# run this cell to start the spark session (any code block will start the session ). This can take around 10 mins.\n", - "print(\"start spark session\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Setup root directory for the samples" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697158549861 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "root-dir", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# please update the dir to ./Users/ (or any custom directory you uploaded the samples to).\n", - "# You can find the name from the directory structure inm the left nav\n", - "root_dir = \"./Users//featurestore_sample\"\n", - "\n", - "if os.path.isdir(root_dir):\n", - " print(\"The folder exists.\")\n", - "else:\n", - " print(\"The folder does not exist. Please create or fix the path\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### (new for sdk/cki track) Setup CLI\n", - "\n", - "1. Install azure ml cli extention\n", - "1. Authenticate\n", - "1. Set the default subscription" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "install-ml-ext-cli", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [] - }, - "outputs": [], - "source": [ - "!az extension add --name ml" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": true, - "source_hidden": false - }, - "name": "auth-cli", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [] - }, - "outputs": [], - "source": [ - "!az login" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697158670086 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "set-default-subs-cli", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [] - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "subscription_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", - "\n", - "!az account set -s $subscription_id" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Initialize the project workspace variables\n", - "This is the current workspace where you will be running the tutorial notebook from" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697158675914 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "init-ws-crud-client", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "project_ws_sub_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", - "project_ws_rg = os.environ[\"AZUREML_ARM_RESOURCEGROUP\"]\n", - "project_ws_name = os.environ[\"AZUREML_ARM_WORKSPACE_NAME\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Initialize the feature store variables\n", - "Ensure you update the `featurestore_name` to reflect what you created in part 1 of this tutorial" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697158680345 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "init-fs-crud-client", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "featurestore_name = (\n", - " \"\" # use the same name from part #1 of the tutorial\n", - ")\n", - "featurestore_subscription_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", - "featurestore_resource_group_name = os.environ[\"AZUREML_ARM_RESOURCEGROUP\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Initialize the feature store consumption client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697158696247 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "init-fs-core-sdk", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# feature store client\n", - "from azureml.featurestore import FeatureStoreClient\n", - "from azure.ai.ml.identity import AzureMLOnBehalfOfCredential\n", - "\n", - "featurestore = FeatureStoreClient(\n", - " credential=AzureMLOnBehalfOfCredential(),\n", - " subscription_id=featurestore_subscription_id,\n", - " resource_group_name=featurestore_resource_group_name,\n", - " name=featurestore_name,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Create compute cluster with name `cpu-cluster-fs` in the project workspace\n", - "This will be needed when we run training/batch inference jobs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697158803646 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "create-compute-cluster", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "compute_cluster_name = \"\"\n", - "type = \"\"\n", - "size = \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!az ml compute create --name $compute_cluster_name --type $type --size $size --idle-time-before-scale-down 360 --resource-group $project_ws_rg --workspace-name $project_ws_name" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 1: Create accounts featureset locally from precomputed data\n", - "In tutorial part 1, we created a transactions featureset that had custom transformations. Now we will create an accounts featureset that will use precomputed values. \n", - "\n", - "For onboarding precomputed features, you can create a featureset spec without writing any transformation code. Featureset spec is a specification to develop and test a featureset in a fully local/dev environment without connecting to any featurestore. In this step you will create the feature set spec locally and sample the values from it. If you want to get managed featurestore capabilities, you need to register the featureset spec with a feature store using a feature asset definition (a future step in this tutorial)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 1a: Explore the source data for accounts\n", - "\n", - "##### Note\n", - " Note that the sample data used in this notebook is hosted in a public accessible blob container. It can only be read in Spark via `wasbs` driver. When you create feature sets using your own source data, please host them in adls gen2 account and use `abfss` driver in the data path. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697158931390 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "explore-accts-fset-src-data", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "accounts_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/accounts-precalculated/*.parquet\"\n", - "accounts_df = spark.read.parquet(accounts_data_path)\n", - "\n", - "display(accounts_df.head(5))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 1b: Create `accounts` feature set spec in local from these precomputed features\n", - "Note that we do not need any transformation code here since we are referencing precomputed features." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697158944918 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "create-accts-fset-spec", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from azureml.featurestore import create_feature_set_spec, FeatureSetSpec\n", - "from azureml.featurestore.contracts import (\n", - " DateTimeOffset,\n", - " FeatureSource,\n", - " TransformationCode,\n", - " Column,\n", - " ColumnType,\n", - " SourceType,\n", - " TimestampColumn,\n", - ")\n", - "\n", - "\n", - "accounts_featureset_spec = create_feature_set_spec(\n", - " source=FeatureSource(\n", - " type=SourceType.parquet,\n", - " path=\"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/accounts-precalculated/*.parquet\",\n", - " timestamp_column=TimestampColumn(name=\"timestamp\"),\n", - " ),\n", - " index_columns=[Column(name=\"accountID\", type=ColumnType.string)],\n", - " infer_schema=True,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "# Generate a spark dataframe from the feature set specification\n", - "accounts_fset_df = accounts_featureset_spec.to_spark_dataframe()\n", - "# display few records\n", - "display(accounts_fset_df.head(5))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 1c: Export as feature set spec\n", - "Inorder to register the feature set spec with the feature store, it needs to be saved in a specific format. \n", - "Action: After running the below cell, please inspect the generated `accounts` FeatureSetSpec: Open this file from the file tree to see the spec: `featurestore/featuresets/accounts/spec/FeatureSetSpec.yaml`\n", - "\n", - "Spec contains these important elements:\n", - "\n", - "1. `source`: reference to a storage. In this case a parquet file in a blob storage.\n", - "1. `features`: list of features and their datatypes. If you provide transformation code (see Day 2 section), the code has to return a dataframe that maps to the features and datatypes. In case where you do not provide transformation code (in this case of `accounts` because it is precomputed), the system will build the query to to map these to the source \n", - "1. `index_columns`: the join keys required to access values from the feature set\n", - "\n", - "Learn more about it in the [top level feature store entities document](fs-concepts-todo) and the [feature set spec yaml reference](reference-yaml-featureset-spec.md).\n", - "\n", - "The additional benefit of persisting it is that it can be source controlled." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697158964056 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "dump-accts-fset-spec", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# create a new folder to dump the feature set spec\n", - "accounts_featureset_spec_folder = root_dir + \"/featurestore/featuresets/accounts/spec\"\n", - "\n", - "# check if the folder exists, create one if not\n", - "if not os.path.exists(accounts_featureset_spec_folder):\n", - " os.makedirs(accounts_featureset_spec_folder)\n", - "\n", - "accounts_featureset_spec.dump(accounts_featureset_spec_folder, overwrite=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 2: Experiment with unregistered features locally and register with feature store when ready\n", - "When you are developing features, you might want to test/validate locally before registering with the feature store or running training pipelines in the cloud. In this step you will generate training data for the ML model from combination of features from a local unregistered feature set (`accounts`) and feature set registered in the feature store (`transactions`)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 2a: Select features for model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697158972685 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "select-unreg-features-for-model", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# get the registered transactions feature set, version 1\n", - "transactions_featureset = featurestore.feature_sets.get(\"transactions\", \"1\")\n", - "# Notice that account feature set spec is in your local dev environment (this notebook): not registered with feature store yet\n", - "features = [\n", - " accounts_featureset_spec.get_feature(\"accountAge\"),\n", - " accounts_featureset_spec.get_feature(\"numPaymentRejects1dPerUser\"),\n", - " transactions_featureset.get_feature(\"transaction_amount_7d_sum\"),\n", - " transactions_featureset.get_feature(\"transaction_amount_3d_sum\"),\n", - " transactions_featureset.get_feature(\"transaction_amount_7d_avg\"),\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 2b: Generate training data locally\n", - "In this step we generate training data for illustrative purpose. You can optionally train models locally with this. In the upcoming steps in this tutorial, you will train a model in the cloud." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697159002277 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "gen-training-data-locally", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from azureml.featurestore import get_offline_features\n", - "\n", - "# Load the observation data. To understand observatio ndata, refer to part 1 of this tutorial\n", - "observation_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/observation_data/train/*.parquet\"\n", - "observation_data_df = spark.read.parquet(observation_data_path)\n", - "obs_data_timestamp_column = \"timestamp\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "# generate training dataframe by using feature data and observation data\n", - "training_df = get_offline_features(\n", - " features=features,\n", - " observation_data=observation_data_df,\n", - " timestamp_column=obs_data_timestamp_column,\n", - ")\n", - "\n", - "# Ignore the message that says feature set is not materialized (materialization is optional). We will enable materialization in the next part of the tutorial.\n", - "display(training_df)\n", - "# Note: display(training_df.head(5)) displays the timestamp column in a different format. You can can call training_df.show() to see correctly formatted value" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 2c: Register the `accounts` featureset with the featurestore\n", - "Once you have experimented with different feature definitions locally and sanity tested it, you can register it with the feature store.\n", - "For this you will register a featureset asset definition with the feature store.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697159036348 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "reg-accts-fset", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "feature_set_yaml = root_dir + \"/featurestore/featuresets/accounts/featureset_asset.yaml\"\n", - "\n", - "!az ml feature-set create --file $feature_set_yaml --resource-group $featurestore_resource_group_name --feature-store-name $featurestore_name" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 2d: Get registered featureset and sanity test" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697159055950 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "sample-accts-fset-data", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# look up the featureset by providing name and version\n", - "accounts_featureset = featurestore.feature_sets.get(\"accounts\", \"1\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "# get access to the feature data\n", - "accounts_feature_df = accounts_featureset.to_spark_dataframe()\n", - "display(accounts_feature_df.head(5))\n", - "# Note: Please ignore this warning: Failure while loading azureml_run_type_providers. Failed to load entrypoint azureml.scriptrun" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 3: Run training experiment\n", - "In this step you will select a list of features, run a training pipeline, and register the model. You can repeat this step till you are happy with the model performance." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### (Optional) Step 3a: Discover features from Feature Store UI\n", - "You have already done this in part 1 of the tutorial after registering the `transactions` feature set. Since you also have `accounts` featureset, you can browse the available features:\n", - "* Goto the [Azure ML global landing page](https://ml.azure.com/home?flight=FeatureStores).\n", - "* Click on `Feature stores` in the left nav\n", - "* You will see the list of feature stores that you have access to. Click on the feature store that you created above.\n", - "\n", - "You can see the feature sets and entity that you created. Click on the feature sets to browse the feature definitions. You can also search for feature sets across feature stores by using the global search box." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### (Optional) Step 3b: Discover features from SDK" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697159077800 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "discover-features-from-sdk", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# List available feature sets\n", - "all_featuresets = featurestore.feature_sets.list()\n", - "for fs in all_featuresets:\n", - " print(fs)\n", - "\n", - "# List of versions for transactions feature set\n", - "all_transactions_featureset_versions = featurestore.feature_sets.list(\n", - " name=\"transactions\"\n", - ")\n", - "for fs in all_transactions_featureset_versions:\n", - " print(fs)\n", - "\n", - "# See properties of the transactions featureset including list of features\n", - "featurestore.feature_sets.get(name=\"transactions\", version=\"1\").features" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 3c: Select features for the model and export it as a feature-retrieval spec\n", - "In the previous steps, you selected features from a combination unregistered and registered feature sets for local experimentation and testing. Now you are ready to experiment in the cloud. Saving the selected features as a feature-retrieval spec and using it in the mlops/cicd flow for training/inference increases your agility in shipping models." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Select features for the model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697159131879 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "select-reg-features", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# you can select features in pythonic way\n", - "features = [\n", - " accounts_featureset.get_feature(\"accountAge\"),\n", - " transactions_featureset.get_feature(\"transaction_amount_7d_sum\"),\n", - " transactions_featureset.get_feature(\"transaction_amount_3d_sum\"),\n", - "]\n", - "\n", - "# you can also specify features in string form: featurestore:featureset:version:feature\n", - "more_features = [\n", - " \"accounts:1:numPaymentRejects1dPerUser\",\n", - " \"transactions:1:transaction_amount_7d_avg\",\n", - "]\n", - "\n", - "more_features = featurestore.resolve_feature_uri(more_features)\n", - "\n", - "features.extend(more_features)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Export selected features as a feature-retrieval spec\n", - "\n", - "#### Note\n", - "Feature retrieval spec is a portable definition of list of features associated with a model. This can help streamline ML model development and operationalizing.This will be an input to the training pipeline (used to generate the training data), then will be packaged along with the model, and will be used during inference to lookup the features. It will be a glue that integrates all phases of the ML lifecycle. Changes to training/inference pipeline can be kept minimal as you experiment and deploy. \n", - "\n", - "Using feature retrieval spec and the built-in feature retrieval component is optional: you can directly use `get_offline_features()` api as shown above.\n", - "\n", - "Note that the name of the spec should be `feature_retrieval_spec.yaml` when it is packaged with the model for the system to recognize it." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697159142136 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "export-as-frspec", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# Create feature retrieval spec\n", - "feature_retrieval_spec_folder = root_dir + \"/project/fraud_model/feature_retrieval_spec\"\n", - "\n", - "# check if the folder exists, create one if not\n", - "if not os.path.exists(feature_retrieval_spec_folder):\n", - " os.makedirs(feature_retrieval_spec_folder)\n", - "\n", - "featurestore.generate_feature_retrieval_spec(feature_retrieval_spec_folder, features)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 4: Train in the cloud using pipelines and register model if satisfactory\n", - "In this step you will manually trigger the training pipeline. In a production scenario, this could be triggered by a ci/cd pipeline based on changes to the feature-retrieval spec in the source repository." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 4a: Run the training pipeline\n", - "The training pipeline has the following steps:\n", - "\n", - "1. Feature retrieval step: This is a built-in component takes as input the feature retrieval spec, the observation data and timestamp column name. It then generates the training data as output. It runs this as a managed spark job.\n", - "1. Training step: This step trains the model based on the training data and generates a model (not registered yet)\n", - "1. Evaluation step: This step validates whether model performance/quailty is within threshold (in our case it is a placeholder/dummy step for illustration purpose)\n", - "1. Register model step: This step registers the model\n", - "\n", - "Note: In part 2 of this tutorial you ran a backfill job to materialize data for `transactions` feature set. Feature retrieval step will read feature values from offline store for this feature set. The behavior will same even if you use `get_offline_features()` api." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697159196815 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "run-training-pipeline", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "training_pipeline_path = (\n", - " root_dir + \"/project/fraud_model/pipelines/training_pipeline.yaml\"\n", - ")\n", - "\n", - "!az ml job create --file $training_pipeline_path --resource-group $project_ws_rg --workspace-name $project_ws_name\n", - "# Note: First time it runs, each step in pipeline can take ~ 15 mins. However subsequent runs can be faster (assuming spark pool is warm - default timeout is 30 mins)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Inspect the training pipeline and the model\n", - "Open the above pipeline run \"web view\" in new window to see the steps in the pipeline.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 4b: Notice the feature retrieval spec in the model artifacts\n", - "1. In the left nav of the current workspace -> right click on `Models` -> Open in new tab or window\n", - "1. Click on `fraud_model`\n", - "1. Click on `Artifacts` in the top nav\n", - "\n", - "You can notice that the feature retrieval spec is packaged along with the model. The model registration step in the training pipeline has done this. You created feature retrieval spec during experimentation, now it has become part of the model definition. In the next tutorial you will see how this will be used during inferencing.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 5: View the feature set and model dependencies" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 5a: View the list of feature sets associated with the model\n", - "In the same models page, click on the `feature sets` tab. Here you can see both `transactions` and `accounts` featuresets that this model depends on." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Step 5b: View the list of models using the feature sets\n", - "1. Open the feature store UI (expalined in a previous step in this tutorial)\n", - "1. Click on `Feature sets` on the left nav\n", - "1. Click on any of the feature set -> click on `Models` tab\n", - "\n", - "You can see the list of models that are using the feature sets (determined from the feature retrieval spec when the model was registered)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Cleanup\n", - "\n", - "Tutorial of `Enable recurrent materialization and run batch inference` has instructions for deleting the resources" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Next steps\n", - "* Enable recurrent materialization and run batch inference" - ] - } - ], - "metadata": { - "celltoolbar": "Edit Metadata", - "kernel_info": { - "name": "synapse_pyspark" - }, - "kernelspec": { - "display_name": "Synapse PySpark", - "language": "Python", - "name": "synapse_pyspark" - }, - "language_info": { - "codemirror_mode": "ipython", - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython", - "version": "3.8.0" - }, - "microsoft": { - "host": { - "AzureML": { - "notebookHasBeenCompleted": true - } - }, - "ms_spell_check": { - "ms_spell_check_language": "en" - } - }, - "nteract": { - "version": "nteract-front-end@1.0.0" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "# Tutorial #2: Experiment and train models using features" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "In this tutorial series you will experience how features seamlessly integrates all the phases of ML lifecycle: Prototyping features, training and operationalizing.\n", + "\n", + "In part 1 of the tutorial you learnt how to create a feature set spec with custom transformations, enable materialization and perform backfill. In this tutorial you will will learn how to experiment with features to improve model performance. You will see how feature store increasing agility in the experimentation and training flows. \n", + "\n", + "You will perform the following:\n", + "- Prototype a create new `acccounts` feature set spec using existing precomputed values as features, unlike part 1 of the tutorial where we created feature set that had custom transformations. You will then Register the local feature set spec as a feature set in the feature store\n", + "- Select features for the model: You will select features from the `transactions` and `accounts` feature sets and save them as a feature-retrieval spec\n", + "- Run training pipeline that uses the Feature retrieval spec to train a new model. This pipeline will use the built in feature-retrieval component to generate the training data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "# Prerequisites\n", + "1. Please ensure you have executed `1. Develop a feature set and register with managed feature store` tutorial" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### (updated) Configure Azure ML spark notebook\n", + "\n", + "1. Running the tutorial: You can either create a new notebook, and execute the instructions in this document step by step or open the existing notebook named `Experiment and train models using features`, and run it. The notebooks are available in `featurestore_sample/notebooks` directory. You can select from `sdk_only` or `sdk_and_cli`. You may keep this document open and refer to it for additional explanation and documentation links.\n", + "1. In the \"Compute\" dropdown in the top nav, select \"Serverless Spark Compute\". \n", + "1. Click on \"configure session\" in top status bar -> click on \"Python packages\" -> click on \"upload conda file\" -> select the file azureml-examples/sdk/python/featurestore-sample/project/env/conda.yml from your local machine; Also increase the session time out (idle time) if you want to avoid running the prerequisites frequently\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Start spark session" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697158542512 }, - "nbformat": 4, - "nbformat_minor": 2 + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "start-spark-session", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# run this cell to start the spark session (any code block will start the session ). This can take around 10 mins.\n", + "print(\"start spark session\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Setup root directory for the samples" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697158549861 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "root-dir", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# please update the dir to ./Users/ (or any custom directory you uploaded the samples to).\n", + "# You can find the name from the directory structure inm the left nav\n", + "root_dir = \"./Users//featurestore_sample\"\n", + "\n", + "if os.path.isdir(root_dir):\n", + " print(\"The folder exists.\")\n", + "else:\n", + " print(\"The folder does not exist. Please create or fix the path\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### (new for sdk/cki track) Setup CLI\n", + "\n", + "1. Install azure ml cli extention\n", + "1. Authenticate\n", + "1. Set the default subscription" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "install-ml-ext-cli", + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [] + }, + "outputs": [], + "source": [ + "!az extension add --name ml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": true, + "source_hidden": false + }, + "name": "auth-cli", + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [] + }, + "outputs": [], + "source": [ + "!az login" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697158670086 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "set-default-subs-cli", + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "subscription_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", + "\n", + "!az account set -s $subscription_id" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Initialize the project workspace variables\n", + "This is the current workspace where you will be running the tutorial notebook from" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697158675914 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "init-ws-crud-client", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "project_ws_sub_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", + "project_ws_rg = os.environ[\"AZUREML_ARM_RESOURCEGROUP\"]\n", + "project_ws_name = os.environ[\"AZUREML_ARM_WORKSPACE_NAME\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Initialize the feature store variables\n", + "Ensure you update the `featurestore_name` to reflect what you created in part 1 of this tutorial" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697158680345 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "init-fs-crud-client", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "featurestore_name = (\n", + " \"\" # use the same name from part #1 of the tutorial\n", + ")\n", + "featurestore_subscription_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", + "featurestore_resource_group_name = os.environ[\"AZUREML_ARM_RESOURCEGROUP\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Initialize the feature store consumption client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697158696247 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "init-fs-core-sdk", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# feature store client\n", + "from azureml.featurestore import FeatureStoreClient\n", + "from azure.ai.ml.identity import AzureMLOnBehalfOfCredential\n", + "\n", + "featurestore = FeatureStoreClient(\n", + " credential=AzureMLOnBehalfOfCredential(),\n", + " subscription_id=featurestore_subscription_id,\n", + " resource_group_name=featurestore_resource_group_name,\n", + " name=featurestore_name,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Create compute cluster with name `cpu-cluster-fs` in the project workspace\n", + "This will be needed when we run training/batch inference jobs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697158803646 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "create-compute-cluster", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "compute_cluster_name = \"\"\n", + "type = \"\"\n", + "size = \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!az ml compute create --name $compute_cluster_name --type $type --size $size --idle-time-before-scale-down 360 --resource-group $project_ws_rg --workspace-name $project_ws_name" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 1: Create accounts featureset locally from precomputed data\n", + "In tutorial part 1, we created a transactions featureset that had custom transformations. Now we will create an accounts featureset that will use precomputed values. \n", + "\n", + "For onboarding precomputed features, you can create a featureset spec without writing any transformation code. Featureset spec is a specification to develop and test a featureset in a fully local/dev environment without connecting to any featurestore. In this step you will create the feature set spec locally and sample the values from it. If you want to get managed featurestore capabilities, you need to register the featureset spec with a feature store using a feature asset definition (a future step in this tutorial)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 1a: Explore the source data for accounts\n", + "\n", + "##### Note\n", + " Note that the sample data used in this notebook is hosted in a public accessible blob container. It can only be read in Spark via `wasbs` driver. When you create feature sets using your own source data, please host them in adls gen2 account and use `abfss` driver in the data path. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697158931390 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "explore-accts-fset-src-data", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "accounts_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/accounts-precalculated/*.parquet\"\n", + "accounts_df = spark.read.parquet(accounts_data_path)\n", + "\n", + "display(accounts_df.head(5))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 1b: Create `accounts` feature set spec in local from these precomputed features\n", + "Note that we do not need any transformation code here since we are referencing precomputed features." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697158944918 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "create-accts-fset-spec", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "from azureml.featurestore import create_feature_set_spec, FeatureSetSpec\n", + "from azureml.featurestore.contracts import (\n", + " DateTimeOffset,\n", + " FeatureSource,\n", + " TransformationCode,\n", + " Column,\n", + " ColumnType,\n", + " SourceType,\n", + " TimestampColumn,\n", + ")\n", + "\n", + "\n", + "accounts_featureset_spec = create_feature_set_spec(\n", + " source=FeatureSource(\n", + " type=SourceType.parquet,\n", + " path=\"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/accounts-precalculated/*.parquet\",\n", + " timestamp_column=TimestampColumn(name=\"timestamp\"),\n", + " ),\n", + " index_columns=[Column(name=\"accountID\", type=ColumnType.string)],\n", + " infer_schema=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "# Generate a spark dataframe from the feature set specification\n", + "accounts_fset_df = accounts_featureset_spec.to_spark_dataframe()\n", + "# display few records\n", + "display(accounts_fset_df.head(5))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 1c: Export as feature set spec\n", + "Inorder to register the feature set spec with the feature store, it needs to be saved in a specific format. \n", + "Action: After running the below cell, please inspect the generated `accounts` FeatureSetSpec: Open this file from the file tree to see the spec: `featurestore/featuresets/accounts/spec/FeatureSetSpec.yaml`\n", + "\n", + "Spec contains these important elements:\n", + "\n", + "1. `source`: reference to a storage. In this case a parquet file in a blob storage.\n", + "1. `features`: list of features and their datatypes. If you provide transformation code (see Day 2 section), the code has to return a dataframe that maps to the features and datatypes. In case where you do not provide transformation code (in this case of `accounts` because it is precomputed), the system will build the query to to map these to the source \n", + "1. `index_columns`: the join keys required to access values from the feature set\n", + "\n", + "Learn more about it in the [top level feature store entities document](fs-concepts-todo) and the [feature set spec yaml reference](reference-yaml-featureset-spec.md).\n", + "\n", + "The additional benefit of persisting it is that it can be source controlled." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697158964056 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "dump-accts-fset-spec", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# create a new folder to dump the feature set spec\n", + "accounts_featureset_spec_folder = root_dir + \"/featurestore/featuresets/accounts/spec\"\n", + "\n", + "# check if the folder exists, create one if not\n", + "if not os.path.exists(accounts_featureset_spec_folder):\n", + " os.makedirs(accounts_featureset_spec_folder)\n", + "\n", + "accounts_featureset_spec.dump(accounts_featureset_spec_folder, overwrite=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 2: Experiment with unregistered features locally and register with feature store when ready\n", + "When you are developing features, you might want to test/validate locally before registering with the feature store or running training pipelines in the cloud. In this step you will generate training data for the ML model from combination of features from a local unregistered feature set (`accounts`) and feature set registered in the feature store (`transactions`)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 2a: Select features for model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697158972685 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "select-unreg-features-for-model", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# get the registered transactions feature set, version 1\n", + "transactions_featureset = featurestore.feature_sets.get(\"transactions\", \"1\")\n", + "# Notice that account feature set spec is in your local dev environment (this notebook): not registered with feature store yet\n", + "features = [\n", + " accounts_featureset_spec.get_feature(\"accountAge\"),\n", + " accounts_featureset_spec.get_feature(\"numPaymentRejects1dPerUser\"),\n", + " transactions_featureset.get_feature(\"transaction_amount_7d_sum\"),\n", + " transactions_featureset.get_feature(\"transaction_amount_3d_sum\"),\n", + " transactions_featureset.get_feature(\"transaction_amount_7d_avg\"),\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 2b: Generate training data locally\n", + "In this step we generate training data for illustrative purpose. You can optionally train models locally with this. In the upcoming steps in this tutorial, you will train a model in the cloud." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697159002277 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "gen-training-data-locally", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "from azureml.featurestore import get_offline_features\n", + "\n", + "# Load the observation data. To understand observatio ndata, refer to part 1 of this tutorial\n", + "observation_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/observation_data/train/*.parquet\"\n", + "observation_data_df = spark.read.parquet(observation_data_path)\n", + "obs_data_timestamp_column = \"timestamp\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "# generate training dataframe by using feature data and observation data\n", + "training_df = get_offline_features(\n", + " features=features,\n", + " observation_data=observation_data_df,\n", + " timestamp_column=obs_data_timestamp_column,\n", + ")\n", + "\n", + "# Ignore the message that says feature set is not materialized (materialization is optional). We will enable materialization in the next part of the tutorial.\n", + "display(training_df)\n", + "# Note: display(training_df.head(5)) displays the timestamp column in a different format. You can can call training_df.show() to see correctly formatted value" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 2c: Register the `accounts` featureset with the featurestore\n", + "Once you have experimented with different feature definitions locally and sanity tested it, you can register it with the feature store.\n", + "For this you will register a featureset asset definition with the feature store.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697159036348 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "reg-accts-fset", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "feature_set_yaml = root_dir + \"/featurestore/featuresets/accounts/featureset_asset.yaml\"\n", + "\n", + "!az ml feature-set create --file $feature_set_yaml --resource-group $featurestore_resource_group_name --feature-store-name $featurestore_name" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 2d: Get registered featureset and sanity test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697159055950 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "sample-accts-fset-data", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# look up the featureset by providing name and version\n", + "accounts_featureset = featurestore.feature_sets.get(\"accounts\", \"1\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "# get access to the feature data\n", + "accounts_feature_df = accounts_featureset.to_spark_dataframe()\n", + "display(accounts_feature_df.head(5))\n", + "# Note: Please ignore this warning: Failure while loading azureml_run_type_providers. Failed to load entrypoint azureml.scriptrun" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 3: Run training experiment\n", + "In this step you will select a list of features, run a training pipeline, and register the model. You can repeat this step till you are happy with the model performance." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### (Optional) Step 3a: Discover features from Feature Store UI\n", + "You have already done this in part 1 of the tutorial after registering the `transactions` feature set. Since you also have `accounts` featureset, you can browse the available features:\n", + "* Goto the [Azure ML global landing page](https://ml.azure.com/home?flight=FeatureStores).\n", + "* Click on `Feature stores` in the left nav\n", + "* You will see the list of feature stores that you have access to. Click on the feature store that you created above.\n", + "\n", + "You can see the feature sets and entity that you created. Click on the feature sets to browse the feature definitions. You can also search for feature sets across feature stores by using the global search box." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### (Optional) Step 3b: Discover features from SDK" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697159077800 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "discover-features-from-sdk", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# List available feature sets\n", + "all_featuresets = featurestore.feature_sets.list()\n", + "for fs in all_featuresets:\n", + " print(fs)\n", + "\n", + "# List of versions for transactions feature set\n", + "all_transactions_featureset_versions = featurestore.feature_sets.list(\n", + " name=\"transactions\"\n", + ")\n", + "for fs in all_transactions_featureset_versions:\n", + " print(fs)\n", + "\n", + "# See properties of the transactions featureset including list of features\n", + "featurestore.feature_sets.get(name=\"transactions\", version=\"1\").features" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 3c: Select features for the model and export it as a feature-retrieval spec\n", + "In the previous steps, you selected features from a combination unregistered and registered feature sets for local experimentation and testing. Now you are ready to experiment in the cloud. Saving the selected features as a feature-retrieval spec and using it in the mlops/cicd flow for training/inference increases your agility in shipping models." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "Select features for the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697159131879 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "select-reg-features", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# you can select features in pythonic way\n", + "features = [\n", + " accounts_featureset.get_feature(\"accountAge\"),\n", + " transactions_featureset.get_feature(\"transaction_amount_7d_sum\"),\n", + " transactions_featureset.get_feature(\"transaction_amount_3d_sum\"),\n", + "]\n", + "\n", + "# you can also specify features in string form: featurestore:featureset:version:feature\n", + "more_features = [\n", + " \"accounts:1:numPaymentRejects1dPerUser\",\n", + " \"transactions:1:transaction_amount_7d_avg\",\n", + "]\n", + "\n", + "more_features = featurestore.resolve_feature_uri(more_features)\n", + "\n", + "features.extend(more_features)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "Export selected features as a feature-retrieval spec\n", + "\n", + "#### Note\n", + "Feature retrieval spec is a portable definition of list of features associated with a model. This can help streamline ML model development and operationalizing.This will be an input to the training pipeline (used to generate the training data), then will be packaged along with the model, and will be used during inference to lookup the features. It will be a glue that integrates all phases of the ML lifecycle. Changes to training/inference pipeline can be kept minimal as you experiment and deploy. \n", + "\n", + "Using feature retrieval spec and the built-in feature retrieval component is optional: you can directly use `get_offline_features()` api as shown above.\n", + "\n", + "Note that the name of the spec should be `feature_retrieval_spec.yaml` when it is packaged with the model for the system to recognize it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697159142136 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "export-as-frspec", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# Create feature retrieval spec\n", + "feature_retrieval_spec_folder = root_dir + \"/project/fraud_model/feature_retrieval_spec\"\n", + "\n", + "# check if the folder exists, create one if not\n", + "if not os.path.exists(feature_retrieval_spec_folder):\n", + " os.makedirs(feature_retrieval_spec_folder)\n", + "\n", + "featurestore.generate_feature_retrieval_spec(feature_retrieval_spec_folder, features)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 4: Train in the cloud using pipelines and register model if satisfactory\n", + "In this step you will manually trigger the training pipeline. In a production scenario, this could be triggered by a ci/cd pipeline based on changes to the feature-retrieval spec in the source repository." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 4a: Run the training pipeline\n", + "The training pipeline has the following steps:\n", + "\n", + "1. Feature retrieval step: This is a built-in component takes as input the feature retrieval spec, the observation data and timestamp column name. It then generates the training data as output. It runs this as a managed spark job.\n", + "1. Training step: This step trains the model based on the training data and generates a model (not registered yet)\n", + "1. Evaluation step: This step validates whether model performance/quailty is within threshold (in our case it is a placeholder/dummy step for illustration purpose)\n", + "1. Register model step: This step registers the model\n", + "\n", + "Note: In part 2 of this tutorial you ran a backfill job to materialize data for `transactions` feature set. Feature retrieval step will read feature values from offline store for this feature set. The behavior will same even if you use `get_offline_features()` api." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697159196815 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "name": "run-training-pipeline", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "training_pipeline_path = (\n", + " root_dir + \"/project/fraud_model/pipelines/training_pipeline.yaml\"\n", + ")\n", + "\n", + "!az ml job create --file $training_pipeline_path --resource-group $project_ws_rg --workspace-name $project_ws_name\n", + "# Note: First time it runs, each step in pipeline can take ~ 15 mins. However subsequent runs can be faster (assuming spark pool is warm - default timeout is 30 mins)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Inspect the training pipeline and the model\n", + "Open the above pipeline run \"web view\" in new window to see the steps in the pipeline.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 4b: Notice the feature retrieval spec in the model artifacts\n", + "1. In the left nav of the current workspace -> right click on `Models` -> Open in new tab or window\n", + "1. Click on `fraud_model`\n", + "1. Click on `Artifacts` in the top nav\n", + "\n", + "You can notice that the feature retrieval spec is packaged along with the model. The model registration step in the training pipeline has done this. You created feature retrieval spec during experimentation, now it has become part of the model definition. In the next tutorial you will see how this will be used during inferencing.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 5: View the feature set and model dependencies" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 5a: View the list of feature sets associated with the model\n", + "In the same models page, click on the `feature sets` tab. Here you can see both `transactions` and `accounts` featuresets that this model depends on." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Step 5b: View the list of models using the feature sets\n", + "1. Open the feature store UI (expalined in a previous step in this tutorial)\n", + "1. Click on `Feature sets` on the left nav\n", + "1. Click on any of the feature set -> click on `Models` tab\n", + "\n", + "You can see the list of models that are using the feature sets (determined from the feature retrieval spec when the model was registered)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Cleanup\n", + "\n", + "Tutorial of `Enable recurrent materialization and run batch inference` has instructions for deleting the resources" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Next steps\n", + "* Enable recurrent materialization and run batch inference" + ] + } + ], + "metadata": { + "celltoolbar": "Edit Metadata", + "kernel_info": { + "name": "synapse_pyspark" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": "ipython", + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython", + "version": "3.8.0" + }, + "microsoft": { + "host": { + "AzureML": { + "notebookHasBeenCompleted": true + } + }, + "ms_spell_check": { + "ms_spell_check_language": "en" + } + }, + "nteract": { + "version": "nteract-front-end@1.0.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/sdk/python/featurestore_sample/notebooks/sdk_and_cli/3.Enable-recurrent-materialization-run-batch-inference.ipynb b/sdk/python/featurestore_sample/notebooks/sdk_and_cli/3.Enable-recurrent-materialization-run-batch-inference.ipynb index 1f51ed39e6..89d5419921 100644 --- a/sdk/python/featurestore_sample/notebooks/sdk_and_cli/3.Enable-recurrent-materialization-run-batch-inference.ipynb +++ b/sdk/python/featurestore_sample/notebooks/sdk_and_cli/3.Enable-recurrent-materialization-run-batch-inference.ipynb @@ -1,630 +1,630 @@ { - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# Tutorial #3: Enable recurrent materialization and run batch inference" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "In this tutorial series you will experience how features seamlessly integrates all the phases of ML lifecycle: Prototyping features, training and operationalizing.\n", - "\n", - "In the previous part of the tutorial you learnt to experiment with features, train the model and register the model along with the feature-retrieval spec. In this tutorial you will learn how to run batch inference for the registered model.\n", - "\n", - "You will perform the following:\n", - "- Enable recurrent materialization for the `transactions` feature set\n", - "- Run batch inference pipeline on the registered model" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# Prerequisites\n", - "1. Please ensure you have executed the previous parts of this tutorial series" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# Setup" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### (updated) Configure Azure ML spark notebook\n", - "\n", - "1. Running the tutorial: You can either create a new notebook, and execute the instructions in this document step by step or open the existing notebook named `Enable recurrent materialization and run batch inference`, and run it. The notebooks are available in `featurestore_sample/notebooks` directory. You can select from `sdk_only` or `sdk_and_cli`. You may keep this document open and refer to it for additional explanation and documentation links.\n", - "1. In the \"Compute\" dropdown in the top nav, select \"Serverless Spark Compute\". \n", - "1. Click on \"configure session\" in top status bar -> click on \"Python packages\" -> click on \"upload conda file\" -> select the file azureml-examples/sdk/python/featurestore-sample/project/env/conda.yml from your local machine; Also increase the session time out (idle time) if you want to avoid running the prerequisites frequently\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Start spark session" - ] + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "# Tutorial #3: Enable recurrent materialization and run batch inference" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "In this tutorial series you will experience how features seamlessly integrates all the phases of ML lifecycle: Prototyping features, training and operationalizing.\n", + "\n", + "In the previous part of the tutorial you learnt to experiment with features, train the model and register the model along with the feature-retrieval spec. In this tutorial you will learn how to run batch inference for the registered model.\n", + "\n", + "You will perform the following:\n", + "- Enable recurrent materialization for the `transactions` feature set\n", + "- Run batch inference pipeline on the registered model" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "# Prerequisites\n", + "1. Please ensure you have executed the previous parts of this tutorial series" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "# Setup" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### (updated) Configure Azure ML spark notebook\n", + "\n", + "1. Running the tutorial: You can either create a new notebook, and execute the instructions in this document step by step or open the existing notebook named `Enable recurrent materialization and run batch inference`, and run it. The notebooks are available in `featurestore_sample/notebooks` directory. You can select from `sdk_only` or `sdk_and_cli`. You may keep this document open and refer to it for additional explanation and documentation links.\n", + "1. In the \"Compute\" dropdown in the top nav, select \"Serverless Spark Compute\". \n", + "1. Click on \"configure session\" in top status bar -> click on \"Python packages\" -> click on \"upload conda file\" -> select the file azureml-examples/sdk/python/featurestore-sample/project/env/conda.yml from your local machine; Also increase the session time out (idle time) if you want to avoid running the prerequisites frequently\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Start spark session" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697159743790 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697159743790 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "start-spark-session", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# run this cell to start the spark session (any code block will start the session ). This can take around 10 mins.\n", - "print(\"start spark session\")" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Setup root directory for the samples" - ] + "name": "start-spark-session", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# run this cell to start the spark session (any code block will start the session ). This can take around 10 mins.\n", + "print(\"start spark session\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Setup root directory for the samples" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697159769987 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697159769987 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "root-dir", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# please update the dir to ./Users/ (or any custom directory you uploaded the samples to).\n", - "# You can find the name from the directory structure inm the left nav\n", - "root_dir = \"./Users//featurestore_sample\"\n", - "\n", - "if os.path.isdir(root_dir):\n", - " print(\"The folder exists.\")\n", - "else:\n", - " print(\"The folder does not exist. Please create or fix the path\")" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### (new for sdk/cki track) Setup CLI\n", - "\n", - "1. Install azure ml cli extention\n", - "1. Authenticate\n", - "1. Set the default subscription" - ] + "name": "root-dir", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# please update the dir to ./Users/ (or any custom directory you uploaded the samples to).\n", + "# You can find the name from the directory structure inm the left nav\n", + "root_dir = \"./Users//featurestore_sample\"\n", + "\n", + "if os.path.isdir(root_dir):\n", + " print(\"The folder exists.\")\n", + "else:\n", + " print(\"The folder does not exist. Please create or fix the path\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### (new for sdk/cki track) Setup CLI\n", + "\n", + "1. Install azure ml cli extention\n", + "1. Authenticate\n", + "1. Set the default subscription" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "install-ml-ext-cli", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "!az extension add --name ml" - ] + "name": "install-ml-ext-cli", + "nteract": { + "transient": { + "deleting": false + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": true, - "source_hidden": false - }, - "name": "auth-cli", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "!az login" - ] + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "!az extension add --name ml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": true, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697159883337 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "set-default-subs-cli", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "subscription_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", - "\n", - "!az account set -s $subscription_id" - ] + "name": "auth-cli", + "nteract": { + "transient": { + "deleting": false + } }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Initialize the project workspace CRUD client\n", - "This is the current workspace where you will be running the tutorial notebook from" - ] + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "!az login" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697159883337 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697159897063 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "init-ws-crud-client", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "### Initialize the MLClient of this project workspace\n", - "import os\n", - "from azure.ai.ml import MLClient\n", - "from azure.ai.ml.identity import AzureMLOnBehalfOfCredential\n", - "\n", - "project_ws_sub_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", - "project_ws_rg = os.environ[\"AZUREML_ARM_RESOURCEGROUP\"]\n", - "project_ws_name = os.environ[\"AZUREML_ARM_WORKSPACE_NAME\"]\n", - "\n", - "# connect to the project workspace\n", - "ws_client = MLClient(\n", - " AzureMLOnBehalfOfCredential(), project_ws_sub_id, project_ws_rg, project_ws_name\n", - ")" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Initialize the feature store variables\n", - "Ensure you update the `featurestore_name` to reflect what you created in part 1 of this tutorial" - ] + "name": "set-default-subs-cli", + "nteract": { + "transient": { + "deleting": false + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697159943735 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "init-fs-crud-client", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "featurestore_name = (\n", - " \"\" # use the same name from part #1 of the tutorial\n", - ")\n", - "featurestore_subscription_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", - "featurestore_resource_group_name = os.environ[\"AZUREML_ARM_RESOURCEGROUP\"]" - ] + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "subscription_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", + "\n", + "!az account set -s $subscription_id" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Initialize the project workspace CRUD client\n", + "This is the current workspace where you will be running the tutorial notebook from" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697159897063 }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Initialize the feature store core sdk client" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697159952751 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "init-fs-core-sdk", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# feature store client\n", - "from azureml.featurestore import FeatureStoreClient\n", - "from azure.ai.ml.identity import AzureMLOnBehalfOfCredential\n", - "\n", - "featurestore = FeatureStoreClient(\n", - " credential=AzureMLOnBehalfOfCredential(),\n", - " subscription_id=featurestore_subscription_id,\n", - " resource_group_name=featurestore_resource_group_name,\n", - " name=featurestore_name,\n", - ")" - ] + "name": "init-ws-crud-client", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "### Initialize the MLClient of this project workspace\n", + "import os\n", + "from azure.ai.ml import MLClient\n", + "from azure.ai.ml.identity import AzureMLOnBehalfOfCredential\n", + "\n", + "project_ws_sub_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", + "project_ws_rg = os.environ[\"AZUREML_ARM_RESOURCEGROUP\"]\n", + "project_ws_name = os.environ[\"AZUREML_ARM_WORKSPACE_NAME\"]\n", + "\n", + "# connect to the project workspace\n", + "ws_client = MLClient(\n", + " AzureMLOnBehalfOfCredential(), project_ws_sub_id, project_ws_rg, project_ws_name\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Initialize the feature store variables\n", + "Ensure you update the `featurestore_name` to reflect what you created in part 1 of this tutorial" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697159943735 }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 1: Enable recurrent materialization on the `transactions` featureset\n", - "\n", - "In part 2 of this tutorial you enabled materialization and performed backfill on the transactions feature set. Backfill is an ondemand one-time operation to compute and store feature values in the materialization store. However when you want to perform inference of the model in production, you might want to keep the materilization store upto date by setting up recurrent materialization jobs. These jobs run on user defined schedule\n", - "The recurrent job schedule works in the following way: \n", - "- A window is defined by the interval and frequency. E.g., interval = 3 and frequency = Hour define a 3-hour window\n", - "- The first window starts at the start_time defined in the RecurenceTrigger, and so on.\n", - "- The first recurrent job will be submitted at the begining of the next window after the update time.\n", - "- Later recurrent jobs will be submitted at every window after the first job.\n", - "\n", - "As explained in the previous parts of the tutorials, once data is materialized (backfill/recurrent materialization), feature retrieval will use the materialized data by default." - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697160005186 - }, - "jupyter": { - "outputs_hidden": true, - "source_hidden": false - }, - "name": "enable-recurrent-mat-txns-fset", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "feature_set_schedule_yaml = (\n", - " root_dir\n", - " + \"/featurestore/featuresets/transactions/featureset_asset_offline_enabled_with_schedule.yaml\"\n", - ")\n", - "\n", - "!az ml feature-set update --file $feature_set_schedule_yaml --resource-group $featurestore_resource_group_name --feature-store-name $featurestore_name" - ] + "name": "init-fs-crud-client", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "featurestore_name = (\n", + " \"\" # use the same name from part #1 of the tutorial\n", + ")\n", + "featurestore_subscription_id = os.environ[\"AZUREML_ARM_SUBSCRIPTION\"]\n", + "featurestore_resource_group_name = os.environ[\"AZUREML_ARM_RESOURCEGROUP\"]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Initialize the feature store core sdk client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697159952751 }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Track status of the recurrent materialization jobs in the feature store studio UI\n", - "This job will every three hours. \n", - "\n", - "__Action__:\n", - "\n", - "1. Feel free to execute the next step for now (batch inference).\n", - "1. In three hours check the recurrent job status via the UI" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Step 2: Run the batch-inference pipeline\n", - "\n", - "In this step you will manually trigger the batch inference pipeline. In a production scenario, this could be trigerred by a ci/cd pipeline based on model registration/approval.\n", - "\n", - "The batch-inference has the following steps:\n", - "\n", - "1. Feature retrieval step: This use the same built-in feature retrieval component that we used in the training pipeline in the part 3 of the tutorial. Incase of training pipeline, we provided feature retreival spec as an input to the component. However in case of batch inference we will pass the registered model as the input and the component will look for feature retrieval spec in the model artifact. Another difference is that in case of training, the observation data had the target variable, however incase of batch inference it will not be present. The feature retrieval step will join the observation data with the features and output the data for batch inference.\n", - "1. Batch inference: This step uses the batch inference input data from previous step, runs inference on the model and outputs the data by appending the predicted value.\n", - "\n", - "__Note:__ In this example we use a job for batch inference. You can also use Azure ML's batch endpoints." - ] + "name": "init-fs-core-sdk", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# feature store client\n", + "from azureml.featurestore import FeatureStoreClient\n", + "from azure.ai.ml.identity import AzureMLOnBehalfOfCredential\n", + "\n", + "featurestore = FeatureStoreClient(\n", + " credential=AzureMLOnBehalfOfCredential(),\n", + " subscription_id=featurestore_subscription_id,\n", + " resource_group_name=featurestore_resource_group_name,\n", + " name=featurestore_name,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 1: Enable recurrent materialization on the `transactions` featureset\n", + "\n", + "In part 2 of this tutorial you enabled materialization and performed backfill on the transactions feature set. Backfill is an ondemand one-time operation to compute and store feature values in the materialization store. However when you want to perform inference of the model in production, you might want to keep the materilization store upto date by setting up recurrent materialization jobs. These jobs run on user defined schedule\n", + "The recurrent job schedule works in the following way: \n", + "- A window is defined by the interval and frequency. E.g., interval = 3 and frequency = Hour define a 3-hour window\n", + "- The first window starts at the start_time defined in the RecurenceTrigger, and so on.\n", + "- The first recurrent job will be submitted at the begining of the next window after the update time.\n", + "- Later recurrent jobs will be submitted at every window after the first job.\n", + "\n", + "As explained in the previous parts of the tutorials, once data is materialized (backfill/recurrent materialization), feature retrieval will use the materialized data by default." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697160005186 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1697160098403 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "run-batch-inf-pipeline", - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# set the batch inference pipeline path\n", - "batch_inference_pipeline_path = (\n", - " root_dir + \"/project/fraud_model/pipelines/batch_inference_pipeline.yaml\"\n", - ")\n", - "\n", - "!az ml job create --file $batch_inference_pipeline_path --resource-group $project_ws_rg --workspace-name $project_ws_name" - ] + "jupyter": { + "outputs_hidden": true, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Inspect the batch inference output data\n", - "1. In the pipeline view, double click on `inference_step` -> in `outputs` card, copy the `Data` field. It will be something like `azureml_995abbc2-3171-461e-8214-c3c5d17ede83_output_data_data_with_prediction:1`. \n", - "1. Paste it in the below cell with name and version separately (notice that the last character is the version, separated by a `:`).\n", - "1. You will see the `predict_is_fraud` column generated by the batch inference pipeline\n", - "\n", - "Explanation: Since we did not provide a `name` and `version` in the `outputs` of the `inference_step` in the batch inference pipeline (`/project/fraud_mode/pipelines/batch_inference_pipeline.yaml`), the system created an untracked data asset with a guid as name and version as 1. In the next cell we will be getting the data path from the asset and displaying it." - ] + "name": "enable-recurrent-mat-txns-fset", + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "feature_set_schedule_yaml = (\n", + " root_dir\n", + " + \"/featurestore/featuresets/transactions/featureset_asset_offline_enabled_with_schedule.yaml\"\n", + ")\n", + "\n", + "!az ml feature-set update --file $feature_set_schedule_yaml --resource-group $featurestore_resource_group_name --feature-store-name $featurestore_name" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Track status of the recurrent materialization jobs in the feature store studio UI\n", + "This job will every three hours. \n", + "\n", + "__Action__:\n", + "\n", + "1. Feel free to execute the next step for now (batch inference).\n", + "1. In three hours check the recurrent job status via the UI" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Step 2: Run the batch-inference pipeline\n", + "\n", + "In this step you will manually trigger the batch inference pipeline. In a production scenario, this could be trigerred by a ci/cd pipeline based on model registration/approval.\n", + "\n", + "The batch-inference has the following steps:\n", + "\n", + "1. Feature retrieval step: This use the same built-in feature retrieval component that we used in the training pipeline in the part 3 of the tutorial. Incase of training pipeline, we provided feature retreival spec as an input to the component. However in case of batch inference we will pass the registered model as the input and the component will look for feature retrieval spec in the model artifact. Another difference is that in case of training, the observation data had the target variable, however incase of batch inference it will not be present. The feature retrieval step will join the observation data with the features and output the data for batch inference.\n", + "1. Batch inference: This step uses the batch inference input data from previous step, runs inference on the model and outputs the data by appending the predicted value.\n", + "\n", + "__Note:__ In this example we use a job for batch inference. You can also use Azure ML's batch endpoints." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1697160098403 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1684255304980 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "name": "inspect-batch-inf-output-data", - "nteract": { - "transient": { - "deleting": false - } - }, - "tags": [ - "active-ipynb" - ] - }, - "outputs": [], - "source": [ - "# inf_data_output = ws_client.data.get(name=\"azureml_1c106662-aa5e-4354-b5f9-57c1b0fdb3a7_output_data_data_with_prediction\", version=\"1\")\n", - "inf_data_output = ws_client.data.get(\n", - " name=\"azureml_0a7417c8-409a-4536-a069-4ea23a08ebfe_output_data_data_with_prediction\",\n", - " version=\"1\",\n", - ")\n", - "inf_output_df = spark.read.parquet(inf_data_output.path + \"data/*.parquet\")\n", - "display(inf_output_df.head(5))" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Cleanup\n", - "If you created a resource group for the tutorial, you can delete the resource group to delete all the resources associated with this tutorial.\n", - "\n", - "Otherwise, you can delete the resources individually:\n", - "\n", - "* Delete the feature store: Go to the resource group in the azure portal, select the feature store and delete it\n", - "* Follow the instructions [here](https://review.learn.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities?pivots=identity-mi-methods-azp&view=azureml-api-2#delete-a-user-assigned-managed-identity) to delete the user assigned managed identity\n", - "* Delete the offline store (storage account): Go to the resource group in the azure portal, select the storage you created and delete it" - ] + "name": "run-batch-inf-pipeline", + "nteract": { + "transient": { + "deleting": false + } } - ], - "metadata": { - "celltoolbar": "Edit Metadata", - "kernel_info": { - "name": "synapse_pyspark" - }, - "kernelspec": { - "display_name": "Synapse PySpark", - "language": "Python", - "name": "synapse_pyspark" + }, + "outputs": [], + "source": [ + "# set the batch inference pipeline path\n", + "batch_inference_pipeline_path = (\n", + " root_dir + \"/project/fraud_model/pipelines/batch_inference_pipeline.yaml\"\n", + ")\n", + "\n", + "!az ml job create --file $batch_inference_pipeline_path --resource-group $project_ws_rg --workspace-name $project_ws_name" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "#### Inspect the batch inference output data\n", + "1. In the pipeline view, double click on `inference_step` -> in `outputs` card, copy the `Data` field. It will be something like `azureml_995abbc2-3171-461e-8214-c3c5d17ede83_output_data_data_with_prediction:1`. \n", + "1. Paste it in the below cell with name and version separately (notice that the last character is the version, separated by a `:`).\n", + "1. You will see the `predict_is_fraud` column generated by the batch inference pipeline\n", + "\n", + "Explanation: Since we did not provide a `name` and `version` in the `outputs` of the `inference_step` in the batch inference pipeline (`/project/fraud_mode/pipelines/batch_inference_pipeline.yaml`), the system created an untracked data asset with a guid as name and version as 1. In the next cell we will be getting the data path from the asset and displaying it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1684255304980 }, - "language_info": { - "codemirror_mode": "ipython", - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython", - "version": "3.8.0" + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - "microsoft": { - "host": { - "AzureML": { - "notebookHasBeenCompleted": true - } - }, - "ms_spell_check": { - "ms_spell_check_language": "en" - } + "name": "inspect-batch-inf-output-data", + "nteract": { + "transient": { + "deleting": false + } }, + "tags": [ + "active-ipynb" + ] + }, + "outputs": [], + "source": [ + "# inf_data_output = ws_client.data.get(name=\"azureml_1c106662-aa5e-4354-b5f9-57c1b0fdb3a7_output_data_data_with_prediction\", version=\"1\")\n", + "inf_data_output = ws_client.data.get(\n", + " name=\"azureml_0a7417c8-409a-4536-a069-4ea23a08ebfe_output_data_data_with_prediction\",\n", + " version=\"1\",\n", + ")\n", + "inf_output_df = spark.read.parquet(inf_data_output.path + \"data/*.parquet\")\n", + "display(inf_output_df.head(5))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { "nteract": { - "version": "nteract-front-end@1.0.0" + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Cleanup\n", + "If you created a resource group for the tutorial, you can delete the resource group to delete all the resources associated with this tutorial.\n", + "\n", + "Otherwise, you can delete the resources individually:\n", + "\n", + "* Delete the feature store: Go to the resource group in the azure portal, select the feature store and delete it\n", + "* Follow the instructions [here](https://review.learn.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities?pivots=identity-mi-methods-azp&view=azureml-api-2#delete-a-user-assigned-managed-identity) to delete the user assigned managed identity\n", + "* Delete the offline store (storage account): Go to the resource group in the azure portal, select the storage you created and delete it" + ] + } + ], + "metadata": { + "celltoolbar": "Edit Metadata", + "kernel_info": { + "name": "synapse_pyspark" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": "ipython", + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython", + "version": "3.8.0" + }, + "microsoft": { + "host": { + "AzureML": { + "notebookHasBeenCompleted": true } + }, + "ms_spell_check": { + "ms_spell_check_language": "en" + } }, - "nbformat": 4, - "nbformat_minor": 2 + "nteract": { + "version": "nteract-front-end@1.0.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/sdk/python/featurestore_sample/notebooks/sdk_and_cli/4.Provision-feature-store.ipynb b/sdk/python/featurestore_sample/notebooks/sdk_and_cli/4.Provision-feature-store.ipynb index 7708c9c37f..e5633783e7 100644 --- a/sdk/python/featurestore_sample/notebooks/sdk_and_cli/4.Provision-feature-store.ipynb +++ b/sdk/python/featurestore_sample/notebooks/sdk_and_cli/4.Provision-feature-store.ipynb @@ -1661,9 +1661,9 @@ "name": "python38-azureml" }, "kernelspec": { - "display_name": "Python 3.8 - AzureML", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python38-azureml" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/featurestore_sample/notebooks/sdk_and_cli/network_isolation/Network-isolation-feature-store.ipynb b/sdk/python/featurestore_sample/notebooks/sdk_and_cli/network_isolation/Network-isolation-feature-store.ipynb index bff5f75cba..e200955dc5 100644 --- a/sdk/python/featurestore_sample/notebooks/sdk_and_cli/network_isolation/Network-isolation-feature-store.ipynb +++ b/sdk/python/featurestore_sample/notebooks/sdk_and_cli/network_isolation/Network-isolation-feature-store.ipynb @@ -2623,9 +2623,9 @@ "name": "synapse_pyspark" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/featurestore_sample/notebooks/sdk_only/1.Develop-feature-set-and-register.ipynb b/sdk/python/featurestore_sample/notebooks/sdk_only/1.Develop-feature-set-and-register.ipynb index 75996e86d9..91822a4fae 100644 --- a/sdk/python/featurestore_sample/notebooks/sdk_only/1.Develop-feature-set-and-register.ipynb +++ b/sdk/python/featurestore_sample/notebooks/sdk_only/1.Develop-feature-set-and-register.ipynb @@ -1280,9 +1280,9 @@ "name": "synapse_pyspark" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/featurestore_sample/notebooks/sdk_only/2.Experiment-train-models-using-features.ipynb b/sdk/python/featurestore_sample/notebooks/sdk_only/2.Experiment-train-models-using-features.ipynb index cbeff4055b..dd55ea393d 100644 --- a/sdk/python/featurestore_sample/notebooks/sdk_only/2.Experiment-train-models-using-features.ipynb +++ b/sdk/python/featurestore_sample/notebooks/sdk_only/2.Experiment-train-models-using-features.ipynb @@ -1185,9 +1185,9 @@ "name": "synapse_pyspark" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/featurestore_sample/notebooks/sdk_only/3.Enable-recurrent-materialization-run-batch-inference.ipynb b/sdk/python/featurestore_sample/notebooks/sdk_only/3.Enable-recurrent-materialization-run-batch-inference.ipynb index 819f09f4a6..80873d0af9 100644 --- a/sdk/python/featurestore_sample/notebooks/sdk_only/3.Enable-recurrent-materialization-run-batch-inference.ipynb +++ b/sdk/python/featurestore_sample/notebooks/sdk_only/3.Enable-recurrent-materialization-run-batch-inference.ipynb @@ -561,9 +561,9 @@ "name": "synapse_pyspark" }, "kernelspec": { - "display_name": "Synapse PySpark", - "language": "Python", - "name": "synapse_pyspark" + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": "ipython", diff --git a/sdk/python/featurestore_sample/notebooks/sdk_only/4.Enable-online-store-run-inference.ipynb b/sdk/python/featurestore_sample/notebooks/sdk_only/4.Enable-online-store-run-inference.ipynb index b1b07611d8..da7c813040 100644 --- a/sdk/python/featurestore_sample/notebooks/sdk_only/4.Enable-online-store-run-inference.ipynb +++ b/sdk/python/featurestore_sample/notebooks/sdk_only/4.Enable-online-store-run-inference.ipynb @@ -1253,9 +1253,9 @@ "name": "synapse_pyspark" }, "kernelspec": { - "display_name": "Synapse PySpark", - "language": "Python", - "name": "synapse_pyspark" + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": "ipython", diff --git a/sdk/python/featurestore_sample/notebooks/sdk_only/5.Develop-feature-set-custom-source.ipynb b/sdk/python/featurestore_sample/notebooks/sdk_only/5.Develop-feature-set-custom-source.ipynb index 9cac0d4d66..66cae18b0a 100644 --- a/sdk/python/featurestore_sample/notebooks/sdk_only/5.Develop-feature-set-custom-source.ipynb +++ b/sdk/python/featurestore_sample/notebooks/sdk_only/5.Develop-feature-set-custom-source.ipynb @@ -477,9 +477,9 @@ "name": "synapse_pyspark" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/featurestore_sample/notebooks/sdk_only/7.Develop-feature-set-domain-specific-language-dsl.ipynb b/sdk/python/featurestore_sample/notebooks/sdk_only/7.Develop-feature-set-domain-specific-language-dsl.ipynb index 55f4aeef1f..fb49e9f383 100644 --- a/sdk/python/featurestore_sample/notebooks/sdk_only/7.Develop-feature-set-domain-specific-language-dsl.ipynb +++ b/sdk/python/featurestore_sample/notebooks/sdk_only/7.Develop-feature-set-domain-specific-language-dsl.ipynb @@ -1628,9 +1628,9 @@ ], "metadata": { "kernelspec": { - "display_name": "fs_env", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "fs_env" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/featurestore_sample/notebooks/sdk_only/ADB Example - Develop a feature set and register with managed feature store.ipynb b/sdk/python/featurestore_sample/notebooks/sdk_only/ADB Example - Develop a feature set and register with managed feature store.ipynb index 06a051cca5..41c2645f96 100644 --- a/sdk/python/featurestore_sample/notebooks/sdk_only/ADB Example - Develop a feature set and register with managed feature store.ipynb +++ b/sdk/python/featurestore_sample/notebooks/sdk_only/ADB Example - Develop a feature set and register with managed feature store.ipynb @@ -1522,9 +1522,9 @@ "name": "synapse_pyspark" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/featurestore_sample/notebooks/sdk_only/Synapse example - Develop feature set and register.ipynb b/sdk/python/featurestore_sample/notebooks/sdk_only/Synapse example - Develop feature set and register.ipynb index 3afbba9226..668a0fd14a 100644 --- a/sdk/python/featurestore_sample/notebooks/sdk_only/Synapse example - Develop feature set and register.ipynb +++ b/sdk/python/featurestore_sample/notebooks/sdk_only/Synapse example - Develop feature set and register.ipynb @@ -1529,9 +1529,9 @@ "name": "synapse_pyspark" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/ai21-labs/jamba-1-5/ai21_azure_client.ipynb b/sdk/python/foundation-models/ai21-labs/jamba-1-5/ai21_azure_client.ipynb index e0356860d2..dbedce09ff 100644 --- a/sdk/python/foundation-models/ai21-labs/jamba-1-5/ai21_azure_client.ipynb +++ b/sdk/python/foundation-models/ai21-labs/jamba-1-5/ai21_azure_client.ipynb @@ -723,9 +723,9 @@ ], "metadata": { "kernelspec": { - "display_name": "jupyter", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/ai21-labs/jamba-1-5/langchain.ipynb b/sdk/python/foundation-models/ai21-labs/jamba-1-5/langchain.ipynb index a1c4afd7f9..b9d803a77f 100644 --- a/sdk/python/foundation-models/ai21-labs/jamba-1-5/langchain.ipynb +++ b/sdk/python/foundation-models/ai21-labs/jamba-1-5/langchain.ipynb @@ -1,226 +1,226 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Use LangChain with AI21 Jamba 1.5 Large and Jamba 1.5 Mini for Azure AI Models-as-a-Service\n", - "\n", - "You can use AI21 Jamba 1.5 Large and Jamba 1.5 Mini deployed as a serverless API endpoint through Azure AI Studio and Azure ML with `langchain` to create more sophisticated intelligent applications.\n", - "\n", - "> Review the [documentation](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for AI21 Jamba 1.5 Large through AI Studio and for ML Studio for details on how to provision inference endpoints, regional availability, pricing and inference schema reference." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisites\n", - "\n", - "Before we start, there are certain steps we need to take to deploy the models:\n", - "\n", - "* Register for a valid Azure account with subscription \n", - "* Make sure you have access to [Azure AI Studio](https://learn.microsoft.com/en-us/azure/ai-studio/what-is-ai-studio?tabs=home)\n", - "* Create a project and resource group\n", - "* Select `AI21 Jamba 1.5 Large` or `AI21 Jamba 1.5 Mini`\n", - "\n", - " > Notice that some models may not be available in all the regions in Azure AI and Azure Machine Learning. On those cases, you can create a workspace or project in the region where the models are available and then consume it with a connection from a different one. To learn more about using connections see [Consume models with connections](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deployments-connections)\n", - "\n", - "* Deploy with \"Pay-as-you-go\"\n", - "\n", - "Once deployed successfully, you should be assigned for an API endpoint and a security key for inference.\n", - "\n", - "For more information, you should consult Azure's official documentation [here](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for model deployment and inference.\n", - "\n", - "To complete this tutorial, you will need to:\n", - "\n", - "* Install `langchain` and `langchain_ai21`:\n", - "\n", - " ```bash\n", - " pip install langchain langchain-ai21\n", - " ```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example\n", - "\n", - "The following example demonstrate how to create a chain that uses AI21 Jamba 1.5 Large model deployed through Azure AI and Azure ML. The chain has been configured with a `ConversationBufferMemory`. This example has been adapted from [LangChain official documentation](https://python.langchain.com/docs/modules/memory/adding_memory)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install langchain\n", - "%pip install langchain-ai21\n", - "%pip install -U \"ai21>=2.13.0\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "imports" - }, - "outputs": [], - "source": [ - "from langchain.chains import LLMChain\n", - "from langchain.memory import ConversationBufferMemory\n", - "from langchain.prompts import (\n", - " ChatPromptTemplate,\n", - " HumanMessagePromptTemplate,\n", - " MessagesPlaceholder,\n", - ")\n", - "from langchain.schema import SystemMessage\n", - "from langchain_ai21.chat_models import ChatAI21" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the below prompt, we have two input keys: one for the actual input (`human_input`), and another for the input from the `Memory` class (`chat_history`)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "prompt-template" - }, - "outputs": [], - "source": [ - "prompt = ChatPromptTemplate.from_messages(\n", - " [\n", - " SystemMessage(\n", - " content=\"You are a chatbot having a conversation with a human. You love making references to pop culture in your answers.\"\n", - " ),\n", - " MessagesPlaceholder(variable_name=\"chat_history\"),\n", - " HumanMessagePromptTemplate.from_template(\"{human_input}\"),\n", - " ]\n", - ")\n", - "\n", - "memory = ConversationBufferMemory(memory_key=\"chat_history\", return_messages=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's create an instance of our model deployed in Azure AI or Azure ML. Use `langchain_ai21` package and configure it as follows:\n", - "\n", - "- `endpoint`: Use the endpoint URL from your deployment. Include `/v1` at the end of the endpoint URL.\n", - "- `api_key`: Use your API key." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "chat_client" - }, - "outputs": [], - "source": [ - "chat_model = ChatAI21(api_host=\"\", api_key=\"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> Tip: Alternatively, you can configure your API key in the environment variables `AI21_API_KEY`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We create the chain as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "chain" - }, - "outputs": [], - "source": [ - "chat_llm_chain = LLMChain(\n", - " llm=chat_model,\n", - " prompt=prompt,\n", - " memory=memory,\n", - " verbose=True,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see how it works:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "chat_invoke" - }, - "outputs": [], - "source": [ - "chat_llm_chain.predict(human_input=\"Hi there my friend\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "chat_invoke_2" - }, - "outputs": [], - "source": [ - "chat_llm_chain.predict(\n", - " human_input=\"Who is the most renowned architect in the world? Provide a short poem that describes their work in the style of Shakespeare with Iambic pentimeter and a rhythm pattern of ABABCC\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Aditional resources\n", - "\n", - "Here are some additional reference: \n", - "\n", - "* [Plan and manage costs (marketplace)](https://learn.microsoft.com/azure/ai-studio/how-to/costs-plan-manage#monitor-costs-for-models-offered-through-the-azure-marketplace)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "jupyter", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - } - }, - "nbformat": 4, - "nbformat_minor": 2 + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Use LangChain with AI21 Jamba 1.5 Large and Jamba 1.5 Mini for Azure AI Models-as-a-Service\n", + "\n", + "You can use AI21 Jamba 1.5 Large and Jamba 1.5 Mini deployed as a serverless API endpoint through Azure AI Studio and Azure ML with `langchain` to create more sophisticated intelligent applications.\n", + "\n", + "> Review the [documentation](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for AI21 Jamba 1.5 Large through AI Studio and for ML Studio for details on how to provision inference endpoints, regional availability, pricing and inference schema reference." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "\n", + "Before we start, there are certain steps we need to take to deploy the models:\n", + "\n", + "* Register for a valid Azure account with subscription \n", + "* Make sure you have access to [Azure AI Studio](https://learn.microsoft.com/en-us/azure/ai-studio/what-is-ai-studio?tabs=home)\n", + "* Create a project and resource group\n", + "* Select `AI21 Jamba 1.5 Large` or `AI21 Jamba 1.5 Mini`\n", + "\n", + " > Notice that some models may not be available in all the regions in Azure AI and Azure Machine Learning. On those cases, you can create a workspace or project in the region where the models are available and then consume it with a connection from a different one. To learn more about using connections see [Consume models with connections](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deployments-connections)\n", + "\n", + "* Deploy with \"Pay-as-you-go\"\n", + "\n", + "Once deployed successfully, you should be assigned for an API endpoint and a security key for inference.\n", + "\n", + "For more information, you should consult Azure's official documentation [here](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for model deployment and inference.\n", + "\n", + "To complete this tutorial, you will need to:\n", + "\n", + "* Install `langchain` and `langchain_ai21`:\n", + "\n", + " ```bash\n", + " pip install langchain langchain-ai21\n", + " ```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example\n", + "\n", + "The following example demonstrate how to create a chain that uses AI21 Jamba 1.5 Large model deployed through Azure AI and Azure ML. The chain has been configured with a `ConversationBufferMemory`. This example has been adapted from [LangChain official documentation](https://python.langchain.com/docs/modules/memory/adding_memory)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install langchain\n", + "%pip install langchain-ai21\n", + "%pip install -U \"ai21>=2.13.0\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "imports" + }, + "outputs": [], + "source": [ + "from langchain.chains import LLMChain\n", + "from langchain.memory import ConversationBufferMemory\n", + "from langchain.prompts import (\n", + " ChatPromptTemplate,\n", + " HumanMessagePromptTemplate,\n", + " MessagesPlaceholder,\n", + ")\n", + "from langchain.schema import SystemMessage\n", + "from langchain_ai21.chat_models import ChatAI21" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the below prompt, we have two input keys: one for the actual input (`human_input`), and another for the input from the `Memory` class (`chat_history`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "prompt-template" + }, + "outputs": [], + "source": [ + "prompt = ChatPromptTemplate.from_messages(\n", + " [\n", + " SystemMessage(\n", + " content=\"You are a chatbot having a conversation with a human. You love making references to pop culture in your answers.\"\n", + " ),\n", + " MessagesPlaceholder(variable_name=\"chat_history\"),\n", + " HumanMessagePromptTemplate.from_template(\"{human_input}\"),\n", + " ]\n", + ")\n", + "\n", + "memory = ConversationBufferMemory(memory_key=\"chat_history\", return_messages=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's create an instance of our model deployed in Azure AI or Azure ML. Use `langchain_ai21` package and configure it as follows:\n", + "\n", + "- `endpoint`: Use the endpoint URL from your deployment. Include `/v1` at the end of the endpoint URL.\n", + "- `api_key`: Use your API key." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "chat_client" + }, + "outputs": [], + "source": [ + "chat_model = ChatAI21(api_host=\"\", api_key=\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> Tip: Alternatively, you can configure your API key in the environment variables `AI21_API_KEY`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We create the chain as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "chain" + }, + "outputs": [], + "source": [ + "chat_llm_chain = LLMChain(\n", + " llm=chat_model,\n", + " prompt=prompt,\n", + " memory=memory,\n", + " verbose=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see how it works:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "chat_invoke" + }, + "outputs": [], + "source": [ + "chat_llm_chain.predict(human_input=\"Hi there my friend\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "chat_invoke_2" + }, + "outputs": [], + "source": [ + "chat_llm_chain.predict(\n", + " human_input=\"Who is the most renowned architect in the world? Provide a short poem that describes their work in the style of Shakespeare with Iambic pentimeter and a rhythm pattern of ABABCC\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Aditional resources\n", + "\n", + "Here are some additional reference: \n", + "\n", + "* [Plan and manage costs (marketplace)](https://learn.microsoft.com/azure/ai-studio/how-to/costs-plan-manage#monitor-costs-for-models-offered-through-the-azure-marketplace)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/sdk/python/foundation-models/ai21-labs/jamba-1-5/litellm.ipynb b/sdk/python/foundation-models/ai21-labs/jamba-1-5/litellm.ipynb index 7f8146fd46..141f92c8a3 100644 --- a/sdk/python/foundation-models/ai21-labs/jamba-1-5/litellm.ipynb +++ b/sdk/python/foundation-models/ai21-labs/jamba-1-5/litellm.ipynb @@ -1,172 +1,172 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Use LiteLLM with AI21 Jamba 1.5 Large and Jamba 1.5 Mini in Azure AI and Azure ML\n", - "\n", - "You can use AI21 Jamba 1.5 Large and Jamba 1.5 Mini deployed as a serverless API endpoint through Azure AI Studio and Azure ML with `litellm` to create more sophisticated intelligent applications.\n", - "\n", - "> Review the [documentation](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for AI21 Jamba 1.5 Large through AI Studio and for ML Studio for details on how to provision inference endpoints, regional availability, pricing and inference schema reference." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisites\n", - "\n", - "Before we start, there are certain steps we need to take to deploy the models:\n", - "\n", - "* Register for a valid Azure account with subscription \n", - "* Make sure you have access to [Azure AI Studio](https://learn.microsoft.com/en-us/azure/ai-studio/what-is-ai-studio?tabs=home)\n", - "* Create a project and resource group\n", - "* Select `AI21 Jamba 1.5 Large` or `AI21 Jamba 1.5 Mini`\n", - "\n", - " > Notice that some models may not be available in all the regions in Azure AI and Azure Machine Learning. On those cases, you can create a workspace or project in the region where the models are available and then consume it with a connection from a different one. To learn more about using connections see [Consume models with connections](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deployments-connections)\n", - "\n", - "* Deploy with \"Pay-as-you-go\"\n", - "\n", - "Once deployed successfully, you should be assigned for an API endpoint and a security key for inference.\n", - "\n", - "For more information, you should consult Azure's official documentation [here](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for model deployment and inference.\n", - "\n", - "To complete this tutorial, you will need to:\n", - "\n", - "* Install `litellm`:\n", - "\n", - " ```bash\n", - " pip install litellm\n", - " ```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example\n", - "\n", - "The following is an example about how to use `litellm` with AI21 Jamba 1.5 Large deployed through Azure AI and Azure ML:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "imports" - }, - "outputs": [], - "source": [ - "import litellm" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You will need to have a Endpoint url and Authentication Key associated with that endpoint. This can be acquired from previous steps. To work with `litellm`, configure the client as follows:\n", - "\n", - "- `base_url`: Use the endpoint URL from your deployment. Include the `/v1` in the URL.\n", - "- `api_key`: Use your API key." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "chat_client" - }, - "outputs": [], - "source": [ - "client = litellm.LiteLLM(base_url=\"\", api_key=\"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Use the client to create chat completions requests:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "chat_invoke" - }, - "outputs": [], - "source": [ - "response = client.chat.completions.create(\n", - " messages=[\n", - " {\n", - " \"content\": \"Who is the most renowned architect in the world? Provide a short poem that describes their work in the style of Shakespeare with Iambic pentimeter and a rhythm pattern of ABABCC\",\n", - " \"role\": \"user\",\n", - " }\n", - " ],\n", - " model=\"jamba-1.5-large\",\n", - " custom_llm_provider=\"custom_openai\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The generated text can be accessed as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "chat_response" - }, - "outputs": [], - "source": [ - "print(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Aditional resources\n", - "\n", - "Here are some additional reference: \n", - "\n", - "* [Plan and manage costs (marketplace)](https://learn.microsoft.com/azure/ai-studio/how-to/costs-plan-manage#monitor-costs-for-models-offered-through-the-azure-marketplace)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Use LiteLLM with AI21 Jamba 1.5 Large and Jamba 1.5 Mini in Azure AI and Azure ML\n", + "\n", + "You can use AI21 Jamba 1.5 Large and Jamba 1.5 Mini deployed as a serverless API endpoint through Azure AI Studio and Azure ML with `litellm` to create more sophisticated intelligent applications.\n", + "\n", + "> Review the [documentation](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for AI21 Jamba 1.5 Large through AI Studio and for ML Studio for details on how to provision inference endpoints, regional availability, pricing and inference schema reference." + ] }, - "nbformat": 4, - "nbformat_minor": 2 + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "\n", + "Before we start, there are certain steps we need to take to deploy the models:\n", + "\n", + "* Register for a valid Azure account with subscription \n", + "* Make sure you have access to [Azure AI Studio](https://learn.microsoft.com/en-us/azure/ai-studio/what-is-ai-studio?tabs=home)\n", + "* Create a project and resource group\n", + "* Select `AI21 Jamba 1.5 Large` or `AI21 Jamba 1.5 Mini`\n", + "\n", + " > Notice that some models may not be available in all the regions in Azure AI and Azure Machine Learning. On those cases, you can create a workspace or project in the region where the models are available and then consume it with a connection from a different one. To learn more about using connections see [Consume models with connections](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deployments-connections)\n", + "\n", + "* Deploy with \"Pay-as-you-go\"\n", + "\n", + "Once deployed successfully, you should be assigned for an API endpoint and a security key for inference.\n", + "\n", + "For more information, you should consult Azure's official documentation [here](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for model deployment and inference.\n", + "\n", + "To complete this tutorial, you will need to:\n", + "\n", + "* Install `litellm`:\n", + "\n", + " ```bash\n", + " pip install litellm\n", + " ```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example\n", + "\n", + "The following is an example about how to use `litellm` with AI21 Jamba 1.5 Large deployed through Azure AI and Azure ML:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install litellm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "imports" + }, + "outputs": [], + "source": [ + "import litellm" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You will need to have a Endpoint url and Authentication Key associated with that endpoint. This can be acquired from previous steps. To work with `litellm`, configure the client as follows:\n", + "\n", + "- `base_url`: Use the endpoint URL from your deployment. Include the `/v1` in the URL.\n", + "- `api_key`: Use your API key." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "chat_client" + }, + "outputs": [], + "source": [ + "client = litellm.LiteLLM(base_url=\"\", api_key=\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use the client to create chat completions requests:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "chat_invoke" + }, + "outputs": [], + "source": [ + "response = client.chat.completions.create(\n", + " messages=[\n", + " {\n", + " \"content\": \"Who is the most renowned architect in the world? Provide a short poem that describes their work in the style of Shakespeare with Iambic pentimeter and a rhythm pattern of ABABCC\",\n", + " \"role\": \"user\",\n", + " }\n", + " ],\n", + " model=\"jamba-1.5-large\",\n", + " custom_llm_provider=\"custom_openai\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The generated text can be accessed as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "chat_response" + }, + "outputs": [], + "source": [ + "print(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Aditional resources\n", + "\n", + "Here are some additional reference: \n", + "\n", + "* [Plan and manage costs (marketplace)](https://learn.microsoft.com/azure/ai-studio/how-to/costs-plan-manage#monitor-costs-for-models-offered-through-the-azure-marketplace)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/sdk/python/foundation-models/ai21-labs/jamba-1-5/openaisdk.ipynb b/sdk/python/foundation-models/ai21-labs/jamba-1-5/openaisdk.ipynb index 09ec42e58e..8e8277f2d8 100644 --- a/sdk/python/foundation-models/ai21-labs/jamba-1-5/openaisdk.ipynb +++ b/sdk/python/foundation-models/ai21-labs/jamba-1-5/openaisdk.ipynb @@ -1,183 +1,183 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Use OpenAI SDK with AI21 Jamba 1.5 Large and Jamba 1.5 Mini in Azure AI and Azure ML\n", - "\n", - "Use `openai` SDK to consume AI21 Jamba 1.5 Large and Jamba 1.5 Mini deployments through Azure AI and Azure ML. AI21 Jamba 1.5 Large through Azure AI and Azure ML offers an API compatible with the OpenAI Chat Completion API. It allows customers and users to transition seamlessly from OpenAI models to AI21 LLMs. \n", - "\n", - "The API can be directly used with OpenAI's client libraries or third-party tools, like LangChain or LiteLLM.\n", - "\n", - "The example below shows how to make this transition using the OpenAI Python Library. Notice that the Azure AI chat completions API supports only a portion of the parameters supported by OpenAI API.\n", - "\n", - "> Review the [documentation](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for AI21 Jamba 1.5 Large through AI Studio and for ML Studio for details on how to provision inference endpoints, regional availability, pricing and inference schema reference." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisites\n", - "\n", - "Before we start, there are certain steps we need to take to deploy the models:\n", - "\n", - "* Register for a valid Azure account with subscription \n", - "* Make sure you have access to [Azure AI Studio](https://learn.microsoft.com/en-us/azure/ai-studio/what-is-ai-studio?tabs=home)\n", - "* Create a project and resource group\n", - "* Select `AI21 Jamba 1.5 Large` or `AI21 Jamba 1.5 Mini`\n", - "\n", - " > Notice that some models may not be available in all the regions in Azure AI and Azure Machine Learning. On those cases, you can create a workspace or project in the region where the models are available and then consume it with a connection from a different one. To learn more about using connections see [Consume models with connections](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deployments-connections)\n", - "\n", - "* Deploy with \"Pay-as-you-go\"\n", - "\n", - "Once deployed successfully, you should be assigned for an API endpoint and a security key for inference.\n", - "\n", - "For more information, you should consult Azure's official documentation [here](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for model deployment and inference.\n", - "\n", - "To complete this tutorial, you will need to:\n", - "\n", - "* Install `openai`:\n", - "\n", - " ```bash\n", - " pip install openai\n", - " ```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example\n", - "\n", - "The following is an example about how to use `openai` with a AI21 Jamba 1.5 Large deployed in Azure AI and Azure ML:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install openai" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "imports" - }, - "outputs": [], - "source": [ - "from openai import OpenAI" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You will need to have a Endpoint url and Authentication Key associated with that endpoint. This can be acquired from previous steps. \n", - "To work with `openai`, configure the client as follows:\n", - "\n", - "- `base_url`: Use the endpoint URL from your deployment. Include `/v1` as part of the URL.\n", - "- `api_key`: Use your API key." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "chat_client" - }, - "outputs": [], - "source": [ - "client = OpenAI(base_url=\"\", api_key=\"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Use the client to create chat completions requests:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "chat_invoke" - }, - "outputs": [], - "source": [ - "response = client.chat.completions.create(\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Who is the most renowned architect in the world? Provide a short poem that describes their work in the style of Shakespeare with Iambic pentimeter and a rhythm pattern of ABABCC\",\n", - " }\n", - " ],\n", - " model=\"jamba-1.5-large\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The generated text can be accessed as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "chat_response" - }, - "outputs": [], - "source": [ - "print(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Notice that not all the parameters supported by the OpenAI API is supported. See Azure AI documentation for the full list of supported arguments." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Aditional resources\n", - "\n", - "Here are some additional reference: \n", - "\n", - "* [Plan and manage costs (marketplace)](https://learn.microsoft.com/azure/ai-studio/how-to/costs-plan-manage#monitor-costs-for-models-offered-through-the-azure-marketplace)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Use OpenAI SDK with AI21 Jamba 1.5 Large and Jamba 1.5 Mini in Azure AI and Azure ML\n", + "\n", + "Use `openai` SDK to consume AI21 Jamba 1.5 Large and Jamba 1.5 Mini deployments through Azure AI and Azure ML. AI21 Jamba 1.5 Large through Azure AI and Azure ML offers an API compatible with the OpenAI Chat Completion API. It allows customers and users to transition seamlessly from OpenAI models to AI21 LLMs. \n", + "\n", + "The API can be directly used with OpenAI's client libraries or third-party tools, like LangChain or LiteLLM.\n", + "\n", + "The example below shows how to make this transition using the OpenAI Python Library. Notice that the Azure AI chat completions API supports only a portion of the parameters supported by OpenAI API.\n", + "\n", + "> Review the [documentation](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for AI21 Jamba 1.5 Large through AI Studio and for ML Studio for details on how to provision inference endpoints, regional availability, pricing and inference schema reference." + ] }, - "nbformat": 4, - "nbformat_minor": 2 + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "\n", + "Before we start, there are certain steps we need to take to deploy the models:\n", + "\n", + "* Register for a valid Azure account with subscription \n", + "* Make sure you have access to [Azure AI Studio](https://learn.microsoft.com/en-us/azure/ai-studio/what-is-ai-studio?tabs=home)\n", + "* Create a project and resource group\n", + "* Select `AI21 Jamba 1.5 Large` or `AI21 Jamba 1.5 Mini`\n", + "\n", + " > Notice that some models may not be available in all the regions in Azure AI and Azure Machine Learning. On those cases, you can create a workspace or project in the region where the models are available and then consume it with a connection from a different one. To learn more about using connections see [Consume models with connections](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deployments-connections)\n", + "\n", + "* Deploy with \"Pay-as-you-go\"\n", + "\n", + "Once deployed successfully, you should be assigned for an API endpoint and a security key for inference.\n", + "\n", + "For more information, you should consult Azure's official documentation [here](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for model deployment and inference.\n", + "\n", + "To complete this tutorial, you will need to:\n", + "\n", + "* Install `openai`:\n", + "\n", + " ```bash\n", + " pip install openai\n", + " ```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example\n", + "\n", + "The following is an example about how to use `openai` with a AI21 Jamba 1.5 Large deployed in Azure AI and Azure ML:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install openai" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "imports" + }, + "outputs": [], + "source": [ + "from openai import OpenAI" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You will need to have a Endpoint url and Authentication Key associated with that endpoint. This can be acquired from previous steps. \n", + "To work with `openai`, configure the client as follows:\n", + "\n", + "- `base_url`: Use the endpoint URL from your deployment. Include `/v1` as part of the URL.\n", + "- `api_key`: Use your API key." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "chat_client" + }, + "outputs": [], + "source": [ + "client = OpenAI(base_url=\"\", api_key=\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use the client to create chat completions requests:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "chat_invoke" + }, + "outputs": [], + "source": [ + "response = client.chat.completions.create(\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Who is the most renowned architect in the world? Provide a short poem that describes their work in the style of Shakespeare with Iambic pentimeter and a rhythm pattern of ABABCC\",\n", + " }\n", + " ],\n", + " model=\"jamba-1.5-large\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The generated text can be accessed as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "chat_response" + }, + "outputs": [], + "source": [ + "print(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that not all the parameters supported by the OpenAI API is supported. See Azure AI documentation for the full list of supported arguments." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Aditional resources\n", + "\n", + "Here are some additional reference: \n", + "\n", + "* [Plan and manage costs (marketplace)](https://learn.microsoft.com/azure/ai-studio/how-to/costs-plan-manage#monitor-costs-for-models-offered-through-the-azure-marketplace)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/sdk/python/foundation-models/ai21-labs/jamba-1-5/webrequests.ipynb b/sdk/python/foundation-models/ai21-labs/jamba-1-5/webrequests.ipynb index d1a7725993..1dfa7fe9c5 100644 --- a/sdk/python/foundation-models/ai21-labs/jamba-1-5/webrequests.ipynb +++ b/sdk/python/foundation-models/ai21-labs/jamba-1-5/webrequests.ipynb @@ -1,282 +1,282 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Use Azure API with AI21 Jamba 1.5 Large and Jamba 1.5 Mini through Azure AI Models-as-a-Service\n", - "\n", - "This notebook shows examples of how to use AI21 Jamba 1.5 Large and Jamba 1.5 Mini offered by Microsoft Azure AI and Azure ML, delivered as serverless APIs through Models-as-a-Service (MaaS).\n", - "\n", - "> Review the [documentation](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for AI21 Jamba 1.5 Large on AI Studio and [ML Studio](https://aka.ms/ai21-jamba-1.5-large-azure-ml-docs) for details on how to provision inference endpoints, regional availability, pricing and inference schema reference." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisite\n", - "\n", - "Before we start, there are certain steps we need to take to deploy the models:\n", - "\n", - "* Register for a valid Azure account with subscription \n", - "* Make sure you have access to [Azure AI Studio](https://learn.microsoft.com/en-us/azure/ai-studio/what-is-ai-studio?tabs=home)\n", - "* Create a project and resource group\n", - "* Select `AI21 Jamba 1.5 Large` or `AI21 Jamba 1.5 Mini`\n", - "\n", - " > Notice that some models may not be available in all the regions in Azure AI and Azure Machine Learning. On those cases, you can create a workspace or project in the region where the models are available and then consume it with a connection from a different one. To learn more about using connections see [Consume models with connections](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deployments-connections)\n", - "\n", - "* Deploy with \"Pay-as-you-go\"\n", - "\n", - "Once deployed successfully, you should be assigned for an API endpoint and a security key for inference.\n", - "\n", - "For more information, you should consult Azure's official documentation [here](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for model deployment and inference." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## HTTP Requests API Usage\n", - "\n", - "Models deployed in Azure AI and Azure ML exposes 2 different APIs you can consume:\n", - "\n", - "* AI21 API\n", - "* Azure AI inference API\n", - "\n", - "- The **AI21 API** is designed to work with AI21 tooling specific tooling and it follows the AI21 public documentation.\n", - "- The **Azure AI inference API** is designed to work with Azure AI compatible tooling. This API is also OpenAI compatible.\n", - "\n", - "The following examples shows how to use Azure AI inference API. For details about AI21 API you can see \"need link here\"\n", - "\n", - "### Basics\n", - "\n", - "For using the REST API, You will need to have a Endpoint url and Authentication Key associated with that endpoint. \n", - "This can be acquired from previous steps. \n", - "\n", - "In this chat completion example, we use a simple curl call for illustration. There are three major components: \n", - "\n", - "* The `host-url` is your endpoint url with chat completion schema `/v1/chat/completions`. \n", - "* The `headers` defines the content type as well as your api key. \n", - "* The `payload` or `data`, which is your prompt detail and model hyper parameters." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!curl -X POST -L https://inference.ai.azure.com/v1/chat/completions -H 'Content-Type: application/json' -H 'Authorization: ' -d '{\"messages\":[{\"content\":\"You are a helpful assistant.\",\"role\":\"system\"},{\"content\":\"What is good about Wuhan?\",\"role\":\"user\"}], \"max_tokens\": 50}'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Streaming\n", - "\n", - "One fantastic feature the API offered is the streaming capability. Streaming allows the generated tokens to be sent as data-only server-sent events whenever they become available. This is extremely important for interactive applications such as chatbots, so the user is always engaged. \n", - "\n", - "To use streaming, simply set `\"stream\":\"True\"` as part of the request payload. \n", - "In the streaming mode, the REST API response will be different from non-streaming mode.\n", - "\n", - "Here is an example: " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!curl -X POST -L https://your-endpoint.inference.ai.azure.com/v1/chat/completions -H 'Content-Type: application/json' -H 'Authorization: your-auth-key' -d '{\"messages\":[{\"content\":\"You are a helpful assistant.\",\"role\":\"system\"},{\"content\":\"What is good about Wuhan?\",\"role\":\"user\"}], \"max_tokens\": 500, \"stream\": \"True\"}'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As you can see the result comes back as a stream of `data` objects, each contains generated information including a `choice`. \n", - "The stream terminated by a `data:[DONE]\\n\\n` message." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Content Safety Filtering\n", - "\n", - "All AI21 Jamba 1.5 Large MaaS endpoints will have content safety feature turned on. Both input prompt and output tokens are filtered by this service automatically. \n", - "To know more about the impact to the request/response payload, please refer to official guide [here](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/content-filter?tabs=python). \n", - "\n", - "For model input and output, if the filter detected there is harmful content. The generation will error out with reponse payload containing the reasoning, along with which type of content violation it is and severity. \n", - "\n", - "Here is an example prompt that triggered content safety filtering:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!curl -X POST -L https://your-endpoint.inference.ai.azure.com/v1/chat/completions -H 'Content-Type: application/json' -H 'Authorization: your-auth-key' -d '{\"messages\":[{\"content\":\"You are a helpful assistant.\",\"role\":\"system\"},{\"content\":\"How to make bomb?\",\"role\":\"user\"}], \"max_tokens\": 50}'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## HTTP Requests API Usage in Python\n", - "\n", - "Besides calling the API directly from command line tools. You can also programatically call them in Python. Here there is a chat completion example:\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import urllib.request\n", - "import json\n", - "\n", - "# Configure payload data sending to API endpoint\n", - "data = {\n", - " \"messages\": [\n", - " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Who is the most renowned architect in the world? Provide a short poem that describes their work in the style of Shakespeare with Iambic pentimeter and a rhythm pattern of ABABCC\",\n", - " },\n", - " ],\n", - " \"max_tokens\": 500,\n", - " \"temperature\": 0.9,\n", - " \"stream\": \"True\",\n", - "}\n", - "\n", - "body = str.encode(json.dumps(data))\n", - "\n", - "# Replace the url with your API endpoint\n", - "url = \"https://.inference.ai.azure.com/v1/chat/completions\"\n", - "\n", - "# Replace this with the key for the endpoint\n", - "api_key = \"\"\n", - "if not api_key:\n", - " raise Exception(\"API Key is missing\")\n", - "\n", - "headers = {\"Content-Type\": \"application/json\", \"Authorization\": (api_key)}\n", - "\n", - "req = urllib.request.Request(url, body, headers)\n", - "\n", - "try:\n", - " response = urllib.request.urlopen(req)\n", - " result = response.read()\n", - " print(result)\n", - "except urllib.error.HTTPError as error:\n", - " print(\"The request failed with status code: \" + str(error.code))\n", - " # Print the headers - they include the requert ID and the timestamp, which are useful for debugging the failure\n", - " print(error.info())\n", - " print(error.read().decode(\"utf8\", \"ignore\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "However in this example, the streamed data content returns back as a single payload. It didn't stream as a serial of data events as we wished. To build true streaming capabilities utilizing the API endpoint, we will utilize [`requests`](https://requests.readthedocs.io/en/latest/) library instead." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Streaming in Python\n", - "\n", - "`Requests` library is a simple HTTP library for Python built with [`urllib3`](https://github.com/urllib3/urllib3). It automatically maintains the keep-alive and HTTP connection pooling. With the `Session` class, we can easily stream the result from our API calls. \n", - "\n", - "Here is a quick example:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install requests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import requests\n", - "\n", - "data = {\n", - " \"messages\": [\n", - " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", - " {\"role\": \"user\", \"content\": \"What is good about Wuhan?\"},\n", - " ],\n", - " \"max_tokens\": 500,\n", - " \"temperature\": 0.9,\n", - " \"stream\": \"True\",\n", - "}\n", - "\n", - "\n", - "def post_stream(url):\n", - " s = requests.Session()\n", - " api_key = \"\"\n", - " headers = {\"Content-Type\": \"application/json\", \"Authorization\": (api_key)}\n", - "\n", - " with s.post(url, data=json.dumps(data), headers=headers, stream=True) as resp:\n", - " print(resp.status_code)\n", - " for line in resp.iter_lines():\n", - " if line:\n", - " print(line)\n", - "\n", - "\n", - "url = \"https://inference.ai.azure.com/v1/chat/completions\"\n", - "post_stream(url)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Aditional resources\n", - "\n", - "Here are some additional reference: \n", - "\n", - "* [Plan and manage costs (marketplace)](https://learn.microsoft.com/azure/ai-studio/how-to/costs-plan-manage#monitor-costs-for-models-offered-through-the-azure-marketplace)\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Use Azure API with AI21 Jamba 1.5 Large and Jamba 1.5 Mini through Azure AI Models-as-a-Service\n", + "\n", + "This notebook shows examples of how to use AI21 Jamba 1.5 Large and Jamba 1.5 Mini offered by Microsoft Azure AI and Azure ML, delivered as serverless APIs through Models-as-a-Service (MaaS).\n", + "\n", + "> Review the [documentation](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for AI21 Jamba 1.5 Large on AI Studio and [ML Studio](https://aka.ms/ai21-jamba-1.5-large-azure-ml-docs) for details on how to provision inference endpoints, regional availability, pricing and inference schema reference." + ] }, - "nbformat": 4, - "nbformat_minor": 2 + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisite\n", + "\n", + "Before we start, there are certain steps we need to take to deploy the models:\n", + "\n", + "* Register for a valid Azure account with subscription \n", + "* Make sure you have access to [Azure AI Studio](https://learn.microsoft.com/en-us/azure/ai-studio/what-is-ai-studio?tabs=home)\n", + "* Create a project and resource group\n", + "* Select `AI21 Jamba 1.5 Large` or `AI21 Jamba 1.5 Mini`\n", + "\n", + " > Notice that some models may not be available in all the regions in Azure AI and Azure Machine Learning. On those cases, you can create a workspace or project in the region where the models are available and then consume it with a connection from a different one. To learn more about using connections see [Consume models with connections](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deployments-connections)\n", + "\n", + "* Deploy with \"Pay-as-you-go\"\n", + "\n", + "Once deployed successfully, you should be assigned for an API endpoint and a security key for inference.\n", + "\n", + "For more information, you should consult Azure's official documentation [here](https://aka.ms/ai21-jamba-1.5-large-azure-ai-studio-docs) for model deployment and inference." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## HTTP Requests API Usage\n", + "\n", + "Models deployed in Azure AI and Azure ML exposes 2 different APIs you can consume:\n", + "\n", + "* AI21 API\n", + "* Azure AI inference API\n", + "\n", + "- The **AI21 API** is designed to work with AI21 tooling specific tooling and it follows the AI21 public documentation.\n", + "- The **Azure AI inference API** is designed to work with Azure AI compatible tooling. This API is also OpenAI compatible.\n", + "\n", + "The following examples shows how to use Azure AI inference API. For details about AI21 API you can see \"need link here\"\n", + "\n", + "### Basics\n", + "\n", + "For using the REST API, You will need to have a Endpoint url and Authentication Key associated with that endpoint. \n", + "This can be acquired from previous steps. \n", + "\n", + "In this chat completion example, we use a simple curl call for illustration. There are three major components: \n", + "\n", + "* The `host-url` is your endpoint url with chat completion schema `/v1/chat/completions`. \n", + "* The `headers` defines the content type as well as your api key. \n", + "* The `payload` or `data`, which is your prompt detail and model hyper parameters." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!curl -X POST -L https://inference.ai.azure.com/v1/chat/completions -H 'Content-Type: application/json' -H 'Authorization: ' -d '{\"messages\":[{\"content\":\"You are a helpful assistant.\",\"role\":\"system\"},{\"content\":\"What is good about Wuhan?\",\"role\":\"user\"}], \"max_tokens\": 50}'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming\n", + "\n", + "One fantastic feature the API offered is the streaming capability. Streaming allows the generated tokens to be sent as data-only server-sent events whenever they become available. This is extremely important for interactive applications such as chatbots, so the user is always engaged. \n", + "\n", + "To use streaming, simply set `\"stream\":\"True\"` as part of the request payload. \n", + "In the streaming mode, the REST API response will be different from non-streaming mode.\n", + "\n", + "Here is an example: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!curl -X POST -L https://your-endpoint.inference.ai.azure.com/v1/chat/completions -H 'Content-Type: application/json' -H 'Authorization: your-auth-key' -d '{\"messages\":[{\"content\":\"You are a helpful assistant.\",\"role\":\"system\"},{\"content\":\"What is good about Wuhan?\",\"role\":\"user\"}], \"max_tokens\": 500, \"stream\": \"True\"}'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see the result comes back as a stream of `data` objects, each contains generated information including a `choice`. \n", + "The stream terminated by a `data:[DONE]\\n\\n` message." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Content Safety Filtering\n", + "\n", + "All AI21 Jamba 1.5 Large MaaS endpoints will have content safety feature turned on. Both input prompt and output tokens are filtered by this service automatically. \n", + "To know more about the impact to the request/response payload, please refer to official guide [here](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/content-filter?tabs=python). \n", + "\n", + "For model input and output, if the filter detected there is harmful content. The generation will error out with reponse payload containing the reasoning, along with which type of content violation it is and severity. \n", + "\n", + "Here is an example prompt that triggered content safety filtering:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!curl -X POST -L https://your-endpoint.inference.ai.azure.com/v1/chat/completions -H 'Content-Type: application/json' -H 'Authorization: your-auth-key' -d '{\"messages\":[{\"content\":\"You are a helpful assistant.\",\"role\":\"system\"},{\"content\":\"How to make bomb?\",\"role\":\"user\"}], \"max_tokens\": 50}'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## HTTP Requests API Usage in Python\n", + "\n", + "Besides calling the API directly from command line tools. You can also programatically call them in Python. Here there is a chat completion example:\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import urllib.request\n", + "import json\n", + "\n", + "# Configure payload data sending to API endpoint\n", + "data = {\n", + " \"messages\": [\n", + " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Who is the most renowned architect in the world? Provide a short poem that describes their work in the style of Shakespeare with Iambic pentimeter and a rhythm pattern of ABABCC\",\n", + " },\n", + " ],\n", + " \"max_tokens\": 500,\n", + " \"temperature\": 0.9,\n", + " \"stream\": \"True\",\n", + "}\n", + "\n", + "body = str.encode(json.dumps(data))\n", + "\n", + "# Replace the url with your API endpoint\n", + "url = \"https://.inference.ai.azure.com/v1/chat/completions\"\n", + "\n", + "# Replace this with the key for the endpoint\n", + "api_key = \"\"\n", + "if not api_key:\n", + " raise Exception(\"API Key is missing\")\n", + "\n", + "headers = {\"Content-Type\": \"application/json\", \"Authorization\": (api_key)}\n", + "\n", + "req = urllib.request.Request(url, body, headers)\n", + "\n", + "try:\n", + " response = urllib.request.urlopen(req)\n", + " result = response.read()\n", + " print(result)\n", + "except urllib.error.HTTPError as error:\n", + " print(\"The request failed with status code: \" + str(error.code))\n", + " # Print the headers - they include the requert ID and the timestamp, which are useful for debugging the failure\n", + " print(error.info())\n", + " print(error.read().decode(\"utf8\", \"ignore\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However in this example, the streamed data content returns back as a single payload. It didn't stream as a serial of data events as we wished. To build true streaming capabilities utilizing the API endpoint, we will utilize [`requests`](https://requests.readthedocs.io/en/latest/) library instead." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming in Python\n", + "\n", + "`Requests` library is a simple HTTP library for Python built with [`urllib3`](https://github.com/urllib3/urllib3). It automatically maintains the keep-alive and HTTP connection pooling. With the `Session` class, we can easily stream the result from our API calls. \n", + "\n", + "Here is a quick example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import requests\n", + "\n", + "data = {\n", + " \"messages\": [\n", + " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", + " {\"role\": \"user\", \"content\": \"What is good about Wuhan?\"},\n", + " ],\n", + " \"max_tokens\": 500,\n", + " \"temperature\": 0.9,\n", + " \"stream\": \"True\",\n", + "}\n", + "\n", + "\n", + "def post_stream(url):\n", + " s = requests.Session()\n", + " api_key = \"\"\n", + " headers = {\"Content-Type\": \"application/json\", \"Authorization\": (api_key)}\n", + "\n", + " with s.post(url, data=json.dumps(data), headers=headers, stream=True) as resp:\n", + " print(resp.status_code)\n", + " for line in resp.iter_lines():\n", + " if line:\n", + " print(line)\n", + "\n", + "\n", + "url = \"https://inference.ai.azure.com/v1/chat/completions\"\n", + "post_stream(url)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Aditional resources\n", + "\n", + "Here are some additional reference: \n", + "\n", + "* [Plan and manage costs (marketplace)](https://learn.microsoft.com/azure/ai-studio/how-to/costs-plan-manage#monitor-costs-for-models-offered-through-the-azure-marketplace)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/sdk/python/foundation-models/ai21-labs/jamba-instruct/ai21_client.ipynb b/sdk/python/foundation-models/ai21-labs/jamba-instruct/ai21_client.ipynb index b4d714ba32..c6b8b6ea15 100644 --- a/sdk/python/foundation-models/ai21-labs/jamba-instruct/ai21_client.ipynb +++ b/sdk/python/foundation-models/ai21-labs/jamba-instruct/ai21_client.ipynb @@ -179,9 +179,9 @@ ], "metadata": { "kernelspec": { - "display_name": "jupyter", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/ai21-labs/jamba-instruct/langchain.ipynb b/sdk/python/foundation-models/ai21-labs/jamba-instruct/langchain.ipynb index 0f1642dd8d..1e9910e320 100644 --- a/sdk/python/foundation-models/ai21-labs/jamba-instruct/langchain.ipynb +++ b/sdk/python/foundation-models/ai21-labs/jamba-instruct/langchain.ipynb @@ -204,9 +204,9 @@ ], "metadata": { "kernelspec": { - "display_name": "jupyter", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/ai21-labs/jamba-instruct/litellm.ipynb b/sdk/python/foundation-models/ai21-labs/jamba-instruct/litellm.ipynb index 77e942bd6b..514791cdf5 100644 --- a/sdk/python/foundation-models/ai21-labs/jamba-instruct/litellm.ipynb +++ b/sdk/python/foundation-models/ai21-labs/jamba-instruct/litellm.ipynb @@ -150,9 +150,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/ai21-labs/jamba-instruct/openaisdk.ipynb b/sdk/python/foundation-models/ai21-labs/jamba-instruct/openaisdk.ipynb index e9767b5564..656d2138ee 100644 --- a/sdk/python/foundation-models/ai21-labs/jamba-instruct/openaisdk.ipynb +++ b/sdk/python/foundation-models/ai21-labs/jamba-instruct/openaisdk.ipynb @@ -161,9 +161,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/ai21-labs/jamba-instruct/webrequests.ipynb b/sdk/python/foundation-models/ai21-labs/jamba-instruct/webrequests.ipynb index 7a10f6812f..8fa0ca273f 100644 --- a/sdk/python/foundation-models/ai21-labs/jamba-instruct/webrequests.ipynb +++ b/sdk/python/foundation-models/ai21-labs/jamba-instruct/webrequests.ipynb @@ -260,9 +260,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/benchmarking/evaluating_claude_models.ipynb b/sdk/python/foundation-models/benchmarking/evaluating_claude_models.ipynb index e8ab04631c..c071cc2a30 100644 --- a/sdk/python/foundation-models/benchmarking/evaluating_claude_models.ipynb +++ b/sdk/python/foundation-models/benchmarking/evaluating_claude_models.ipynb @@ -194,9 +194,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/benchmarking/evaluating_oai_models.ipynb b/sdk/python/foundation-models/benchmarking/evaluating_oai_models.ipynb index ef81f023b1..befe4f1066 100644 --- a/sdk/python/foundation-models/benchmarking/evaluating_oai_models.ipynb +++ b/sdk/python/foundation-models/benchmarking/evaluating_oai_models.ipynb @@ -188,9 +188,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/cohere/cohere-aisearch-langchain-rag.ipynb b/sdk/python/foundation-models/cohere/cohere-aisearch-langchain-rag.ipynb index 2cf9888132..2396cc05fe 100644 --- a/sdk/python/foundation-models/cohere/cohere-aisearch-langchain-rag.ipynb +++ b/sdk/python/foundation-models/cohere/cohere-aisearch-langchain-rag.ipynb @@ -354,9 +354,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/cohere/cohere-aisearch-rag.ipynb b/sdk/python/foundation-models/cohere/cohere-aisearch-rag.ipynb index 161a8290f0..6a08501017 100644 --- a/sdk/python/foundation-models/cohere/cohere-aisearch-rag.ipynb +++ b/sdk/python/foundation-models/cohere/cohere-aisearch-rag.ipynb @@ -508,9 +508,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/cohere/cohere-cmdR.ipynb b/sdk/python/foundation-models/cohere/cohere-cmdR.ipynb index 219ed251b7..f46b12c464 100644 --- a/sdk/python/foundation-models/cohere/cohere-cmdR.ipynb +++ b/sdk/python/foundation-models/cohere/cohere-cmdR.ipynb @@ -142,9 +142,9 @@ ], "metadata": { "kernelspec": { - "display_name": "jupyter", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/cohere/cohere-embed.ipynb b/sdk/python/foundation-models/cohere/cohere-embed.ipynb index d90425b3bb..fd227bc434 100644 --- a/sdk/python/foundation-models/cohere/cohere-embed.ipynb +++ b/sdk/python/foundation-models/cohere/cohere-embed.ipynb @@ -159,9 +159,9 @@ ], "metadata": { "kernelspec": { - "display_name": "jupyter", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/cohere/cohere_faiss_langchain_embed.ipynb b/sdk/python/foundation-models/cohere/cohere_faiss_langchain_embed.ipynb index bcb9198f36..720a433c39 100644 --- a/sdk/python/foundation-models/cohere/cohere_faiss_langchain_embed.ipynb +++ b/sdk/python/foundation-models/cohere/cohere_faiss_langchain_embed.ipynb @@ -167,9 +167,9 @@ ], "metadata": { "kernelspec": { - "display_name": "rag-app", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/cohere/command_faiss_langchain.ipynb b/sdk/python/foundation-models/cohere/command_faiss_langchain.ipynb index 6f69bd5fdd..0408412ff2 100644 --- a/sdk/python/foundation-models/cohere/command_faiss_langchain.ipynb +++ b/sdk/python/foundation-models/cohere/command_faiss_langchain.ipynb @@ -257,9 +257,9 @@ ], "metadata": { "kernelspec": { - "display_name": "rag-app", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/cohere/command_tools-langchain.ipynb b/sdk/python/foundation-models/cohere/command_tools-langchain.ipynb index 69b0faf1d7..014fc3750b 100644 --- a/sdk/python/foundation-models/cohere/command_tools-langchain.ipynb +++ b/sdk/python/foundation-models/cohere/command_tools-langchain.ipynb @@ -335,9 +335,9 @@ ], "metadata": { "kernelspec": { - "display_name": "rag-app", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/cohere/langchain.ipynb b/sdk/python/foundation-models/cohere/langchain.ipynb index a2b47adeb2..60db9c3368 100644 --- a/sdk/python/foundation-models/cohere/langchain.ipynb +++ b/sdk/python/foundation-models/cohere/langchain.ipynb @@ -196,9 +196,9 @@ ], "metadata": { "kernelspec": { - "display_name": "jupyter", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/cohere/litellm.ipynb b/sdk/python/foundation-models/cohere/litellm.ipynb index 875b22a7de..7a991856d6 100644 --- a/sdk/python/foundation-models/cohere/litellm.ipynb +++ b/sdk/python/foundation-models/cohere/litellm.ipynb @@ -139,7 +139,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/cohere/openaisdk.ipynb b/sdk/python/foundation-models/cohere/openaisdk.ipynb index 1b5903d61e..23b8af288b 100644 --- a/sdk/python/foundation-models/cohere/openaisdk.ipynb +++ b/sdk/python/foundation-models/cohere/openaisdk.ipynb @@ -154,9 +154,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/cohere/rerank-3-langchain.ipynb b/sdk/python/foundation-models/cohere/rerank-3-langchain.ipynb index e05d500658..ff4086b5bf 100644 --- a/sdk/python/foundation-models/cohere/rerank-3-langchain.ipynb +++ b/sdk/python/foundation-models/cohere/rerank-3-langchain.ipynb @@ -301,8 +301,9 @@ "provenance": [] }, "kernelspec": { - "display_name": "Python 3", - "name": "python3" + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/cohere/rerank-cohere-client.ipynb b/sdk/python/foundation-models/cohere/rerank-cohere-client.ipynb index 73053c5694..564caf40a5 100644 --- a/sdk/python/foundation-models/cohere/rerank-cohere-client.ipynb +++ b/sdk/python/foundation-models/cohere/rerank-cohere-client.ipynb @@ -163,9 +163,9 @@ ], "metadata": { "kernelspec": { - "display_name": "jupyter", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/cohere/rerank-webrequests.ipynb b/sdk/python/foundation-models/cohere/rerank-webrequests.ipynb index 30a27eb2cb..ce2226a06b 100644 --- a/sdk/python/foundation-models/cohere/rerank-webrequests.ipynb +++ b/sdk/python/foundation-models/cohere/rerank-webrequests.ipynb @@ -78,9 +78,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/cohere/webrequests.ipynb b/sdk/python/foundation-models/cohere/webrequests.ipynb index 73f0e7fb2f..db61c0e456 100644 --- a/sdk/python/foundation-models/cohere/webrequests.ipynb +++ b/sdk/python/foundation-models/cohere/webrequests.ipynb @@ -248,7 +248,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/healthcare-ai/cxrreportgen/cxr-deploy.ipynb b/sdk/python/foundation-models/healthcare-ai/cxrreportgen/cxr-deploy.ipynb index 5884cff76d..7a94ce3f91 100644 --- a/sdk/python/foundation-models/healthcare-ai/cxrreportgen/cxr-deploy.ipynb +++ b/sdk/python/foundation-models/healthcare-ai/cxrreportgen/cxr-deploy.ipynb @@ -420,9 +420,9 @@ ], "metadata": { "kernelspec": { - "display_name": "azureml_py310_sdkv2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/healthcare-ai/medimageinsight/adapter-training.ipynb b/sdk/python/foundation-models/healthcare-ai/medimageinsight/adapter-training.ipynb index 050b7b2084..90f28a0e97 100644 --- a/sdk/python/foundation-models/healthcare-ai/medimageinsight/adapter-training.ipynb +++ b/sdk/python/foundation-models/healthcare-ai/medimageinsight/adapter-training.ipynb @@ -2347,9 +2347,9 @@ "name": "python3" }, "kernelspec": { - "display_name": "azureml_py310_sdkv2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/healthcare-ai/medimageinsight/advanced-call-example.ipynb b/sdk/python/foundation-models/healthcare-ai/medimageinsight/advanced-call-example.ipynb index 5412942a6b..4ba00545ca 100644 --- a/sdk/python/foundation-models/healthcare-ai/medimageinsight/advanced-call-example.ipynb +++ b/sdk/python/foundation-models/healthcare-ai/medimageinsight/advanced-call-example.ipynb @@ -524,9 +524,9 @@ ], "metadata": { "kernelspec": { - "display_name": "azureml_py310_sdkv2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/healthcare-ai/medimageinsight/exam-parameter-demo/exam-parameter-detection.ipynb b/sdk/python/foundation-models/healthcare-ai/medimageinsight/exam-parameter-demo/exam-parameter-detection.ipynb index 92b723ef31..ac7de0801a 100644 --- a/sdk/python/foundation-models/healthcare-ai/medimageinsight/exam-parameter-demo/exam-parameter-detection.ipynb +++ b/sdk/python/foundation-models/healthcare-ai/medimageinsight/exam-parameter-demo/exam-parameter-detection.ipynb @@ -8002,9 +8002,9 @@ "name": "python310-sdkv2" }, "kernelspec": { - "display_name": "examparamdetection", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/healthcare-ai/medimageinsight/mi2-deploy.ipynb b/sdk/python/foundation-models/healthcare-ai/medimageinsight/mi2-deploy.ipynb index f05551e939..d8a36c01fc 100644 --- a/sdk/python/foundation-models/healthcare-ai/medimageinsight/mi2-deploy.ipynb +++ b/sdk/python/foundation-models/healthcare-ai/medimageinsight/mi2-deploy.ipynb @@ -339,9 +339,9 @@ ], "metadata": { "kernelspec": { - "display_name": "azureml_py310_sdkv2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/healthcare-ai/medimageinsight/zero-shot-classification.ipynb b/sdk/python/foundation-models/healthcare-ai/medimageinsight/zero-shot-classification.ipynb index f78112cba7..8d77c237e4 100644 --- a/sdk/python/foundation-models/healthcare-ai/medimageinsight/zero-shot-classification.ipynb +++ b/sdk/python/foundation-models/healthcare-ai/medimageinsight/zero-shot-classification.ipynb @@ -551,9 +551,9 @@ "name": "python3" }, "kernelspec": { - "display_name": "azureml_py310_sdkv2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/healthcare-ai/medimageparse/mip-deploy.ipynb b/sdk/python/foundation-models/healthcare-ai/medimageparse/mip-deploy.ipynb index 86cbd34cfa..d8d8385fb1 100644 --- a/sdk/python/foundation-models/healthcare-ai/medimageparse/mip-deploy.ipynb +++ b/sdk/python/foundation-models/healthcare-ai/medimageparse/mip-deploy.ipynb @@ -280,9 +280,9 @@ ], "metadata": { "kernelspec": { - "display_name": "azureml_py310_sdkv2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/healthcare-ai/medimageparse/sematic-segmentation-demo/medimageparse_segmentation_demo.ipynb b/sdk/python/foundation-models/healthcare-ai/medimageparse/sematic-segmentation-demo/medimageparse_segmentation_demo.ipynb index ed84bc791c..8efc55a474 100644 --- a/sdk/python/foundation-models/healthcare-ai/medimageparse/sematic-segmentation-demo/medimageparse_segmentation_demo.ipynb +++ b/sdk/python/foundation-models/healthcare-ai/medimageparse/sematic-segmentation-demo/medimageparse_segmentation_demo.ipynb @@ -739,9 +739,9 @@ "name": "biomedparse" }, "kernelspec": { - "display_name": "azureml_py310_sdkv2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/jais/litellm.ipynb b/sdk/python/foundation-models/jais/litellm.ipynb index 76101db1db..09df48b193 100644 --- a/sdk/python/foundation-models/jais/litellm.ipynb +++ b/sdk/python/foundation-models/jais/litellm.ipynb @@ -139,7 +139,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/jais/openaisdk.ipynb b/sdk/python/foundation-models/jais/openaisdk.ipynb index 4ee0cede92..4eec653143 100644 --- a/sdk/python/foundation-models/jais/openaisdk.ipynb +++ b/sdk/python/foundation-models/jais/openaisdk.ipynb @@ -154,9 +154,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/jais/webrequests.ipynb b/sdk/python/foundation-models/jais/webrequests.ipynb index 30d6406283..5b63300366 100644 --- a/sdk/python/foundation-models/jais/webrequests.ipynb +++ b/sdk/python/foundation-models/jais/webrequests.ipynb @@ -216,7 +216,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/llama2/langchain.ipynb b/sdk/python/foundation-models/llama2/langchain.ipynb index e5f5c1eec2..3503b3c1c2 100644 --- a/sdk/python/foundation-models/llama2/langchain.ipynb +++ b/sdk/python/foundation-models/llama2/langchain.ipynb @@ -196,7 +196,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/llama2/litellm.ipynb b/sdk/python/foundation-models/llama2/litellm.ipynb index 5c12693bfe..1da585f018 100644 --- a/sdk/python/foundation-models/llama2/litellm.ipynb +++ b/sdk/python/foundation-models/llama2/litellm.ipynb @@ -138,7 +138,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/llama2/openaisdk.ipynb b/sdk/python/foundation-models/llama2/openaisdk.ipynb index cf5292498f..b62735cd85 100644 --- a/sdk/python/foundation-models/llama2/openaisdk.ipynb +++ b/sdk/python/foundation-models/llama2/openaisdk.ipynb @@ -146,7 +146,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/llama2/webrequests.ipynb b/sdk/python/foundation-models/llama2/webrequests.ipynb index cc8ac04e12..ea4928dc00 100644 --- a/sdk/python/foundation-models/llama2/webrequests.ipynb +++ b/sdk/python/foundation-models/llama2/webrequests.ipynb @@ -338,9 +338,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/meta-llama-3.1/langchain.ipynb b/sdk/python/foundation-models/meta-llama-3.1/langchain.ipynb index 40ae6eba26..6a88bfa685 100644 --- a/sdk/python/foundation-models/meta-llama-3.1/langchain.ipynb +++ b/sdk/python/foundation-models/meta-llama-3.1/langchain.ipynb @@ -210,9 +210,9 @@ ], "metadata": { "kernelspec": { - "display_name": "jupyter", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/meta-llama-3.1/litellm.ipynb b/sdk/python/foundation-models/meta-llama-3.1/litellm.ipynb index 54179f8291..47132cf5df 100644 --- a/sdk/python/foundation-models/meta-llama-3.1/litellm.ipynb +++ b/sdk/python/foundation-models/meta-llama-3.1/litellm.ipynb @@ -158,9 +158,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/meta-llama-3.1/openaisdk.ipynb b/sdk/python/foundation-models/meta-llama-3.1/openaisdk.ipynb index 84483672d5..3f53b44d76 100644 --- a/sdk/python/foundation-models/meta-llama-3.1/openaisdk.ipynb +++ b/sdk/python/foundation-models/meta-llama-3.1/openaisdk.ipynb @@ -171,9 +171,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/meta-llama-3.1/webrequests.ipynb b/sdk/python/foundation-models/meta-llama-3.1/webrequests.ipynb index 66ea1cb991..587e919a37 100644 --- a/sdk/python/foundation-models/meta-llama-3.1/webrequests.ipynb +++ b/sdk/python/foundation-models/meta-llama-3.1/webrequests.ipynb @@ -299,9 +299,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/meta-llama3/langchain.ipynb b/sdk/python/foundation-models/meta-llama3/langchain.ipynb index a2b6076a8d..7323c1b1df 100644 --- a/sdk/python/foundation-models/meta-llama3/langchain.ipynb +++ b/sdk/python/foundation-models/meta-llama3/langchain.ipynb @@ -195,7 +195,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/meta-llama3/litellm.ipynb b/sdk/python/foundation-models/meta-llama3/litellm.ipynb index 45c07ec69f..473c05c68e 100644 --- a/sdk/python/foundation-models/meta-llama3/litellm.ipynb +++ b/sdk/python/foundation-models/meta-llama3/litellm.ipynb @@ -137,7 +137,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/meta-llama3/openaisdk.ipynb b/sdk/python/foundation-models/meta-llama3/openaisdk.ipynb index bdcb8607e8..a1c2a1e687 100644 --- a/sdk/python/foundation-models/meta-llama3/openaisdk.ipynb +++ b/sdk/python/foundation-models/meta-llama3/openaisdk.ipynb @@ -145,7 +145,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/meta-llama3/webrequests.ipynb b/sdk/python/foundation-models/meta-llama3/webrequests.ipynb index 39b736bbdc..81c3a3c5fd 100644 --- a/sdk/python/foundation-models/meta-llama3/webrequests.ipynb +++ b/sdk/python/foundation-models/meta-llama3/webrequests.ipynb @@ -337,9 +337,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/mistral/langchain.ipynb b/sdk/python/foundation-models/mistral/langchain.ipynb index e37baeebff..c565cfbf6a 100644 --- a/sdk/python/foundation-models/mistral/langchain.ipynb +++ b/sdk/python/foundation-models/mistral/langchain.ipynb @@ -194,7 +194,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/mistral/litellm.ipynb b/sdk/python/foundation-models/mistral/litellm.ipynb index c1a231b29b..800d434d0a 100644 --- a/sdk/python/foundation-models/mistral/litellm.ipynb +++ b/sdk/python/foundation-models/mistral/litellm.ipynb @@ -137,7 +137,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/mistral/mistralai.ipynb b/sdk/python/foundation-models/mistral/mistralai.ipynb index be6f481420..70a69de00b 100644 --- a/sdk/python/foundation-models/mistral/mistralai.ipynb +++ b/sdk/python/foundation-models/mistral/mistralai.ipynb @@ -267,9 +267,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/mistral/openaisdk.ipynb b/sdk/python/foundation-models/mistral/openaisdk.ipynb index 710e2fc2f5..b2d9fa5d0e 100644 --- a/sdk/python/foundation-models/mistral/openaisdk.ipynb +++ b/sdk/python/foundation-models/mistral/openaisdk.ipynb @@ -197,7 +197,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/mistral/webrequests.ipynb b/sdk/python/foundation-models/mistral/webrequests.ipynb index 095bdd5260..a80fc0f28d 100644 --- a/sdk/python/foundation-models/mistral/webrequests.ipynb +++ b/sdk/python/foundation-models/mistral/webrequests.ipynb @@ -238,7 +238,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/nixtla/01_quickstart_forecast.ipynb b/sdk/python/foundation-models/nixtla/01_quickstart_forecast.ipynb index 711531f5c2..ae40787498 100644 --- a/sdk/python/foundation-models/nixtla/01_quickstart_forecast.ipynb +++ b/sdk/python/foundation-models/nixtla/01_quickstart_forecast.ipynb @@ -102,9 +102,9 @@ ], "metadata": { "kernelspec": { - "display_name": "python3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/nixtla/02_finetuning.ipynb b/sdk/python/foundation-models/nixtla/02_finetuning.ipynb index 264a54591e..cdba59a8e5 100644 --- a/sdk/python/foundation-models/nixtla/02_finetuning.ipynb +++ b/sdk/python/foundation-models/nixtla/02_finetuning.ipynb @@ -171,9 +171,9 @@ ], "metadata": { "kernelspec": { - "display_name": "python3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/nixtla/03_anomaly_detection.ipynb b/sdk/python/foundation-models/nixtla/03_anomaly_detection.ipynb index 24c39c0062..1ecb566ae6 100644 --- a/sdk/python/foundation-models/nixtla/03_anomaly_detection.ipynb +++ b/sdk/python/foundation-models/nixtla/03_anomaly_detection.ipynb @@ -243,9 +243,9 @@ ], "metadata": { "kernelspec": { - "display_name": "python3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/nixtla/04_exogenous_variables.ipynb b/sdk/python/foundation-models/nixtla/04_exogenous_variables.ipynb index 4573cebeac..efa109f63c 100644 --- a/sdk/python/foundation-models/nixtla/04_exogenous_variables.ipynb +++ b/sdk/python/foundation-models/nixtla/04_exogenous_variables.ipynb @@ -434,9 +434,9 @@ ], "metadata": { "kernelspec": { - "display_name": "python3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/nixtla/05_demand_forecasting.ipynb b/sdk/python/foundation-models/nixtla/05_demand_forecasting.ipynb index fad4738c6a..47939c54d3 100644 --- a/sdk/python/foundation-models/nixtla/05_demand_forecasting.ipynb +++ b/sdk/python/foundation-models/nixtla/05_demand_forecasting.ipynb @@ -595,9 +595,9 @@ ], "metadata": { "kernelspec": { - "display_name": "python3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/nvidia-nemo/deploy_infer.ipynb b/sdk/python/foundation-models/nvidia-nemo/deploy_infer.ipynb index efe9d75219..6733e66454 100644 --- a/sdk/python/foundation-models/nvidia-nemo/deploy_infer.ipynb +++ b/sdk/python/foundation-models/nvidia-nemo/deploy_infer.ipynb @@ -544,9 +544,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/nvidia-nemo/evaluate/nemo_ptuning_summarization_with_text_gen.ipynb b/sdk/python/foundation-models/nvidia-nemo/evaluate/nemo_ptuning_summarization_with_text_gen.ipynb index 8a2b443d4e..40321be4da 100644 --- a/sdk/python/foundation-models/nvidia-nemo/evaluate/nemo_ptuning_summarization_with_text_gen.ipynb +++ b/sdk/python/foundation-models/nvidia-nemo/evaluate/nemo_ptuning_summarization_with_text_gen.ipynb @@ -554,9 +554,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-classification-evaluation.ipynb b/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-classification-evaluation.ipynb index 7d1f229753..7d4a8e4383 100644 --- a/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-classification-evaluation.ipynb +++ b/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-classification-evaluation.ipynb @@ -1,408 +1,408 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1699605126817 - } - }, - "outputs": [], - "source": [ - "# Prequisites\n", - "\n", - "# %pip install --upgrade azure-ai-ml\n", - "# %pip install --upgrade azure-identity\n", - "# %pip install --upgrade datasets==2.9.0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700023808094 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from azure.ai.ml import MLClient\n", - "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", - "from azure.ai.ml.entities import AmlCompute\n", - "import time\n", - "\n", - "try:\n", - " credential = DefaultAzureCredential()\n", - " credential.get_token(\"https://management.azure.com/.default\")\n", - "except Exception as ex:\n", - " credential = InteractiveBrowserCredential()\n", - "\n", - "workspace_ml_client = None\n", - "try:\n", - " workspace_ml_client = MLClient.from_config(credential)\n", - " subscription_id = workspace_ml_client.subscription_id\n", - " workspace = workspace_ml_client.workspace_name\n", - " resource_group = workspace_ml_client.resource_group_name\n", - "except Exception as ex:\n", - " print(ex)\n", - " # Enter details of your AML workspace\n", - " subscription_id = \"\"\n", - " resource_group = \"\"\n", - " workspace = \"\"\n", - " workspace_ml_client = MLClient(\n", - " credential, subscription_id, resource_group, workspace\n", - " )\n", - "\n", - "# replace with the registry name\n", - "nemo_registry = \"nvidia-ai\"\n", - "\n", - "# registry where models are present\n", - "model_registry = \"azureml-preview-test1\"\n", - "\n", - "nemo_registry_ml_client = MLClient(\n", - " credential, subscription_id, resource_group, registry_name=nemo_registry\n", - ")\n", - "model_registry_ml_client = MLClient(\n", - " credential, subscription_id, resource_group, registry_name=model_registry\n", - ")\n", - "nemo_registry_ml_client" - ] + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1699605126817 + } + }, + "outputs": [], + "source": [ + "# Prequisites\n", + "\n", + "# %pip install --upgrade azure-ai-ml\n", + "# %pip install --upgrade azure-identity\n", + "# %pip install --upgrade datasets==2.9.0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700023808094 }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Creating Computes" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700023872400 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# If you already have a gpu cluster, mention it here. Else will create a new one with the name 'ghyadav-westus-a100'\n", - "# model will only run on an a100 instance\n", - "\n", - "compute_cluster = \"nemo-westus\"\n", - "try:\n", - " compute = workspace_ml_client.compute.get(compute_cluster)\n", - " print(f\"GPU compute '{compute_cluster}' found.\")\n", - "except Exception as ex:\n", - " print(f\"GPU compute '{compute_cluster}' not found. Creating new one.\")\n", - " compute = AmlCompute(\n", - " name=compute_cluster,\n", - " size=\"STANDARD_ND96AMSR_A100_V4\",\n", - " max_instances=2, # For multi node training set this to an integer value more than 1\n", - " )\n", - " workspace_ml_client.compute.begin_create_or_update(compute).wait()\n", - "\n", - "# generating a unique timestamp that can be used for names and versions that need to be unique\n", - "timestamp = str(int(time.time()))\n", - "\n", - "\n", - "# This is the number of GPUs in a single node of the selected 'vm_size' compute.\n", - "# Setting this to less than the number of GPUs will result in underutilized GPUs, taking longer to train.\n", - "# Setting this to more than the number of GPUs will result in an error.\n", - "gpus_per_node = 1 # default value\n", - "gpu_count_found = False\n", - "ws_computes = workspace_ml_client.compute.list_sizes()\n", - "for ws_compute in ws_computes:\n", - " if ws_compute.name.lower() == compute.size.lower():\n", - " gpus_per_node = ws_compute.gpus\n", - " print(f\"Number of GPUs in compute {ws_compute.name} are {ws_compute.gpus}\")\n", - "# if gpu_count_found not found, then print an error\n", - "if gpus_per_node > 0:\n", - " gpu_count_found = True\n", - "else:\n", - " gpu_count_found = False\n", - " print(f\"No GPUs found in compute. Number of GPUs in compute {compute.size} 0.\")" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "from azure.ai.ml import MLClient\n", + "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", + "from azure.ai.ml.entities import AmlCompute\n", + "import time\n", + "\n", + "try:\n", + " credential = DefaultAzureCredential()\n", + " credential.get_token(\"https://management.azure.com/.default\")\n", + "except Exception as ex:\n", + " credential = InteractiveBrowserCredential()\n", + "\n", + "workspace_ml_client = None\n", + "try:\n", + " workspace_ml_client = MLClient.from_config(credential)\n", + " subscription_id = workspace_ml_client.subscription_id\n", + " workspace = workspace_ml_client.workspace_name\n", + " resource_group = workspace_ml_client.resource_group_name\n", + "except Exception as ex:\n", + " print(ex)\n", + " # Enter details of your AML workspace\n", + " subscription_id = \"\"\n", + " resource_group = \"\"\n", + " workspace = \"\"\n", + " workspace_ml_client = MLClient(\n", + " credential, subscription_id, resource_group, workspace\n", + " )\n", + "\n", + "# replace with the registry name\n", + "nemo_registry = \"nvidia-ai\"\n", + "\n", + "# registry where models are present\n", + "model_registry = \"azureml-preview-test1\"\n", + "\n", + "nemo_registry_ml_client = MLClient(\n", + " credential, subscription_id, resource_group, registry_name=nemo_registry\n", + ")\n", + "model_registry_ml_client = MLClient(\n", + " credential, subscription_id, resource_group, registry_name=model_registry\n", + ")\n", + "nemo_registry_ml_client" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Creating Computes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700023872400 }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Input Data for Evaluation" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700023876381 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "evaluation_dataset = \"./data/classification/\"" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# If you already have a gpu cluster, mention it here. Else will create a new one with the name 'ghyadav-westus-a100'\n", + "# model will only run on an a100 instance\n", + "\n", + "compute_cluster = \"nemo-westus\"\n", + "try:\n", + " compute = workspace_ml_client.compute.get(compute_cluster)\n", + " print(f\"GPU compute '{compute_cluster}' found.\")\n", + "except Exception as ex:\n", + " print(f\"GPU compute '{compute_cluster}' not found. Creating new one.\")\n", + " compute = AmlCompute(\n", + " name=compute_cluster,\n", + " size=\"STANDARD_ND96AMSR_A100_V4\",\n", + " max_instances=2, # For multi node training set this to an integer value more than 1\n", + " )\n", + " workspace_ml_client.compute.begin_create_or_update(compute).wait()\n", + "\n", + "# generating a unique timestamp that can be used for names and versions that need to be unique\n", + "timestamp = str(int(time.time()))\n", + "\n", + "\n", + "# This is the number of GPUs in a single node of the selected 'vm_size' compute.\n", + "# Setting this to less than the number of GPUs will result in underutilized GPUs, taking longer to train.\n", + "# Setting this to more than the number of GPUs will result in an error.\n", + "gpus_per_node = 1 # default value\n", + "gpu_count_found = False\n", + "ws_computes = workspace_ml_client.compute.list_sizes()\n", + "for ws_compute in ws_computes:\n", + " if ws_compute.name.lower() == compute.size.lower():\n", + " gpus_per_node = ws_compute.gpus\n", + " print(f\"Number of GPUs in compute {ws_compute.name} are {ws_compute.gpus}\")\n", + "# if gpu_count_found not found, then print an error\n", + "if gpus_per_node > 0:\n", + " gpu_count_found = True\n", + "else:\n", + " gpu_count_found = False\n", + " print(f\"No GPUs found in compute. Number of GPUs in compute {compute.size} 0.\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Input Data for Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700023876381 }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Loading model from Registry" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700023878996 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "model_name = \"Nemotron-3-8B-Base-4k\"\n", - "model_version = \"latest\"\n", - "\n", - "nemo_model_object = nemo_registry_ml_client.models.get(model_name, label=\"latest\")" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "evaluation_dataset = \"./data/classification/\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Loading model from Registry" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700023878996 }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# Prompt" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700023881068 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "prompt_path = \"./prompts/prompt-classification.txt\"" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "model_name = \"Nemotron-3-8B-Base-4k\"\n", + "model_version = \"latest\"\n", + "\n", + "nemo_model_object = nemo_registry_ml_client.models.get(model_name, label=\"latest\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "# Prompt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700023881068 }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Submitting Evaluation Pipeline" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700023931981 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from azure.ai.ml.dsl import pipeline\n", - "from azure.ai.ml import Input\n", - "from azure.ai.ml.constants import AssetTypes\n", - "\n", - "# fetch the pipeline component\n", - "pipeline_component_func = nemo_registry_ml_client.components.get(\n", - " name=\"nemo_text_classification_evaluation\", label=\"latest\"\n", - ")\n", - "\n", - "\n", - "# define the pipeline job\n", - "@pipeline()\n", - "def evaluation_pipeline(mlflow_model):\n", - " evaluation_job = pipeline_component_func(\n", - " dataset_path=Input(type=AssetTypes.URI_FOLDER, path=\"./data/classification/\"),\n", - " model_path=Input(type=AssetTypes.TRITON_MODEL, path=f\"{nemo_model_object.id}\"),\n", - " prompt_schema=Input(type=AssetTypes.URI_FILE, path=prompt_path),\n", - " )\n", - " return {\"evaluation_result\": evaluation_job.outputs.evaluation_result}" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "prompt_path = \"./prompts/prompt-classification.txt\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Submitting Evaluation Pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700023931981 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1699964441042 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "experiment_name = \"nemo-classification-eval-pipeline\"\n", - "pipeline_jobs = []\n", - "\n", - "pipeline_object = evaluation_pipeline()\n", - "\n", - "# don't reuse cached results from previous jobs\n", - "pipeline_object.settings.force_rerun = True\n", - "pipeline_object.settings.default_compute = compute_cluster\n", - "pipeline_job = workspace_ml_client.jobs.create_or_update(\n", - " pipeline_object, experiment_name=experiment_name\n", - ")\n", - "# add model['name'] and pipeline_job.name as key value pairs to a dictionary\n", - "pipeline_jobs.append({\"model_name\": model_name, \"job_name\": pipeline_job.name})\n", - "# wait for the pipeline job to complete\n", - "workspace_ml_client.jobs.stream(pipeline_job.name)" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [] + "nteract": { + "transient": { + "deleting": false + } } - ], - "metadata": { - "kernel_info": { - "name": "ghyadav1" - }, - "kernelspec": { - "display_name": "ghyadav1", - "language": "python", - "name": "ghyadav1" + }, + "outputs": [], + "source": [ + "from azure.ai.ml.dsl import pipeline\n", + "from azure.ai.ml import Input\n", + "from azure.ai.ml.constants import AssetTypes\n", + "\n", + "# fetch the pipeline component\n", + "pipeline_component_func = nemo_registry_ml_client.components.get(\n", + " name=\"nemo_text_classification_evaluation\", label=\"latest\"\n", + ")\n", + "\n", + "\n", + "# define the pipeline job\n", + "@pipeline()\n", + "def evaluation_pipeline(mlflow_model):\n", + " evaluation_job = pipeline_component_func(\n", + " dataset_path=Input(type=AssetTypes.URI_FOLDER, path=\"./data/classification/\"),\n", + " model_path=Input(type=AssetTypes.TRITON_MODEL, path=f\"{nemo_model_object.id}\"),\n", + " prompt_schema=Input(type=AssetTypes.URI_FILE, path=prompt_path),\n", + " )\n", + " return {\"evaluation_result\": evaluation_job.outputs.evaluation_result}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1699964441042 }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.18" + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - "microsoft": { - "host": { - "AzureML": { - "notebookHasBeenCompleted": true - } - }, - "ms_spell_check": { - "ms_spell_check_language": "en" - } + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "experiment_name = \"nemo-classification-eval-pipeline\"\n", + "pipeline_jobs = []\n", + "\n", + "pipeline_object = evaluation_pipeline()\n", + "\n", + "# don't reuse cached results from previous jobs\n", + "pipeline_object.settings.force_rerun = True\n", + "pipeline_object.settings.default_compute = compute_cluster\n", + "pipeline_job = workspace_ml_client.jobs.create_or_update(\n", + " pipeline_object, experiment_name=experiment_name\n", + ")\n", + "# add model['name'] and pipeline_job.name as key value pairs to a dictionary\n", + "pipeline_jobs.append({\"model_name\": model_name, \"job_name\": pipeline_job.name})\n", + "# wait for the pipeline job to complete\n", + "workspace_ml_client.jobs.stream(pipeline_job.name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, "nteract": { - "version": "nteract-front-end@1.0.0" + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernel_info": { + "name": "ghyadav1" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + }, + "microsoft": { + "host": { + "AzureML": { + "notebookHasBeenCompleted": true } + }, + "ms_spell_check": { + "ms_spell_check_language": "en" + } }, - "nbformat": 4, - "nbformat_minor": 2 + "nteract": { + "version": "nteract-front-end@1.0.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-gen-evaluation.ipynb b/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-gen-evaluation.ipynb index 28c84bfbb6..8561288970 100644 --- a/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-gen-evaluation.ipynb +++ b/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-gen-evaluation.ipynb @@ -1,486 +1,486 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700016896350 - } - }, - "outputs": [], - "source": [ - "# Prequisites\n", - "\n", - "%pip install --upgrade azure-ai-ml\n", - "%pip install --upgrade azure-identity\n", - "%pip install --upgrade datasets==2.9.0\n", - "%pip install py7zr" - ] + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700016896350 + } + }, + "outputs": [], + "source": [ + "# Prequisites\n", + "\n", + "%pip install --upgrade azure-ai-ml\n", + "%pip install --upgrade azure-identity\n", + "%pip install --upgrade datasets==2.9.0\n", + "%pip install py7zr" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700016814248 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700016814248 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from azure.ai.ml import MLClient\n", - "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", - "from azure.ai.ml.entities import AmlCompute\n", - "import time\n", - "\n", - "try:\n", - " credential = DefaultAzureCredential()\n", - " credential.get_token(\"https://management.azure.com/.default\")\n", - "except Exception as ex:\n", - " credential = InteractiveBrowserCredential()\n", - "\n", - "workspace_ml_client = None\n", - "try:\n", - " workspace_ml_client = MLClient.from_config(credential)\n", - " subscription_id = workspace_ml_client.subscription_id\n", - " workspace = workspace_ml_client.workspace_name\n", - " resource_group = workspace_ml_client.resource_group_name\n", - "except Exception as ex:\n", - " print(ex)\n", - " # Enter details of your AML workspace\n", - " subscription_id = \"\"\n", - " resource_group = \"\"\n", - " workspace = \"\"\n", - " workspace_ml_client = MLClient(\n", - " credential, subscription_id, resource_group, workspace\n", - " )\n", - "\n", - "# replace with the registry name\n", - "nemo_registry = \"nvidia-ai\"\n", - "\n", - "nemo_registry_ml_client = MLClient(\n", - " credential, subscription_id, resource_group, registry_name=nemo_registry\n", - ")\n", - "\n", - "nemo_registry_ml_client" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "from azure.ai.ml import MLClient\n", + "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", + "from azure.ai.ml.entities import AmlCompute\n", + "import time\n", + "\n", + "try:\n", + " credential = DefaultAzureCredential()\n", + " credential.get_token(\"https://management.azure.com/.default\")\n", + "except Exception as ex:\n", + " credential = InteractiveBrowserCredential()\n", + "\n", + "workspace_ml_client = None\n", + "try:\n", + " workspace_ml_client = MLClient.from_config(credential)\n", + " subscription_id = workspace_ml_client.subscription_id\n", + " workspace = workspace_ml_client.workspace_name\n", + " resource_group = workspace_ml_client.resource_group_name\n", + "except Exception as ex:\n", + " print(ex)\n", + " # Enter details of your AML workspace\n", + " subscription_id = \"\"\n", + " resource_group = \"\"\n", + " workspace = \"\"\n", + " workspace_ml_client = MLClient(\n", + " credential, subscription_id, resource_group, workspace\n", + " )\n", + "\n", + "# replace with the registry name\n", + "nemo_registry = \"nvidia-ai\"\n", + "\n", + "nemo_registry_ml_client = MLClient(\n", + " credential, subscription_id, resource_group, registry_name=nemo_registry\n", + ")\n", + "\n", + "nemo_registry_ml_client" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Creating Computes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700018891994 }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Creating Computes" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700018891994 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# If you already have a gpu cluster, mention it here. Else will create a new one with the name 'ghyadav-westus-a100'\n", - "# model will only run on an a100 instance\n", - "\n", - "compute_cluster = \"do-not-delete-nvidia-testing\"\n", - "\n", - "try:\n", - " compute = workspace_ml_client.compute.get(compute_cluster)\n", - " print(f\"GPU compute '{compute_cluster}' found.\")\n", - "except Exception as ex:\n", - " print(f\"GPU compute '{compute_cluster}' not found. Creating new one.\")\n", - " compute = AmlCompute(\n", - " name=compute_cluster,\n", - " size=\"STANDARD_ND96AMSR_A100_V4\",\n", - " max_instances=2, # For multi node training set this to an integer value more than 1\n", - " )\n", - " workspace_ml_client.compute.begin_create_or_update(compute).wait()\n", - "\n", - "# generating a unique timestamp that can be used for names and versions that need to be unique\n", - "timestamp = str(int(time.time()))\n", - "\n", - "\n", - "# This is the number of GPUs in a single node of the selected 'vm_size' compute.\n", - "# Setting this to less than the number of GPUs will result in underutilized GPUs, taking longer to train.\n", - "# Setting this to more than the number of GPUs will result in an error.\n", - "\n", - "gpus_per_node = 1 # default value\n", - "gpu_count_found = False\n", - "ws_computes = workspace_ml_client.compute.list_sizes()\n", - "for ws_compute in ws_computes:\n", - " if ws_compute.name.lower() == compute.size.lower():\n", - " gpus_per_node = ws_compute.gpus\n", - " print(f\"Number of GPUs in compute {ws_compute.name} are {ws_compute.gpus}\")\n", - "# if gpu_count_found not found, then print an error\n", - "if gpus_per_node > 0:\n", - " gpu_count_found = True\n", - "else:\n", - " gpu_count_found = False\n", - " print(f\"No GPUs found in compute. Number of GPUs in compute {compute.size} 0.\")" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# If you already have a gpu cluster, mention it here. Else will create a new one with the name 'ghyadav-westus-a100'\n", + "# model will only run on an a100 instance\n", + "\n", + "compute_cluster = \"do-not-delete-nvidia-testing\"\n", + "\n", + "try:\n", + " compute = workspace_ml_client.compute.get(compute_cluster)\n", + " print(f\"GPU compute '{compute_cluster}' found.\")\n", + "except Exception as ex:\n", + " print(f\"GPU compute '{compute_cluster}' not found. Creating new one.\")\n", + " compute = AmlCompute(\n", + " name=compute_cluster,\n", + " size=\"STANDARD_ND96AMSR_A100_V4\",\n", + " max_instances=2, # For multi node training set this to an integer value more than 1\n", + " )\n", + " workspace_ml_client.compute.begin_create_or_update(compute).wait()\n", + "\n", + "# generating a unique timestamp that can be used for names and versions that need to be unique\n", + "timestamp = str(int(time.time()))\n", + "\n", + "\n", + "# This is the number of GPUs in a single node of the selected 'vm_size' compute.\n", + "# Setting this to less than the number of GPUs will result in underutilized GPUs, taking longer to train.\n", + "# Setting this to more than the number of GPUs will result in an error.\n", + "\n", + "gpus_per_node = 1 # default value\n", + "gpu_count_found = False\n", + "ws_computes = workspace_ml_client.compute.list_sizes()\n", + "for ws_compute in ws_computes:\n", + " if ws_compute.name.lower() == compute.size.lower():\n", + " gpus_per_node = ws_compute.gpus\n", + " print(f\"Number of GPUs in compute {ws_compute.name} are {ws_compute.gpus}\")\n", + "# if gpu_count_found not found, then print an error\n", + "if gpus_per_node > 0:\n", + " gpu_count_found = True\n", + "else:\n", + " gpu_count_found = False\n", + " print(f\"No GPUs found in compute. Number of GPUs in compute {compute.size} 0.\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Input Data for Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700021765687 }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Input Data for Evaluation" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700021765687 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# download the dataset using the helper script. This needs datasets library: https://pypi.org/project/datasets/\n", - "import os\n", - "from datasets import load_dataset, get_dataset_split_names\n", - "\n", - "dataset_dir = \"samsum-dataset\"\n", - "dataset_name = \"samsum\"\n", - "# create the download directory if it does not exist\n", - "if not os.path.exists(dataset_dir):\n", - " os.makedirs(dataset_dir)\n", - "\n", - "\n", - "# import hugging face datasets library\n", - "\n", - "split = \"test\"\n", - "dataset = load_dataset(dataset_name, split=split)\n", - "# save the split of the dataset to the download directory as json lines file\n", - "dataset.to_json(os.path.join(dataset_dir, f\"{split}.jsonl\"))\n", - "# print dataset features" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# download the dataset using the helper script. This needs datasets library: https://pypi.org/project/datasets/\n", + "import os\n", + "from datasets import load_dataset, get_dataset_split_names\n", + "\n", + "dataset_dir = \"samsum-dataset\"\n", + "dataset_name = \"samsum\"\n", + "# create the download directory if it does not exist\n", + "if not os.path.exists(dataset_dir):\n", + " os.makedirs(dataset_dir)\n", + "\n", + "\n", + "# import hugging face datasets library\n", + "\n", + "split = \"test\"\n", + "dataset = load_dataset(dataset_name, split=split)\n", + "# save the split of the dataset to the download directory as json lines file\n", + "dataset.to_json(os.path.join(dataset_dir, f\"{split}.jsonl\"))\n", + "# print dataset features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700021773282 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700021773282 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "evaluation_dataset = \"./samsum-dataset/test.jsonl\"\n", - "pd.set_option(\n", - " \"display.max_colwidth\", 0\n", - ") # set the max column width to 0 to display the full text\n", - "df = pd.read_json(\"./samsum-dataset/test.jsonl\", lines=True)\n", - "df = df[[\"dialogue\"]]\n", - "df" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700021784720 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "frac = 0.01\n", - "evaluation_dataset_frac = \"./samsum-dataset/test/test_frac.jsonl\"\n", - "os.makedirs(\"./samsum-dataset/test\", exist_ok=True)\n", - "df.sample(frac=frac).to_json(evaluation_dataset_frac, orient=\"records\", lines=True)" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "evaluation_dataset = \"./samsum-dataset/test.jsonl\"\n", + "pd.set_option(\n", + " \"display.max_colwidth\", 0\n", + ") # set the max column width to 0 to display the full text\n", + "df = pd.read_json(\"./samsum-dataset/test.jsonl\", lines=True)\n", + "df = df[[\"dialogue\"]]\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700021784720 }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Loading model from Registry" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700020630465 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "model_name = \"Nemotron-3-8B-Base-4k\"\n", - "model_version = \"latest\"\n", - "\n", - "nemo_model_object = nemo_registry_ml_client.models.get(model_name, label=\"latest\")" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "frac = 0.01\n", + "evaluation_dataset_frac = \"./samsum-dataset/test/test_frac.jsonl\"\n", + "os.makedirs(\"./samsum-dataset/test\", exist_ok=True)\n", + "df.sample(frac=frac).to_json(evaluation_dataset_frac, orient=\"records\", lines=True)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Loading model from Registry" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700020630465 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700021668795 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "nemo_model_object" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# Prompt" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "model_name = \"Nemotron-3-8B-Base-4k\"\n", + "model_version = \"latest\"\n", + "\n", + "nemo_model_object = nemo_registry_ml_client.models.get(model_name, label=\"latest\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700021668795 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700018241647 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "prompt = \"Summarize this dialog:\\n{prompt}\\n---\\nSummary:\\n\"\n", - "prompt_path = \"prompt.txt\"\n", - "with open(prompt_path, \"w\") as f:\n", - " f.write(prompt)" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Submitting Evaluation Pipeline" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "nemo_model_object" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "# Prompt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700018241647 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700022099970 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from azure.ai.ml.dsl import pipeline\n", - "from azure.ai.ml import Input\n", - "from azure.ai.ml.constants import AssetTypes\n", - "\n", - "# fetch the pipeline component\n", - "pipeline_component_func = nemo_registry_ml_client.components.get(\n", - " name=\"nemo_text_generation_evaluation\", label=\"latest\"\n", - ")\n", - "\n", - "\n", - "# define the pipeline job\n", - "@pipeline()\n", - "def evaluation_pipeline(mlflow_model):\n", - " evaluation_job = pipeline_component_func(\n", - " dataset_path=Input(type=AssetTypes.URI_FOLDER, path=\"./samsum-dataset/test/\"),\n", - " model_path=Input(type=AssetTypes.TRITON_MODEL, path=f\"{nemo_model_object.id}\"),\n", - " prompt_schema=Input(type=AssetTypes.URI_FILE, path=prompt_path),\n", - " )\n", - " return {\"evaluation_result\": evaluation_job.outputs.evaluation_result}" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700021439682 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "experiment_name = \"nemo-text-gen-eval-pipeline\"\n", - "pipeline_jobs = []\n", - "\n", - "pipeline_object = evaluation_pipeline()\n", - "\n", - "# don't reuse cached results from previous jobs\n", - "pipeline_object.settings.force_rerun = True\n", - "pipeline_object.settings.default_compute = compute_cluster\n", - "pipeline_job = workspace_ml_client.jobs.create_or_update(\n", - " pipeline_object, experiment_name=experiment_name\n", - ")\n", - "# add model['name'] and pipeline_job.name as key value pairs to a dictionary\n", - "pipeline_jobs.append({\"model_name\": model_name, \"job_name\": pipeline_job.name})\n", - "# wait for the pipeline job to complete\n", - "workspace_ml_client.jobs.stream(pipeline_job.name)" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "prompt = \"Summarize this dialog:\\n{prompt}\\n---\\nSummary:\\n\"\n", + "prompt_path = \"prompt.txt\"\n", + "with open(prompt_path, \"w\") as f:\n", + " f.write(prompt)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } } - ], - "metadata": { - "kernel_info": { - "name": "python38-azureml" + }, + "source": [ + "## Submitting Evaluation Pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700022099970 }, - "kernelspec": { - "display_name": "Python 3.8 - AzureML", - "language": "python", - "name": "python38-azureml" + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "from azure.ai.ml.dsl import pipeline\n", + "from azure.ai.ml import Input\n", + "from azure.ai.ml.constants import AssetTypes\n", + "\n", + "# fetch the pipeline component\n", + "pipeline_component_func = nemo_registry_ml_client.components.get(\n", + " name=\"nemo_text_generation_evaluation\", label=\"latest\"\n", + ")\n", + "\n", + "\n", + "# define the pipeline job\n", + "@pipeline()\n", + "def evaluation_pipeline(mlflow_model):\n", + " evaluation_job = pipeline_component_func(\n", + " dataset_path=Input(type=AssetTypes.URI_FOLDER, path=\"./samsum-dataset/test/\"),\n", + " model_path=Input(type=AssetTypes.TRITON_MODEL, path=f\"{nemo_model_object.id}\"),\n", + " prompt_schema=Input(type=AssetTypes.URI_FILE, path=prompt_path),\n", + " )\n", + " return {\"evaluation_result\": evaluation_job.outputs.evaluation_result}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700021439682 }, - "microsoft": { - "host": { - "AzureML": { - "notebookHasBeenCompleted": true - } - }, - "ms_spell_check": { - "ms_spell_check_language": "en" - } + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, "nteract": { - "version": "nteract-front-end@1.0.0" + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "experiment_name = \"nemo-text-gen-eval-pipeline\"\n", + "pipeline_jobs = []\n", + "\n", + "pipeline_object = evaluation_pipeline()\n", + "\n", + "# don't reuse cached results from previous jobs\n", + "pipeline_object.settings.force_rerun = True\n", + "pipeline_object.settings.default_compute = compute_cluster\n", + "pipeline_job = workspace_ml_client.jobs.create_or_update(\n", + " pipeline_object, experiment_name=experiment_name\n", + ")\n", + "# add model['name'] and pipeline_job.name as key value pairs to a dictionary\n", + "pipeline_jobs.append({\"model_name\": model_name, \"job_name\": pipeline_job.name})\n", + "# wait for the pipeline job to complete\n", + "workspace_ml_client.jobs.stream(pipeline_job.name)" + ] + } + ], + "metadata": { + "kernel_info": { + "name": "python38-azureml" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + }, + "microsoft": { + "host": { + "AzureML": { + "notebookHasBeenCompleted": true } + }, + "ms_spell_check": { + "ms_spell_check_language": "en" + } }, - "nbformat": 4, - "nbformat_minor": 2 + "nteract": { + "version": "nteract-front-end@1.0.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-qna-evaluation.ipynb b/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-qna-evaluation.ipynb index 901467ad4c..e778eb147a 100644 --- a/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-qna-evaluation.ipynb +++ b/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-qna-evaluation.ipynb @@ -1,546 +1,546 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700045134531 - } - }, - "outputs": [], - "source": [ - "# Prequisites\n", - "\n", - "# %pip install --upgrade azure-ai-ml\n", - "# %pip install --upgrade azure-identity\n", - "# %pip install --upgrade datasets==2.9.0\n", - "# %pip install py7zr" - ] + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700045134531 + } + }, + "outputs": [], + "source": [ + "# Prequisites\n", + "\n", + "# %pip install --upgrade azure-ai-ml\n", + "# %pip install --upgrade azure-identity\n", + "# %pip install --upgrade datasets==2.9.0\n", + "# %pip install py7zr" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700049432488 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700049432488 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from azure.ai.ml import MLClient\n", - "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", - "from azure.ai.ml.entities import AmlCompute\n", - "import time\n", - "\n", - "try:\n", - " credential = DefaultAzureCredential()\n", - " credential.get_token(\"https://management.azure.com/.default\")\n", - "except Exception as ex:\n", - " credential = InteractiveBrowserCredential()\n", - "\n", - "workspace_ml_client = None\n", - "try:\n", - " workspace_ml_client = MLClient.from_config(credential)\n", - " subscription_id = workspace_ml_client.subscription_id\n", - " workspace = workspace_ml_client.workspace_name\n", - " resource_group = workspace_ml_client.resource_group_name\n", - "except Exception as ex:\n", - " print(ex)\n", - " # Enter details of your AML workspace\n", - " subscription_id = \"\"\n", - " resource_group = \"\"\n", - " workspace = \"\"\n", - " workspace_ml_client = MLClient(\n", - " credential, subscription_id, resource_group, workspace\n", - " )\n", - "\n", - "# replace with the registry name\n", - "nemo_registry = \"nvidia-ai\" # \"azureml\"\n", - "\n", - "# registry where models are present\n", - "model_registry = \"azureml-preview-test1\"\n", - "\n", - "nemo_registry_ml_client = MLClient(\n", - " credential, subscription_id, resource_group, registry_name=nemo_registry\n", - ")\n", - "model_registry_ml_client = MLClient(\n", - " credential, subscription_id, resource_group, registry_name=model_registry\n", - ")\n", - "nemo_registry_ml_client" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Creating Computes" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "from azure.ai.ml import MLClient\n", + "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", + "from azure.ai.ml.entities import AmlCompute\n", + "import time\n", + "\n", + "try:\n", + " credential = DefaultAzureCredential()\n", + " credential.get_token(\"https://management.azure.com/.default\")\n", + "except Exception as ex:\n", + " credential = InteractiveBrowserCredential()\n", + "\n", + "workspace_ml_client = None\n", + "try:\n", + " workspace_ml_client = MLClient.from_config(credential)\n", + " subscription_id = workspace_ml_client.subscription_id\n", + " workspace = workspace_ml_client.workspace_name\n", + " resource_group = workspace_ml_client.resource_group_name\n", + "except Exception as ex:\n", + " print(ex)\n", + " # Enter details of your AML workspace\n", + " subscription_id = \"\"\n", + " resource_group = \"\"\n", + " workspace = \"\"\n", + " workspace_ml_client = MLClient(\n", + " credential, subscription_id, resource_group, workspace\n", + " )\n", + "\n", + "# replace with the registry name\n", + "nemo_registry = \"nvidia-ai\" # \"azureml\"\n", + "\n", + "# registry where models are present\n", + "model_registry = \"azureml-preview-test1\"\n", + "\n", + "nemo_registry_ml_client = MLClient(\n", + " credential, subscription_id, resource_group, registry_name=nemo_registry\n", + ")\n", + "model_registry_ml_client = MLClient(\n", + " credential, subscription_id, resource_group, registry_name=model_registry\n", + ")\n", + "nemo_registry_ml_client" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Creating Computes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700049435366 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700049435366 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# If you already have a gpu cluster, mention it here. Else will create a new one with the name 'ghyadav-westus-a100'\n", - "# model will only run on an a100 instance\n", - "\n", - "compute_cluster = \"nemo-westus\"\n", - "try:\n", - " compute = workspace_ml_client.compute.get(compute_cluster)\n", - " print(f\"GPU compute '{compute_cluster}' found.\")\n", - "except Exception as ex:\n", - " print(f\"GPU compute '{compute_cluster}' not found. Creating new one.\")\n", - " compute = AmlCompute(\n", - " name=compute_cluster,\n", - " size=\"STANDARD_ND96AMSR_A100_V4\",\n", - " max_instances=2, # For multi node training set this to an integer value more than 1\n", - " )\n", - " workspace_ml_client.compute.begin_create_or_update(compute).wait()\n", - "\n", - "# generating a unique timestamp that can be used for names and versions that need to be unique\n", - "timestamp = str(int(time.time()))\n", - "\n", - "\n", - "# This is the number of GPUs in a single node of the selected 'vm_size' compute.\n", - "# Setting this to less than the number of GPUs will result in underutilized GPUs, taking longer to train.\n", - "# Setting this to more than the number of GPUs will result in an error.\n", - "gpus_per_node = 1 # default value\n", - "gpu_count_found = False\n", - "ws_computes = workspace_ml_client.compute.list_sizes()\n", - "for ws_compute in ws_computes:\n", - " if ws_compute.name.lower() == compute.size.lower():\n", - " gpus_per_node = ws_compute.gpus\n", - " print(f\"Number of GPUs in compute {ws_compute.name} are {ws_compute.gpus}\")\n", - "# if gpu_count_found not found, then print an error\n", - "if gpus_per_node > 0:\n", - " gpu_count_found = True\n", - "else:\n", - " gpu_count_found = False\n", - " print(f\"No GPUs found in compute. Number of GPUs in compute {compute.size} 0.\")" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Input Data for Evaluation" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# If you already have a gpu cluster, mention it here. Else will create a new one with the name 'ghyadav-westus-a100'\n", + "# model will only run on an a100 instance\n", + "\n", + "compute_cluster = \"nemo-westus\"\n", + "try:\n", + " compute = workspace_ml_client.compute.get(compute_cluster)\n", + " print(f\"GPU compute '{compute_cluster}' found.\")\n", + "except Exception as ex:\n", + " print(f\"GPU compute '{compute_cluster}' not found. Creating new one.\")\n", + " compute = AmlCompute(\n", + " name=compute_cluster,\n", + " size=\"STANDARD_ND96AMSR_A100_V4\",\n", + " max_instances=2, # For multi node training set this to an integer value more than 1\n", + " )\n", + " workspace_ml_client.compute.begin_create_or_update(compute).wait()\n", + "\n", + "# generating a unique timestamp that can be used for names and versions that need to be unique\n", + "timestamp = str(int(time.time()))\n", + "\n", + "\n", + "# This is the number of GPUs in a single node of the selected 'vm_size' compute.\n", + "# Setting this to less than the number of GPUs will result in underutilized GPUs, taking longer to train.\n", + "# Setting this to more than the number of GPUs will result in an error.\n", + "gpus_per_node = 1 # default value\n", + "gpu_count_found = False\n", + "ws_computes = workspace_ml_client.compute.list_sizes()\n", + "for ws_compute in ws_computes:\n", + " if ws_compute.name.lower() == compute.size.lower():\n", + " gpus_per_node = ws_compute.gpus\n", + " print(f\"Number of GPUs in compute {ws_compute.name} are {ws_compute.gpus}\")\n", + "# if gpu_count_found not found, then print an error\n", + "if gpus_per_node > 0:\n", + " gpu_count_found = True\n", + "else:\n", + " gpu_count_found = False\n", + " print(f\"No GPUs found in compute. Number of GPUs in compute {compute.size} 0.\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Input Data for Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700049437492 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700049437492 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# download the dataset using the helper script. This needs datasets library: https://pypi.org/project/datasets/\n", - "import os\n", - "from datasets import load_dataset, get_dataset_split_names\n", - "\n", - "dataset_dir = \"pubmed-dataset\"\n", - "dataset_name = \"pubmed_qa\"\n", - "# create the download directory if it does not exist\n", - "if not os.path.exists(dataset_dir):\n", - " os.makedirs(dataset_dir)\n", - "\n", - "\n", - "# import hugging face datasets library\n", - "\n", - "split = \"train\" # Only test available\n", - "dataset = load_dataset(dataset_name, \"pqa_labeled\", split=split)\n", - "# save the split of the dataset to the download directory as json lines file\n", - "dataset.to_json(os.path.join(dataset_dir, f\"{split}.jsonl\"))\n", - "# print dataset features" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700045144197 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "dataset" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# download the dataset using the helper script. This needs datasets library: https://pypi.org/project/datasets/\n", + "import os\n", + "from datasets import load_dataset, get_dataset_split_names\n", + "\n", + "dataset_dir = \"pubmed-dataset\"\n", + "dataset_name = \"pubmed_qa\"\n", + "# create the download directory if it does not exist\n", + "if not os.path.exists(dataset_dir):\n", + " os.makedirs(dataset_dir)\n", + "\n", + "\n", + "# import hugging face datasets library\n", + "\n", + "split = \"train\" # Only test available\n", + "dataset = load_dataset(dataset_name, \"pqa_labeled\", split=split)\n", + "# save the split of the dataset to the download directory as json lines file\n", + "dataset.to_json(os.path.join(dataset_dir, f\"{split}.jsonl\"))\n", + "# print dataset features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700045144197 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700045144360 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# evaluation_dataset = 'text-generation-data.jsonl' # evaluation_dataset_path\n", - "# prompt_data = '' # prompt_data_path\n", - "# load the ./samsum-dataset/train.jsonl file into a pandas dataframe and show the first 5 rows\n", - "import pandas as pd\n", - "\n", - "evaluation_dataset = \"./pubmed-dataset/test/test.jsonl\"\n", - "pd.set_option(\n", - " \"display.max_colwidth\", 0\n", - ") # set the max column width to 0 to display the full text\n", - "df = pd.read_json(\"./pubmed-dataset/train.jsonl\", lines=True)\n", - "df = df[:10]" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700045144673 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "def form_question(obj):\n", - " st = \"\"\n", - " st += f\"QUESTION: {obj['question']}\\n\"\n", - " st += \"CONTEXT: \"\n", - " for i in range(len(obj[\"context\"][\"labels\"])):\n", - " st += f\"{obj['context']['contexts'][i]}\\n\"\n", - " st += f\"TARGET: the answer to the question given the context is (yes|no|maybe): \"\n", - " return st\n", - "\n", - "\n", - "result = []\n", - "for i, row in df.iterrows():\n", - " result.append(form_question(row))\n", - "data = pd.DataFrame({\"input\": result})\n", - "data" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700045144360 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700045144799 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "frac = 1\n", - "evaluation_dataset = \"./pubmedqa-dataset/test/test_frac.jsonl\"\n", - "os.makedirs(\"./pubmedqa-dataset/test\", exist_ok=True)\n", - "data.sample(frac=frac).to_json(evaluation_dataset, orient=\"records\", lines=True)" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Loading model from Registry" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# evaluation_dataset = 'text-generation-data.jsonl' # evaluation_dataset_path\n", + "# prompt_data = '' # prompt_data_path\n", + "# load the ./samsum-dataset/train.jsonl file into a pandas dataframe and show the first 5 rows\n", + "import pandas as pd\n", + "\n", + "evaluation_dataset = \"./pubmed-dataset/test/test.jsonl\"\n", + "pd.set_option(\n", + " \"display.max_colwidth\", 0\n", + ") # set the max column width to 0 to display the full text\n", + "df = pd.read_json(\"./pubmed-dataset/train.jsonl\", lines=True)\n", + "df = df[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700045144673 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700049445465 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "model_name = \"Nemotron-3-8B-Base-4k\"\n", - "model_version = \"latest\"\n", - "\n", - "nemo_model_object = nemo_registry_ml_client.models.get(model_name, label=\"latest\")" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# Prompt" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "def form_question(obj):\n", + " st = \"\"\n", + " st += f\"QUESTION: {obj['question']}\\n\"\n", + " st += \"CONTEXT: \"\n", + " for i in range(len(obj[\"context\"][\"labels\"])):\n", + " st += f\"{obj['context']['contexts'][i]}\\n\"\n", + " st += f\"TARGET: the answer to the question given the context is (yes|no|maybe): \"\n", + " return st\n", + "\n", + "\n", + "result = []\n", + "for i, row in df.iterrows():\n", + " result.append(form_question(row))\n", + "data = pd.DataFrame({\"input\": result})\n", + "data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700045144799 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700049447829 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "prompt_path = \"prompts/prompt-qna.txt\"" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Submitting Evaluation Pipeline" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "frac = 1\n", + "evaluation_dataset = \"./pubmedqa-dataset/test/test_frac.jsonl\"\n", + "os.makedirs(\"./pubmedqa-dataset/test\", exist_ok=True)\n", + "data.sample(frac=frac).to_json(evaluation_dataset, orient=\"records\", lines=True)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Loading model from Registry" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700049445465 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700049645884 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from azure.ai.ml.dsl import pipeline\n", - "from azure.ai.ml import Input\n", - "from azure.ai.ml.constants import AssetTypes\n", - "\n", - "# fetch the pipeline component\n", - "pipeline_component_func = nemo_registry_ml_client.components.get(\n", - " name=\"nemo_qna_evaluation\",\n", - " label=\"latest\"\n", - " # name = \"nemo_prediction_base\", label=\"latest\"\n", - ")\n", - "openai_params = '{\"type\":\"azure_open_ai\",\"model_name\":\"gpt-35-turbo\",\"deployment_name\":\"gpt-35-turbo\",\"questions\":\"input\",\"contexts\":\"input\"}'\n", - "\n", - "\n", - "# define the pipeline job\n", - "@pipeline()\n", - "def evaluation_pipeline(mlflow_model):\n", - " evaluation_job = pipeline_component_func(\n", - " dataset_path=Input(type=AssetTypes.URI_FOLDER, path=\"./pubmedqa-dataset/test/\"),\n", - " model_path=Input(type=AssetTypes.TRITON_MODEL, path=f\"{nemo_model_object.id}\"),\n", - " openai_config_params=openai_params,\n", - " # prompt_schema=Input(type=AssetTypes.URI_FILE, path=prompt_path), # Not required for text-gen task\n", - " # prediction_column_name=\"output\",\n", - " )\n", - " return {\"evaluation_result\": evaluation_job.outputs.evaluation_result}" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "model_name = \"Nemotron-3-8B-Base-4k\"\n", + "model_version = \"latest\"\n", + "\n", + "nemo_model_object = nemo_registry_ml_client.models.get(model_name, label=\"latest\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "# Prompt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700049447829 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700049692743 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "experiment_name = \"nemo-qna-eval-pipeline\"\n", - "pipeline_jobs = []\n", - "\n", - "pipeline_object = evaluation_pipeline()\n", - "\n", - "# don't reuse cached results from previous jobs\n", - "pipeline_object.settings.force_rerun = True\n", - "pipeline_object.settings.default_compute = compute_cluster\n", - "pipeline_job = workspace_ml_client.jobs.create_or_update(\n", - " pipeline_object, experiment_name=experiment_name\n", - ")\n", - "# add model['name'] and pipeline_job.name as key value pairs to a dictionary\n", - "pipeline_jobs.append({\"model_name\": model_name, \"job_name\": pipeline_job.name})\n", - "# wait for the pipeline job to complete\n", - "workspace_ml_client.jobs.stream(pipeline_job.name)" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [] + "nteract": { + "transient": { + "deleting": false + } } - ], - "metadata": { - "kernel_info": { - "name": "ghyadav1" + }, + "outputs": [], + "source": [ + "prompt_path = \"prompts/prompt-qna.txt\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Submitting Evaluation Pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700049645884 }, - "kernelspec": { - "display_name": "ghyadav1", - "language": "python", - "name": "ghyadav1" + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.18" + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "from azure.ai.ml.dsl import pipeline\n", + "from azure.ai.ml import Input\n", + "from azure.ai.ml.constants import AssetTypes\n", + "\n", + "# fetch the pipeline component\n", + "pipeline_component_func = nemo_registry_ml_client.components.get(\n", + " name=\"nemo_qna_evaluation\",\n", + " label=\"latest\"\n", + " # name = \"nemo_prediction_base\", label=\"latest\"\n", + ")\n", + "openai_params = '{\"type\":\"azure_open_ai\",\"model_name\":\"gpt-35-turbo\",\"deployment_name\":\"gpt-35-turbo\",\"questions\":\"input\",\"contexts\":\"input\"}'\n", + "\n", + "\n", + "# define the pipeline job\n", + "@pipeline()\n", + "def evaluation_pipeline(mlflow_model):\n", + " evaluation_job = pipeline_component_func(\n", + " dataset_path=Input(type=AssetTypes.URI_FOLDER, path=\"./pubmedqa-dataset/test/\"),\n", + " model_path=Input(type=AssetTypes.TRITON_MODEL, path=f\"{nemo_model_object.id}\"),\n", + " openai_config_params=openai_params,\n", + " # prompt_schema=Input(type=AssetTypes.URI_FILE, path=prompt_path), # Not required for text-gen task\n", + " # prediction_column_name=\"output\",\n", + " )\n", + " return {\"evaluation_result\": evaluation_job.outputs.evaluation_result}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700049692743 }, - "microsoft": { - "host": { - "AzureML": { - "notebookHasBeenCompleted": true - } - }, - "ms_spell_check": { - "ms_spell_check_language": "en" - } + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, "nteract": { - "version": "nteract-front-end@1.0.0" + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "experiment_name = \"nemo-qna-eval-pipeline\"\n", + "pipeline_jobs = []\n", + "\n", + "pipeline_object = evaluation_pipeline()\n", + "\n", + "# don't reuse cached results from previous jobs\n", + "pipeline_object.settings.force_rerun = True\n", + "pipeline_object.settings.default_compute = compute_cluster\n", + "pipeline_job = workspace_ml_client.jobs.create_or_update(\n", + " pipeline_object, experiment_name=experiment_name\n", + ")\n", + "# add model['name'] and pipeline_job.name as key value pairs to a dictionary\n", + "pipeline_jobs.append({\"model_name\": model_name, \"job_name\": pipeline_job.name})\n", + "# wait for the pipeline job to complete\n", + "workspace_ml_client.jobs.stream(pipeline_job.name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernel_info": { + "name": "ghyadav1" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + }, + "microsoft": { + "host": { + "AzureML": { + "notebookHasBeenCompleted": true } + }, + "ms_spell_check": { + "ms_spell_check_language": "en" + } }, - "nbformat": 4, - "nbformat_minor": 2 + "nteract": { + "version": "nteract-front-end@1.0.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-translation-evaluation.ipynb b/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-translation-evaluation.ipynb index 42619a9181..80285f5012 100644 --- a/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-translation-evaluation.ipynb +++ b/sdk/python/foundation-models/nvidia-nemo/evaluate/nvidia-text-translation-evaluation.ipynb @@ -1,430 +1,430 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1699605126817 - } - }, - "outputs": [], - "source": [ - "# Prequisites\n", - "\n", - "# %pip install --upgrade azure-ai-ml\n", - "# %pip install --upgrade azure-identity\n", - "# %pip install --upgrade datasets==2.9.0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700024000036 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from azure.ai.ml import MLClient\n", - "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", - "from azure.ai.ml.entities import AmlCompute\n", - "import time\n", - "\n", - "try:\n", - " credential = DefaultAzureCredential()\n", - " credential.get_token(\"https://management.azure.com/.default\")\n", - "except Exception as ex:\n", - " credential = InteractiveBrowserCredential()\n", - "\n", - "workspace_ml_client = None\n", - "try:\n", - " workspace_ml_client = MLClient.from_config(credential)\n", - " subscription_id = workspace_ml_client.subscription_id\n", - " workspace = workspace_ml_client.workspace_name\n", - " resource_group = workspace_ml_client.resource_group_name\n", - "except Exception as ex:\n", - " print(ex)\n", - " # Enter details of your AML workspace\n", - " subscription_id = \"\"\n", - " resource_group = \"\"\n", - " workspace = \"\"\n", - " workspace_ml_client = MLClient(\n", - " credential, subscription_id, resource_group, workspace\n", - " )\n", - "\n", - "# replace with the registry name\n", - "nemo_registry = \"nvidia-ai\" # \"azureml\"\n", - "\n", - "# registry where models are present\n", - "model_registry = \"azureml-preview-test1\"\n", - "\n", - "nemo_registry_ml_client = MLClient(\n", - " credential, subscription_id, resource_group, registry_name=nemo_registry\n", - ")\n", - "model_registry_ml_client = MLClient(\n", - " credential, subscription_id, resource_group, registry_name=model_registry\n", - ")\n", - "nemo_registry_ml_client" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Creating Computes" - ] + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1699605126817 + } + }, + "outputs": [], + "source": [ + "# Prequisites\n", + "\n", + "# %pip install --upgrade azure-ai-ml\n", + "# %pip install --upgrade azure-identity\n", + "# %pip install --upgrade datasets==2.9.0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700024000036 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700024014133 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "# If you already have a gpu cluster, mention it here. Else will create a new one with the name 'ghyadav-westus-a100'\n", - "# model will only run on an a100 instance\n", - "\n", - "compute_cluster = \"nemo-westus\"\n", - "try:\n", - " compute = workspace_ml_client.compute.get(compute_cluster)\n", - " print(f\"GPU compute '{compute_cluster}' found.\")\n", - "except Exception as ex:\n", - " print(f\"GPU compute '{compute_cluster}' not found. Creating new one.\")\n", - " compute = AmlCompute(\n", - " name=compute_cluster,\n", - " size=\"STANDARD_ND96AMSR_A100_V4\",\n", - " max_instances=2, # For multi node training set this to an integer value more than 1\n", - " )\n", - " workspace_ml_client.compute.begin_create_or_update(compute).wait()\n", - "\n", - "# generating a unique timestamp that can be used for names and versions that need to be unique\n", - "timestamp = str(int(time.time()))\n", - "\n", - "\n", - "# This is the number of GPUs in a single node of the selected 'vm_size' compute.\n", - "# Setting this to less than the number of GPUs will result in underutilized GPUs, taking longer to train.\n", - "# Setting this to more than the number of GPUs will result in an error.\n", - "gpus_per_node = 1 # default value\n", - "gpu_count_found = False\n", - "ws_computes = workspace_ml_client.compute.list_sizes()\n", - "for ws_compute in ws_computes:\n", - " if ws_compute.name.lower() == compute.size.lower():\n", - " gpus_per_node = ws_compute.gpus\n", - " print(f\"Number of GPUs in compute {ws_compute.name} are {ws_compute.gpus}\")\n", - "# if gpu_count_found not found, then print an error\n", - "if gpus_per_node > 0:\n", - " gpu_count_found = True\n", - "else:\n", - " gpu_count_found = False\n", - " print(f\"No GPUs found in compute. Number of GPUs in compute {compute.size} 0.\")" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Input Data for Evaluation" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "from azure.ai.ml import MLClient\n", + "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", + "from azure.ai.ml.entities import AmlCompute\n", + "import time\n", + "\n", + "try:\n", + " credential = DefaultAzureCredential()\n", + " credential.get_token(\"https://management.azure.com/.default\")\n", + "except Exception as ex:\n", + " credential = InteractiveBrowserCredential()\n", + "\n", + "workspace_ml_client = None\n", + "try:\n", + " workspace_ml_client = MLClient.from_config(credential)\n", + " subscription_id = workspace_ml_client.subscription_id\n", + " workspace = workspace_ml_client.workspace_name\n", + " resource_group = workspace_ml_client.resource_group_name\n", + "except Exception as ex:\n", + " print(ex)\n", + " # Enter details of your AML workspace\n", + " subscription_id = \"\"\n", + " resource_group = \"\"\n", + " workspace = \"\"\n", + " workspace_ml_client = MLClient(\n", + " credential, subscription_id, resource_group, workspace\n", + " )\n", + "\n", + "# replace with the registry name\n", + "nemo_registry = \"nvidia-ai\" # \"azureml\"\n", + "\n", + "# registry where models are present\n", + "model_registry = \"azureml-preview-test1\"\n", + "\n", + "nemo_registry_ml_client = MLClient(\n", + " credential, subscription_id, resource_group, registry_name=nemo_registry\n", + ")\n", + "model_registry_ml_client = MLClient(\n", + " credential, subscription_id, resource_group, registry_name=model_registry\n", + ")\n", + "nemo_registry_ml_client" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Creating Computes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700024014133 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700024014219 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "evaluation_dataset = \"./data/translation/\"" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Loading model from Registry" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "# If you already have a gpu cluster, mention it here. Else will create a new one with the name 'ghyadav-westus-a100'\n", + "# model will only run on an a100 instance\n", + "\n", + "compute_cluster = \"nemo-westus\"\n", + "try:\n", + " compute = workspace_ml_client.compute.get(compute_cluster)\n", + " print(f\"GPU compute '{compute_cluster}' found.\")\n", + "except Exception as ex:\n", + " print(f\"GPU compute '{compute_cluster}' not found. Creating new one.\")\n", + " compute = AmlCompute(\n", + " name=compute_cluster,\n", + " size=\"STANDARD_ND96AMSR_A100_V4\",\n", + " max_instances=2, # For multi node training set this to an integer value more than 1\n", + " )\n", + " workspace_ml_client.compute.begin_create_or_update(compute).wait()\n", + "\n", + "# generating a unique timestamp that can be used for names and versions that need to be unique\n", + "timestamp = str(int(time.time()))\n", + "\n", + "\n", + "# This is the number of GPUs in a single node of the selected 'vm_size' compute.\n", + "# Setting this to less than the number of GPUs will result in underutilized GPUs, taking longer to train.\n", + "# Setting this to more than the number of GPUs will result in an error.\n", + "gpus_per_node = 1 # default value\n", + "gpu_count_found = False\n", + "ws_computes = workspace_ml_client.compute.list_sizes()\n", + "for ws_compute in ws_computes:\n", + " if ws_compute.name.lower() == compute.size.lower():\n", + " gpus_per_node = ws_compute.gpus\n", + " print(f\"Number of GPUs in compute {ws_compute.name} are {ws_compute.gpus}\")\n", + "# if gpu_count_found not found, then print an error\n", + "if gpus_per_node > 0:\n", + " gpu_count_found = True\n", + "else:\n", + " gpu_count_found = False\n", + " print(f\"No GPUs found in compute. Number of GPUs in compute {compute.size} 0.\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Input Data for Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700024014219 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700052548098 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "model_name = \"Nemotron-3-8B-Base-4k\"\n", - "model_version = \"latest\"\n", - "\n", - "nemo_model_object = nemo_registry_ml_client.models.get(model_name, label=\"latest\")" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# Prompt" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "evaluation_dataset = \"./data/translation/\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Loading model from Registry" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700052548098 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700052554188 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "prompt_path = \"./prompts/prompt-translate.txt\"" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## Submitting Evaluation Pipeline" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "model_name = \"Nemotron-3-8B-Base-4k\"\n", + "model_version = \"latest\"\n", + "\n", + "nemo_model_object = nemo_registry_ml_client.models.get(model_name, label=\"latest\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "# Prompt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700052554188 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1700052555084 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from azure.ai.ml.dsl import pipeline\n", - "from azure.ai.ml import Input\n", - "from azure.ai.ml.constants import AssetTypes\n", - "\n", - "# fetch the pipeline component\n", - "pipeline_component_func = nemo_registry_ml_client.components.get(\n", - " name=\"nemo_translation_evaluation\",\n", - " label=\"latest\"\n", - " # name = \"nemo_prediction_base\", label=\"latest\"\n", - ")\n", - "\n", - "\n", - "# define the pipeline job\n", - "@pipeline()\n", - "def evaluation_pipeline(mlflow_model):\n", - " evaluation_job = pipeline_component_func(\n", - " dataset_path=Input(type=AssetTypes.URI_FOLDER, path=\"./data/translation/\"),\n", - " model_path=Input(type=AssetTypes.TRITON_MODEL, path=f\"{nemo_model_object.id}\"),\n", - " prompt_schema=Input(\n", - " type=AssetTypes.URI_FILE, path=prompt_path\n", - " ), # Not required for text-gen task\n", - " # prediction_column_name=\"output\",\n", - " )\n", - " return {\"evaluation_result\": evaluation_job.outputs.evaluation_result}" - ] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "gather": { - "logged": 1699965532217 - }, - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "experiment_name = \"nemo-translate-eval-pipeline\"\n", - "pipeline_jobs = []\n", - "\n", - "pipeline_object = evaluation_pipeline()\n", - "\n", - "# don't reuse cached results from previous jobs\n", - "pipeline_object.settings.force_rerun = True\n", - "pipeline_object.settings.default_compute = compute_cluster\n", - "pipeline_job = workspace_ml_client.jobs.create_or_update(\n", - " pipeline_object, experiment_name=experiment_name\n", - ")\n", - "# add model['name'] and pipeline_job.name as key value pairs to a dictionary\n", - "pipeline_jobs.append({\"model_name\": model_name, \"job_name\": pipeline_job.name})\n", - "# wait for the pipeline job to complete\n", - "workspace_ml_client.jobs.stream(pipeline_job.name)" - ] + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "prompt_path = \"./prompts/prompt-translate.txt\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Submitting Evaluation Pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1700052555084 }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [] + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [] + "nteract": { + "transient": { + "deleting": false + } } - ], - "metadata": { - "kernel_info": { - "name": "ghyadav1" + }, + "outputs": [], + "source": [ + "from azure.ai.ml.dsl import pipeline\n", + "from azure.ai.ml import Input\n", + "from azure.ai.ml.constants import AssetTypes\n", + "\n", + "# fetch the pipeline component\n", + "pipeline_component_func = nemo_registry_ml_client.components.get(\n", + " name=\"nemo_translation_evaluation\",\n", + " label=\"latest\"\n", + " # name = \"nemo_prediction_base\", label=\"latest\"\n", + ")\n", + "\n", + "\n", + "# define the pipeline job\n", + "@pipeline()\n", + "def evaluation_pipeline(mlflow_model):\n", + " evaluation_job = pipeline_component_func(\n", + " dataset_path=Input(type=AssetTypes.URI_FOLDER, path=\"./data/translation/\"),\n", + " model_path=Input(type=AssetTypes.TRITON_MODEL, path=f\"{nemo_model_object.id}\"),\n", + " prompt_schema=Input(\n", + " type=AssetTypes.URI_FILE, path=prompt_path\n", + " ), # Not required for text-gen task\n", + " # prediction_column_name=\"output\",\n", + " )\n", + " return {\"evaluation_result\": evaluation_job.outputs.evaluation_result}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1699965532217 }, - "kernelspec": { - "display_name": "ghyadav1", - "language": "python", - "name": "ghyadav1" + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.18" + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "experiment_name = \"nemo-translate-eval-pipeline\"\n", + "pipeline_jobs = []\n", + "\n", + "pipeline_object = evaluation_pipeline()\n", + "\n", + "# don't reuse cached results from previous jobs\n", + "pipeline_object.settings.force_rerun = True\n", + "pipeline_object.settings.default_compute = compute_cluster\n", + "pipeline_job = workspace_ml_client.jobs.create_or_update(\n", + " pipeline_object, experiment_name=experiment_name\n", + ")\n", + "# add model['name'] and pipeline_job.name as key value pairs to a dictionary\n", + "pipeline_jobs.append({\"model_name\": model_name, \"job_name\": pipeline_job.name})\n", + "# wait for the pipeline job to complete\n", + "workspace_ml_client.jobs.stream(pipeline_job.name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, - "microsoft": { - "host": { - "AzureML": { - "notebookHasBeenCompleted": true - } - }, - "ms_spell_check": { - "ms_spell_check_language": "en" - } + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, "nteract": { - "version": "nteract-front-end@1.0.0" + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernel_info": { + "name": "ghyadav1" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + }, + "microsoft": { + "host": { + "AzureML": { + "notebookHasBeenCompleted": true } + }, + "ms_spell_check": { + "ms_spell_check_language": "en" + } }, - "nbformat": 4, - "nbformat_minor": 2 + "nteract": { + "version": "nteract-front-end@1.0.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/sdk/python/foundation-models/nvidia-nemo/ptune_evaluate.ipynb b/sdk/python/foundation-models/nvidia-nemo/ptune_evaluate.ipynb index 298ec4f73b..070bcf5468 100644 --- a/sdk/python/foundation-models/nvidia-nemo/ptune_evaluate.ipynb +++ b/sdk/python/foundation-models/nvidia-nemo/ptune_evaluate.ipynb @@ -793,9 +793,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/nvidia-nim-llama3-8b/aiinference.ipynb b/sdk/python/foundation-models/nvidia-nim-llama3-8b/aiinference.ipynb index 8c62d8e7aa..6b7d5852f2 100644 --- a/sdk/python/foundation-models/nvidia-nim-llama3-8b/aiinference.ipynb +++ b/sdk/python/foundation-models/nvidia-nim-llama3-8b/aiinference.ipynb @@ -67,9 +67,9 @@ ], "metadata": { "kernelspec": { - "display_name": "dev", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/nvidia-nim-llama3-8b/openaisdk.ipynb b/sdk/python/foundation-models/nvidia-nim-llama3-8b/openaisdk.ipynb index cc3411d552..00cca7f237 100644 --- a/sdk/python/foundation-models/nvidia-nim-llama3-8b/openaisdk.ipynb +++ b/sdk/python/foundation-models/nvidia-nim-llama3-8b/openaisdk.ipynb @@ -132,9 +132,9 @@ ], "metadata": { "kernelspec": { - "display_name": "dev", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/nvidia-nim-llama3-8b/webrequests.ipynb b/sdk/python/foundation-models/nvidia-nim-llama3-8b/webrequests.ipynb index 208ffe8244..78ff419e37 100644 --- a/sdk/python/foundation-models/nvidia-nim-llama3-8b/webrequests.ipynb +++ b/sdk/python/foundation-models/nvidia-nim-llama3-8b/webrequests.ipynb @@ -309,9 +309,9 @@ ], "metadata": { "kernelspec": { - "display_name": "dev", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/phi-3/litellm.ipynb b/sdk/python/foundation-models/phi-3/litellm.ipynb index 0b53b16c77..d19aa5a0c5 100644 --- a/sdk/python/foundation-models/phi-3/litellm.ipynb +++ b/sdk/python/foundation-models/phi-3/litellm.ipynb @@ -252,7 +252,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/phi-3/openaisdk.ipynb b/sdk/python/foundation-models/phi-3/openaisdk.ipynb index 5e21947702..9d51135a3d 100644 --- a/sdk/python/foundation-models/phi-3/openaisdk.ipynb +++ b/sdk/python/foundation-models/phi-3/openaisdk.ipynb @@ -257,9 +257,9 @@ ], "metadata": { "kernelspec": { - "display_name": "azureml_py38_PT_TF", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/phi-3/webrequests.ipynb b/sdk/python/foundation-models/phi-3/webrequests.ipynb index 782dc3fd99..8509bc92ef 100644 --- a/sdk/python/foundation-models/phi-3/webrequests.ipynb +++ b/sdk/python/foundation-models/phi-3/webrequests.ipynb @@ -253,7 +253,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/stabilityai/Image_to_Image.ipynb b/sdk/python/foundation-models/stabilityai/Image_to_Image.ipynb index c7bf63b5b3..d7646634fc 100644 --- a/sdk/python/foundation-models/stabilityai/Image_to_Image.ipynb +++ b/sdk/python/foundation-models/stabilityai/Image_to_Image.ipynb @@ -199,9 +199,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/stabilityai/Text_to_Image_openai_library.ipynb b/sdk/python/foundation-models/stabilityai/Text_to_Image_openai_library.ipynb index 65d1b90a6a..3001a9e18c 100644 --- a/sdk/python/foundation-models/stabilityai/Text_to_Image_openai_library.ipynb +++ b/sdk/python/foundation-models/stabilityai/Text_to_Image_openai_library.ipynb @@ -143,9 +143,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/stabilityai/Text_to_Image_requests_library.ipynb b/sdk/python/foundation-models/stabilityai/Text_to_Image_requests_library.ipynb index 8d19ddc6db..750aed776b 100644 --- a/sdk/python/foundation-models/stabilityai/Text_to_Image_requests_library.ipynb +++ b/sdk/python/foundation-models/stabilityai/Text_to_Image_requests_library.ipynb @@ -176,9 +176,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/distillation/conversation/distillation_conversational_task.ipynb b/sdk/python/foundation-models/system/distillation/conversation/distillation_conversational_task.ipynb index 8d17f74ca4..99f72e976a 100644 --- a/sdk/python/foundation-models/system/distillation/conversation/distillation_conversational_task.ipynb +++ b/sdk/python/foundation-models/system/distillation/conversation/distillation_conversational_task.ipynb @@ -811,9 +811,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/distillation/math/distillation_math.ipynb b/sdk/python/foundation-models/system/distillation/math/distillation_math.ipynb index cdb8af72bc..d64046d5bc 100644 --- a/sdk/python/foundation-models/system/distillation/math/distillation_math.ipynb +++ b/sdk/python/foundation-models/system/distillation/math/distillation_math.ipynb @@ -820,6 +820,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/distillation/nli/distillation_chat_completion.ipynb b/sdk/python/foundation-models/system/distillation/nli/distillation_chat_completion.ipynb index 452b12f8e2..0e13a24a0b 100644 --- a/sdk/python/foundation-models/system/distillation/nli/distillation_chat_completion.ipynb +++ b/sdk/python/foundation-models/system/distillation/nli/distillation_chat_completion.ipynb @@ -825,6 +825,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/distillation/nlu_qa/distillation_nlu_qa_task.ipynb b/sdk/python/foundation-models/system/distillation/nlu_qa/distillation_nlu_qa_task.ipynb index edf24d706d..c4f406c2fc 100644 --- a/sdk/python/foundation-models/system/distillation/nlu_qa/distillation_nlu_qa_task.ipynb +++ b/sdk/python/foundation-models/system/distillation/nlu_qa/distillation_nlu_qa_task.ipynb @@ -838,9 +838,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/distillation/nlu_qa/distillation_qa_math.ipynb b/sdk/python/foundation-models/system/distillation/nlu_qa/distillation_qa_math.ipynb index 9368a04e99..a9cee09681 100644 --- a/sdk/python/foundation-models/system/distillation/nlu_qa/distillation_qa_math.ipynb +++ b/sdk/python/foundation-models/system/distillation/nlu_qa/distillation_qa_math.ipynb @@ -808,9 +808,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/distillation/summarization/distillation_summarization.ipynb b/sdk/python/foundation-models/system/distillation/summarization/distillation_summarization.ipynb index c368ef0f0c..6ceff30fd4 100644 --- a/sdk/python/foundation-models/system/distillation/summarization/distillation_summarization.ipynb +++ b/sdk/python/foundation-models/system/distillation/summarization/distillation_summarization.ipynb @@ -789,6 +789,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/evaluation/chat-completion/chat-completion.ipynb b/sdk/python/foundation-models/system/evaluation/chat-completion/chat-completion.ipynb index dc313efb52..a5a10b49ad 100644 --- a/sdk/python/foundation-models/system/evaluation/chat-completion/chat-completion.ipynb +++ b/sdk/python/foundation-models/system/evaluation/chat-completion/chat-completion.ipynb @@ -464,7 +464,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/system/evaluation/image-classification/multiclass-classification/image-multiclass-classification.ipynb b/sdk/python/foundation-models/system/evaluation/image-classification/multiclass-classification/image-multiclass-classification.ipynb index fbe11bb3e1..cb210eaca7 100644 --- a/sdk/python/foundation-models/system/evaluation/image-classification/multiclass-classification/image-multiclass-classification.ipynb +++ b/sdk/python/foundation-models/system/evaluation/image-classification/multiclass-classification/image-multiclass-classification.ipynb @@ -693,6 +693,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/evaluation/image-classification/multilabel-classification/image-multilabel-classification.ipynb b/sdk/python/foundation-models/system/evaluation/image-classification/multilabel-classification/image-multilabel-classification.ipynb index 3bc21c2163..f83e5f9501 100644 --- a/sdk/python/foundation-models/system/evaluation/image-classification/multilabel-classification/image-multilabel-classification.ipynb +++ b/sdk/python/foundation-models/system/evaluation/image-classification/multilabel-classification/image-multilabel-classification.ipynb @@ -691,6 +691,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/evaluation/image-instance-segmentation/image-instance-segmentation.ipynb b/sdk/python/foundation-models/system/evaluation/image-instance-segmentation/image-instance-segmentation.ipynb index e893dcb59b..a3f8966558 100644 --- a/sdk/python/foundation-models/system/evaluation/image-instance-segmentation/image-instance-segmentation.ipynb +++ b/sdk/python/foundation-models/system/evaluation/image-instance-segmentation/image-instance-segmentation.ipynb @@ -738,6 +738,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/evaluation/image-object-detection/image-object-detection.ipynb b/sdk/python/foundation-models/system/evaluation/image-object-detection/image-object-detection.ipynb index bbf1dc7a61..9a8f3d2919 100644 --- a/sdk/python/foundation-models/system/evaluation/image-object-detection/image-object-detection.ipynb +++ b/sdk/python/foundation-models/system/evaluation/image-object-detection/image-object-detection.ipynb @@ -723,6 +723,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/finetune/Llama-notebooks/datagen/synthetic-data-generation.ipynb b/sdk/python/foundation-models/system/finetune/Llama-notebooks/datagen/synthetic-data-generation.ipynb index f8c5a588d7..53e0d6105e 100644 --- a/sdk/python/foundation-models/system/finetune/Llama-notebooks/datagen/synthetic-data-generation.ipynb +++ b/sdk/python/foundation-models/system/finetune/Llama-notebooks/datagen/synthetic-data-generation.ipynb @@ -416,9 +416,9 @@ "name": "python310-sdkv2" }, "kernelspec": { - "name": "python310-sdkv2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "display_name": "Python 3.10 - SDK v2" + "name": "python310-sdkv2" }, "language_info": { "name": "python", @@ -448,4 +448,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/emotion-detection-llama-multinode-serverless.ipynb b/sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/emotion-detection-llama-multinode-serverless.ipynb index ec1c495bb2..47be839c64 100644 --- a/sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/emotion-detection-llama-multinode-serverless.ipynb +++ b/sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/emotion-detection-llama-multinode-serverless.ipynb @@ -701,9 +701,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { @@ -721,4 +721,3 @@ "nbformat": 4, "nbformat_minor": 2 } - diff --git a/sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/emotion-detection-llama-multinode.ipynb b/sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/emotion-detection-llama-multinode.ipynb index 9885a52032..00609b8c69 100644 --- a/sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/emotion-detection-llama-multinode.ipynb +++ b/sdk/python/foundation-models/system/finetune/Llama-notebooks/multinode-text-classification/emotion-detection-llama-multinode.ipynb @@ -850,9 +850,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification/emotion-detection-llama-serverless-compute.ipynb b/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification/emotion-detection-llama-serverless-compute.ipynb index c8772ad753..69f2486ce0 100644 --- a/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification/emotion-detection-llama-serverless-compute.ipynb +++ b/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification/emotion-detection-llama-serverless-compute.ipynb @@ -663,9 +663,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification/emotion-detection-llama.ipynb b/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification/emotion-detection-llama.ipynb index e0472bce8c..fb40a72927 100644 --- a/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification/emotion-detection-llama.ipynb +++ b/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-classification/emotion-detection-llama.ipynb @@ -812,9 +812,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation/abstractive_qna_with_text_gen.ipynb b/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation/abstractive_qna_with_text_gen.ipynb index 6f302c9824..313362bc5a 100644 --- a/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation/abstractive_qna_with_text_gen.ipynb +++ b/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation/abstractive_qna_with_text_gen.ipynb @@ -831,9 +831,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation/summarization_with_text_gen.ipynb b/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation/summarization_with_text_gen.ipynb index f7349fdc3a..9db4f1d7d3 100644 --- a/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation/summarization_with_text_gen.ipynb +++ b/sdk/python/foundation-models/system/finetune/Llama-notebooks/text-generation/summarization_with_text_gen.ipynb @@ -805,9 +805,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/chat-completion/chat-completion.ipynb b/sdk/python/foundation-models/system/finetune/chat-completion/chat-completion.ipynb index ea52cd621a..406bac7da0 100644 --- a/sdk/python/foundation-models/system/finetune/chat-completion/chat-completion.ipynb +++ b/sdk/python/foundation-models/system/finetune/chat-completion/chat-completion.ipynb @@ -752,9 +752,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/image-classification/multiclass-classification/hftransformers-fridgeobjects-multiclass-classification.ipynb b/sdk/python/foundation-models/system/finetune/image-classification/multiclass-classification/hftransformers-fridgeobjects-multiclass-classification.ipynb index fcb272dcc8..280e3ba441 100644 --- a/sdk/python/foundation-models/system/finetune/image-classification/multiclass-classification/hftransformers-fridgeobjects-multiclass-classification.ipynb +++ b/sdk/python/foundation-models/system/finetune/image-classification/multiclass-classification/hftransformers-fridgeobjects-multiclass-classification.ipynb @@ -1170,6 +1170,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/finetune/image-classification/multilabel-classification/hftransformers-fridgeobjects-multilabel-classification.ipynb b/sdk/python/foundation-models/system/finetune/image-classification/multilabel-classification/hftransformers-fridgeobjects-multilabel-classification.ipynb index 3d125f32f9..b5a521e056 100644 --- a/sdk/python/foundation-models/system/finetune/image-classification/multilabel-classification/hftransformers-fridgeobjects-multilabel-classification.ipynb +++ b/sdk/python/foundation-models/system/finetune/image-classification/multilabel-classification/hftransformers-fridgeobjects-multilabel-classification.ipynb @@ -1176,6 +1176,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/finetune/image-instance-segmentation/mmdetection-fridgeobjects-instance-segmentation.ipynb b/sdk/python/foundation-models/system/finetune/image-instance-segmentation/mmdetection-fridgeobjects-instance-segmentation.ipynb index 79e77d4e5f..fa64cb0487 100644 --- a/sdk/python/foundation-models/system/finetune/image-instance-segmentation/mmdetection-fridgeobjects-instance-segmentation.ipynb +++ b/sdk/python/foundation-models/system/finetune/image-instance-segmentation/mmdetection-fridgeobjects-instance-segmentation.ipynb @@ -1219,6 +1219,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/finetune/image-object-detection/mmdetection-fridgeobjects-object-detection.ipynb b/sdk/python/foundation-models/system/finetune/image-object-detection/mmdetection-fridgeobjects-object-detection.ipynb index 78a8eb3c46..8fc8332873 100644 --- a/sdk/python/foundation-models/system/finetune/image-object-detection/mmdetection-fridgeobjects-object-detection.ipynb +++ b/sdk/python/foundation-models/system/finetune/image-object-detection/mmdetection-fridgeobjects-object-detection.ipynb @@ -1299,6 +1299,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/finetune/pipeline_validations/common.ipynb b/sdk/python/foundation-models/system/finetune/pipeline_validations/common.ipynb index cb541ee1ee..c06a280879 100644 --- a/sdk/python/foundation-models/system/finetune/pipeline_validations/common.ipynb +++ b/sdk/python/foundation-models/system/finetune/pipeline_validations/common.ipynb @@ -153,9 +153,9 @@ ], "metadata": { "kernelspec": { - "display_name": "streaming", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/question-answering/extractive-qa.ipynb b/sdk/python/foundation-models/system/finetune/question-answering/extractive-qa.ipynb index 23fa9c976f..cdeaecd626 100644 --- a/sdk/python/foundation-models/system/finetune/question-answering/extractive-qa.ipynb +++ b/sdk/python/foundation-models/system/finetune/question-answering/extractive-qa.ipynb @@ -788,9 +788,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/scenarios/very-large-model-deepspeed-stage3-finetuning/summarization_with_text_gen.ipynb b/sdk/python/foundation-models/system/finetune/scenarios/very-large-model-deepspeed-stage3-finetuning/summarization_with_text_gen.ipynb index 318652dcaa..b84fe9141d 100644 --- a/sdk/python/foundation-models/system/finetune/scenarios/very-large-model-deepspeed-stage3-finetuning/summarization_with_text_gen.ipynb +++ b/sdk/python/foundation-models/system/finetune/scenarios/very-large-model-deepspeed-stage3-finetuning/summarization_with_text_gen.ipynb @@ -797,9 +797,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Finetune", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "finetuneenv" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/summarization/news-summary.ipynb b/sdk/python/foundation-models/system/finetune/summarization/news-summary.ipynb index 142922d83b..ac594a0307 100644 --- a/sdk/python/foundation-models/system/finetune/summarization/news-summary.ipynb +++ b/sdk/python/foundation-models/system/finetune/summarization/news-summary.ipynb @@ -787,9 +787,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/text-classification/emotion-detection.ipynb b/sdk/python/foundation-models/system/finetune/text-classification/emotion-detection.ipynb index abf821c819..2164284c55 100644 --- a/sdk/python/foundation-models/system/finetune/text-classification/emotion-detection.ipynb +++ b/sdk/python/foundation-models/system/finetune/text-classification/emotion-detection.ipynb @@ -809,9 +809,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/text-generation/abstractive_qna_with_text_gen.ipynb b/sdk/python/foundation-models/system/finetune/text-generation/abstractive_qna_with_text_gen.ipynb index f850289ca9..c2b4149dca 100644 --- a/sdk/python/foundation-models/system/finetune/text-generation/abstractive_qna_with_text_gen.ipynb +++ b/sdk/python/foundation-models/system/finetune/text-generation/abstractive_qna_with_text_gen.ipynb @@ -898,9 +898,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/text-generation/summarization_with_text_gen.ipynb b/sdk/python/foundation-models/system/finetune/text-generation/summarization_with_text_gen.ipynb index b4bd9a7c62..3f440987ea 100644 --- a/sdk/python/foundation-models/system/finetune/text-generation/summarization_with_text_gen.ipynb +++ b/sdk/python/foundation-models/system/finetune/text-generation/summarization_with_text_gen.ipynb @@ -870,9 +870,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/text-to-image/diffusers-dreambooth-dog-text-to-image.ipynb b/sdk/python/foundation-models/system/finetune/text-to-image/diffusers-dreambooth-dog-text-to-image.ipynb index eb4b6c4efd..af06185e4f 100644 --- a/sdk/python/foundation-models/system/finetune/text-to-image/diffusers-dreambooth-dog-text-to-image.ipynb +++ b/sdk/python/foundation-models/system/finetune/text-to-image/diffusers-dreambooth-dog-text-to-image.ipynb @@ -874,6 +874,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/finetune/token-classification/token-classification.ipynb b/sdk/python/foundation-models/system/finetune/token-classification/token-classification.ipynb index 3fe6db10d0..1fee85d62d 100644 --- a/sdk/python/foundation-models/system/finetune/token-classification/token-classification.ipynb +++ b/sdk/python/foundation-models/system/finetune/token-classification/token-classification.ipynb @@ -780,9 +780,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/translation/translation.ipynb b/sdk/python/foundation-models/system/finetune/translation/translation.ipynb index 163e900f18..ed923ec317 100644 --- a/sdk/python/foundation-models/system/finetune/translation/translation.ipynb +++ b/sdk/python/foundation-models/system/finetune/translation/translation.ipynb @@ -777,9 +777,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/finetune/video-multi-object-tracking/mmtracking-video-multi-object-tracking.ipynb b/sdk/python/foundation-models/system/finetune/video-multi-object-tracking/mmtracking-video-multi-object-tracking.ipynb index 6cbf256f95..29f50fc73a 100644 --- a/sdk/python/foundation-models/system/finetune/video-multi-object-tracking/mmtracking-video-multi-object-tracking.ipynb +++ b/sdk/python/foundation-models/system/finetune/video-multi-object-tracking/mmtracking-video-multi-object-tracking.ipynb @@ -1166,6 +1166,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/automatic-speech-recognition/asr-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/automatic-speech-recognition/asr-batch-endpoint.ipynb index 4d1aced21d..3c672ca8bc 100644 --- a/sdk/python/foundation-models/system/inference/automatic-speech-recognition/asr-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/automatic-speech-recognition/asr-batch-endpoint.ipynb @@ -528,9 +528,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/automatic-speech-recognition/asr-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/automatic-speech-recognition/asr-online-endpoint.ipynb index a1ac2c5f02..745f1547ce 100644 --- a/sdk/python/foundation-models/system/inference/automatic-speech-recognition/asr-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/automatic-speech-recognition/asr-online-endpoint.ipynb @@ -210,9 +210,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/fill-mask/fill-mask-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/fill-mask/fill-mask-batch-endpoint.ipynb index 1a655c8222..832b505d01 100644 --- a/sdk/python/foundation-models/system/inference/fill-mask/fill-mask-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/fill-mask/fill-mask-batch-endpoint.ipynb @@ -530,9 +530,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/fill-mask/fill-mask-online-endpoint-oss.ipynb b/sdk/python/foundation-models/system/inference/fill-mask/fill-mask-online-endpoint-oss.ipynb index 5c3d8ea631..74de2389fa 100644 --- a/sdk/python/foundation-models/system/inference/fill-mask/fill-mask-online-endpoint-oss.ipynb +++ b/sdk/python/foundation-models/system/inference/fill-mask/fill-mask-online-endpoint-oss.ipynb @@ -322,9 +322,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/fill-mask/fill-mask-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/fill-mask/fill-mask-online-endpoint.ipynb index 07d199bc73..aea12d27e9 100644 --- a/sdk/python/foundation-models/system/inference/fill-mask/fill-mask-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/fill-mask/fill-mask-online-endpoint.ipynb @@ -311,9 +311,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/image-classification/image-classification-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/image-classification/image-classification-batch-endpoint.ipynb index 4d737dcc7d..a90eeb30d6 100644 --- a/sdk/python/foundation-models/system/inference/image-classification/image-classification-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/image-classification/image-classification-batch-endpoint.ipynb @@ -486,6 +486,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/image-classification/image-classification-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/image-classification/image-classification-online-endpoint.ipynb index 2a47ef5f76..acb78716ea 100644 --- a/sdk/python/foundation-models/system/inference/image-classification/image-classification-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/image-classification/image-classification-online-endpoint.ipynb @@ -365,6 +365,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/image-embeddings/image-embeddings-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/image-embeddings/image-embeddings-batch-endpoint.ipynb index 632d1dfd44..4e09c9dac9 100644 --- a/sdk/python/foundation-models/system/inference/image-embeddings/image-embeddings-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/image-embeddings/image-embeddings-batch-endpoint.ipynb @@ -473,6 +473,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/image-embeddings/image-embeddings-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/image-embeddings/image-embeddings-online-endpoint.ipynb index 47b57af0b6..f2ecbe9c71 100644 --- a/sdk/python/foundation-models/system/inference/image-embeddings/image-embeddings-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/image-embeddings/image-embeddings-online-endpoint.ipynb @@ -324,7 +324,13 @@ ] } ], - "metadata": {}, + "metadata": { + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + } + }, "nbformat": 4, "nbformat_minor": 2 } diff --git a/sdk/python/foundation-models/system/inference/image-instance-segmentation/image-instance-segmentation-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/image-instance-segmentation/image-instance-segmentation-batch-endpoint.ipynb index 624a988816..d599fac480 100644 --- a/sdk/python/foundation-models/system/inference/image-instance-segmentation/image-instance-segmentation-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/image-instance-segmentation/image-instance-segmentation-batch-endpoint.ipynb @@ -527,6 +527,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/image-instance-segmentation/image-instance-segmentation-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/image-instance-segmentation/image-instance-segmentation-online-endpoint.ipynb index 2198c1cc6f..e99d99ea46 100644 --- a/sdk/python/foundation-models/system/inference/image-instance-segmentation/image-instance-segmentation-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/image-instance-segmentation/image-instance-segmentation-online-endpoint.ipynb @@ -356,6 +356,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/image-object-detection/image-object-detection-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/image-object-detection/image-object-detection-batch-endpoint.ipynb index 93f946a615..5a25336bf6 100644 --- a/sdk/python/foundation-models/system/inference/image-object-detection/image-object-detection-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/image-object-detection/image-object-detection-batch-endpoint.ipynb @@ -527,6 +527,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/image-object-detection/image-object-detection-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/image-object-detection/image-object-detection-online-endpoint.ipynb index d62e0f57c5..e6dee264b8 100644 --- a/sdk/python/foundation-models/system/inference/image-object-detection/image-object-detection-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/image-object-detection/image-object-detection-online-endpoint.ipynb @@ -357,6 +357,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/image-text-embeddings/image-text-embeddings-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/image-text-embeddings/image-text-embeddings-batch-endpoint.ipynb index b610478e7b..1e37c21d6b 100644 --- a/sdk/python/foundation-models/system/inference/image-text-embeddings/image-text-embeddings-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/image-text-embeddings/image-text-embeddings-batch-endpoint.ipynb @@ -716,9 +716,9 @@ ], "metadata": { "kernelspec": { - "display_name": "rc_133", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/image-text-embeddings/image-text-embeddings-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/image-text-embeddings/image-text-embeddings-online-endpoint.ipynb index 2871662e7e..526697fcb3 100644 --- a/sdk/python/foundation-models/system/inference/image-text-embeddings/image-text-embeddings-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/image-text-embeddings/image-text-embeddings-online-endpoint.ipynb @@ -462,9 +462,9 @@ ], "metadata": { "kernelspec": { - "display_name": "rc_133", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/image-text-embeddings/text-to-image-retrieval.ipynb b/sdk/python/foundation-models/system/inference/image-text-embeddings/text-to-image-retrieval.ipynb index 8d95dab697..ced0e6968e 100644 --- a/sdk/python/foundation-models/system/inference/image-text-embeddings/text-to-image-retrieval.ipynb +++ b/sdk/python/foundation-models/system/inference/image-text-embeddings/text-to-image-retrieval.ipynb @@ -576,9 +576,9 @@ ], "metadata": { "kernelspec": { - "display_name": "rc_133", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/image-to-text/image-to-text-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/image-to-text/image-to-text-batch-endpoint.ipynb index c767466d37..05ef956421 100644 --- a/sdk/python/foundation-models/system/inference/image-to-text/image-to-text-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/image-to-text/image-to-text-batch-endpoint.ipynb @@ -456,9 +456,9 @@ ], "metadata": { "kernelspec": { - "display_name": "sdkv2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "sdkv2" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/image-to-text/image-to-text-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/image-to-text/image-to-text-online-endpoint.ipynb index 23a2a46d10..83e8e5f193 100644 --- a/sdk/python/foundation-models/system/inference/image-to-text/image-to-text-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/image-to-text/image-to-text-online-endpoint.ipynb @@ -329,9 +329,9 @@ ], "metadata": { "kernelspec": { - "display_name": "sdkv2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "sdkv2" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/mask-generation/mask-generation-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/mask-generation/mask-generation-batch-endpoint.ipynb index 4d2011cdf1..3a056df5a0 100644 --- a/sdk/python/foundation-models/system/inference/mask-generation/mask-generation-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/mask-generation/mask-generation-batch-endpoint.ipynb @@ -492,9 +492,9 @@ ], "metadata": { "kernelspec": { - "display_name": "dri", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "dri" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/mask-generation/mask-generation-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/mask-generation/mask-generation-online-endpoint.ipynb index a9df8b9c59..f3641bfdc2 100644 --- a/sdk/python/foundation-models/system/inference/mask-generation/mask-generation-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/mask-generation/mask-generation-online-endpoint.ipynb @@ -470,9 +470,9 @@ ], "metadata": { "kernelspec": { - "display_name": "dri", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "dri" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b-instruct/online-endpoint-chat-completions-inference.ipynb b/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b-instruct/online-endpoint-chat-completions-inference.ipynb index f96027e1b0..fa8981a8e0 100644 --- a/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b-instruct/online-endpoint-chat-completions-inference.ipynb +++ b/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b-instruct/online-endpoint-chat-completions-inference.ipynb @@ -215,9 +215,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b/online-endpoint-completions-inference.ipynb b/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b/online-endpoint-completions-inference.ipynb index 65a4588572..af2585c70c 100644 --- a/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b/online-endpoint-completions-inference.ipynb +++ b/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-70b/online-endpoint-completions-inference.ipynb @@ -215,9 +215,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b-instruct/online-endpoint-chat-completions-inference.ipynb b/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b-instruct/online-endpoint-chat-completions-inference.ipynb index 100aeb641a..247105b229 100644 --- a/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b-instruct/online-endpoint-chat-completions-inference.ipynb +++ b/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b-instruct/online-endpoint-chat-completions-inference.ipynb @@ -215,9 +215,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b/online-endpoint-completions-inference.ipynb b/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b/online-endpoint-completions-inference.ipynb index ddb8743f3a..ef939211f2 100644 --- a/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b/online-endpoint-completions-inference.ipynb +++ b/sdk/python/foundation-models/system/inference/meta-llama-3/meta-llama-3-8b/online-endpoint-completions-inference.ipynb @@ -215,9 +215,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/question-answering/question-answering-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/question-answering/question-answering-batch-endpoint.ipynb index d4a1b585f8..bb89ed91ec 100644 --- a/sdk/python/foundation-models/system/inference/question-answering/question-answering-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/question-answering/question-answering-batch-endpoint.ipynb @@ -498,9 +498,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/question-answering/question-answering-online-endpoint-oss.ipynb b/sdk/python/foundation-models/system/inference/question-answering/question-answering-online-endpoint-oss.ipynb index acdea6d115..95d89d42a5 100644 --- a/sdk/python/foundation-models/system/inference/question-answering/question-answering-online-endpoint-oss.ipynb +++ b/sdk/python/foundation-models/system/inference/question-answering/question-answering-online-endpoint-oss.ipynb @@ -301,9 +301,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/question-answering/question-answering-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/question-answering/question-answering-online-endpoint.ipynb index ede836fac3..e88af3e14b 100644 --- a/sdk/python/foundation-models/system/inference/question-answering/question-answering-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/question-answering/question-answering-online-endpoint.ipynb @@ -289,9 +289,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/summarization/summarization-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/summarization/summarization-batch-endpoint.ipynb index 14381a3e52..724c19b8e7 100644 --- a/sdk/python/foundation-models/system/inference/summarization/summarization-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/summarization/summarization-batch-endpoint.ipynb @@ -522,9 +522,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/summarization/summarization-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/summarization/summarization-online-endpoint.ipynb index 253892dc2a..b15a023683 100644 --- a/sdk/python/foundation-models/system/inference/summarization/summarization-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/summarization/summarization-online-endpoint.ipynb @@ -285,9 +285,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/text-classification/entailment-contradiction-batch.ipynb b/sdk/python/foundation-models/system/inference/text-classification/entailment-contradiction-batch.ipynb index 1d9aa5b19a..8e77e56fb4 100644 --- a/sdk/python/foundation-models/system/inference/text-classification/entailment-contradiction-batch.ipynb +++ b/sdk/python/foundation-models/system/inference/text-classification/entailment-contradiction-batch.ipynb @@ -624,9 +624,9 @@ "name": "amlv2" }, "kernelspec": { - "display_name": "hf", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/text-classification/text-classification-online-endpoint-oss.ipynb b/sdk/python/foundation-models/system/inference/text-classification/text-classification-online-endpoint-oss.ipynb index 854694e479..b424fd6dbe 100644 --- a/sdk/python/foundation-models/system/inference/text-classification/text-classification-online-endpoint-oss.ipynb +++ b/sdk/python/foundation-models/system/inference/text-classification/text-classification-online-endpoint-oss.ipynb @@ -325,9 +325,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/text-classification/text-classification-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/text-classification/text-classification-online-endpoint.ipynb index 028030e98e..27b73a1784 100644 --- a/sdk/python/foundation-models/system/inference/text-classification/text-classification-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/text-classification/text-classification-online-endpoint.ipynb @@ -300,9 +300,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/text-generation/aacs-prepare-uai.ipynb b/sdk/python/foundation-models/system/inference/text-generation/aacs-prepare-uai.ipynb index 666c6786a1..c6e4f5bdc4 100644 --- a/sdk/python/foundation-models/system/inference/text-generation/aacs-prepare-uai.ipynb +++ b/sdk/python/foundation-models/system/inference/text-generation/aacs-prepare-uai.ipynb @@ -426,9 +426,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/text-generation/falcon-safe-online-deployment.ipynb b/sdk/python/foundation-models/system/inference/text-generation/falcon-safe-online-deployment.ipynb index dad51194f6..8c115501b1 100644 --- a/sdk/python/foundation-models/system/inference/text-generation/falcon-safe-online-deployment.ipynb +++ b/sdk/python/foundation-models/system/inference/text-generation/falcon-safe-online-deployment.ipynb @@ -628,7 +628,7 @@ "name": "python310-sdkv2" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/foundation-models/system/inference/text-generation/llama-safe-batch-deployment.ipynb b/sdk/python/foundation-models/system/inference/text-generation/llama-safe-batch-deployment.ipynb index f56d5bc0fb..d9f30fe874 100644 --- a/sdk/python/foundation-models/system/inference/text-generation/llama-safe-batch-deployment.ipynb +++ b/sdk/python/foundation-models/system/inference/text-generation/llama-safe-batch-deployment.ipynb @@ -709,9 +709,9 @@ ], "metadata": { "kernelspec": { - "display_name": "env", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/text-generation/llama-safe-online-deployment.ipynb b/sdk/python/foundation-models/system/inference/text-generation/llama-safe-online-deployment.ipynb index ed1143ab1c..852e826c93 100644 --- a/sdk/python/foundation-models/system/inference/text-generation/llama-safe-online-deployment.ipynb +++ b/sdk/python/foundation-models/system/inference/text-generation/llama-safe-online-deployment.ipynb @@ -1,740 +1,740 @@ { - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# How to create an Azure AI Content Safety enabled Llama 2 online endpoint (Preview)\n", - "### This notebook will walk you through the steps to create an __Azure AI Content Safety__ enabled __Llama 2__ online endpoint.\n", - "### This notebook is under preview\n", - "### The steps are:\n", - "1. Create an __Azure AI Content Safety__ resource for moderating the request from user and response from the __Llama 2__ online endpoint.\n", - "2. Create a new __Azure AI Content Safety__ enabled __Llama 2__ online endpoint with a custom score.py which will integrate with the __Azure AI Content Safety__ resource to moderate the response from the __Llama 2__ model and the request from the user, but to make the custom score.py to successfully authenticated to the __Azure AI Content Safety__ resource, is to create a User Assigned Identity (UAI) and assign appropriate roles to the UAI. Then, the custom score.py can obtain the access token of the UAI from the AAD server to access the Azure AI Content Safety resource. Use [this notebook](aacs-prepare-uai.ipynb) to create UAI account for step 3 below" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1. Prerequisites\n", - "#### 1.1 Check List:\n", - "- [x] You have created a new Python virtual environment for this notebook.\n", - "- [x] The identity you are using to execute this notebook(yourself or your VM) need to have the __Contributor__ role on the resource group where the AML Workspace your specified is located, because this notebook will create an Azure AI Content Safety resource using that identity." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 1.2 Assign variables for the workspace and deployment" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# The public registry name contains Llama 2 models\n", - "registry_name = \"azureml-meta\"\n", - "\n", - "# Name of the Llama 2 model to be deployed\n", - "# available_llama_models_text_generation = [\"Llama-2-7b\", \"Llama-2-13b\", \"Llama-2-70b\"]\n", - "# available_llama_models_chat_complete = [\"Llama-2-7b-chat\", \"Llama-2-13b-chat\", \"Llama-2-70b-chat\"]\n", - "model_name = \"Llama-2-7b\"\n", - "\n", - "endpoint_name = f\"{model_name}-test-ep\" # Replace with your endpoint name\n", - "deployment_name = \"llama\" # Replace with your deployment name, lower case only!!!\n", - "sku_name = \"STANDARD_NC4AS_T4_V3\" # Name of the sku(instance type) Check the model-list(can be found in the parent folder(inference)) to get the most optimal sku for your model (Default: Standard_DS2_v2)\n", - "\n", - "# The severity level that will trigger response be blocked\n", - "# Please reference Azure AI content documentation for more details\n", - "# https://learn.microsoft.com/en-us/azure/cognitive-services/content-safety/concepts/harm-categories\n", - "content_severity_threshold = \"2\"\n", - "\n", - "# UAI to be used for endpoint if you choose to use UAI as authentication method\n", - "uai_name = \"\" # default to \"aacs-uai\" in prepare uai notebook" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 1.3 Install Dependencies(as needed)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# uncomment the following lines to install the required packages\n", - "# %pip install azure-identity==1.13.0\n", - "# %pip install azure-mgmt-cognitiveservices==13.4.0\n", - "# %pip install azure-ai-ml>=1.23.1\n", - "# %pip install azure-mgmt-msi==7.0.0\n", - "# %pip install azure-mgmt-authorization==3.0.0" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 1.4 Get credential" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", - "\n", - "try:\n", - " credential = DefaultAzureCredential()\n", - " # Check if given credential can get token successfully.\n", - " credential.get_token(\"https://management.azure.com/.default\")\n", - "except Exception as ex:\n", - " # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work\n", - " credential = InteractiveBrowserCredential()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 1.5 Configure workspace " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.ml import MLClient\n", - "\n", - "try:\n", - " ml_client = MLClient.from_config(credential=credential)\n", - "except Exception as ex:\n", - " # enter details of your AML workspace\n", - " subscription_id = \"\"\n", - " resource_group = \"\"\n", - " workspace = \"\"\n", - "\n", - " # get a handle to the workspace\n", - " ml_client = MLClient(credential, subscription_id, resource_group, workspace)\n", - "\n", - "subscription_id = ml_client.subscription_id\n", - "resource_group = ml_client.resource_group_name\n", - "workspace = ml_client.workspace_name\n", - "\n", - "print(f\"Connected to workspace {workspace}\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 1.6 Assign variables for Azure Content Safety\n", - "Currently, Azure AI Content Safety is in a limited set of regions:\n", - "\n", - "\n", - "__NOTE__: before you choose the region to deploy the Azure AI Content Safety, please be aware that your data will be transferred to the region you choose and by selecting a region outside your current location, you may be allowing the transmission of your data to regions outside your jurisdiction. It is important to note that data protection and privacy laws may vary between jurisdictions. Before proceeding, we strongly advise you to familiarize yourself with the local laws and regulations governing data transfer and ensure that you are legally permitted to transmit your data to an overseas location for processing. By continuing with the selection of a different region, you acknowledge that you have understood and accepted any potential risks associated with such data transmission. Please proceed with caution." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.mgmt.cognitiveservices import CognitiveServicesManagementClient\n", - "\n", - "acs_client = CognitiveServicesManagementClient(credential, subscription_id)\n", - "\n", - "\n", - "# settings for the Azure AI Content Safety resource\n", - "# we will choose existing AACS resource if it exists, otherwise create a new one\n", - "# name of azure ai content safety resource, has to be unique\n", - "import time\n", - "\n", - "aacs_name = f\"{endpoint_name}-aacs-{str(time.time()).replace('.','')}\"\n", - "available_aacs_locations = [\"east us\", \"west europe\"]\n", - "\n", - "# create a new Cognitive Services Account\n", - "kind = \"ContentSafety\"\n", - "aacs_sku_name = \"S0\"\n", - "aacs_location = available_aacs_locations[0]\n", - "\n", - "\n", - "print(\"Available SKUs:\")\n", - "aacs_skus = acs_client.resource_skus.list()\n", - "print(\"SKU Name\\tSKU Tier\\tLocations\")\n", - "for sku in aacs_skus:\n", - " if sku.kind == \"ContentSafety\":\n", - " locations = \",\".join(sku.locations)\n", - " print(sku.name + \"\\t\" + sku.tier + \"\\t\" + locations)\n", - "\n", - "print(\n", - " f\"Choose a new Azure AI Content Safety resource in {aacs_location} with SKU {aacs_sku_name}\"\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2. Create Azure AI Content Safety" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.mgmt.cognitiveservices.models import Account, Sku, AccountProperties\n", - "\n", - "\n", - "parameters = Account(\n", - " sku=Sku(name=aacs_sku_name),\n", - " kind=kind,\n", - " location=aacs_location,\n", - " properties=AccountProperties(\n", - " custom_sub_domain_name=aacs_name, public_network_access=\"Enabled\"\n", - " ),\n", - ")\n", - "# How many seconds to wait between checking the status of an async operation.\n", - "wait_time = 10\n", - "\n", - "\n", - "def find_acs(accounts):\n", - " return next(\n", - " x\n", - " for x in accounts\n", - " if x.kind == \"ContentSafety\"\n", - " and x.location == aacs_location\n", - " and x.sku.name == aacs_sku_name\n", - " )\n", - "\n", - "\n", - "try:\n", - " # check if AACS exists\n", - " aacs = acs_client.accounts.get(resource_group, aacs_name)\n", - " print(f\"Found existing Azure AI content safety Account {aacs.name}.\")\n", - "except:\n", - " try:\n", - " # check if there is an existing AACS resource within same resource group\n", - " aacs = find_acs(acs_client.accounts.list_by_resource_group(resource_group))\n", - " print(\n", - " f\"Found existing Azure AI content safety Account {aacs.name} in resource group {resource_group}.\"\n", - " )\n", - " except:\n", - " print(f\"Creating Azure AI content safety Account {aacs_name}.\")\n", - " acs_client.accounts.begin_create(resource_group, aacs_name, parameters).wait()\n", - " print(\"Resource created.\")\n", - " aacs = acs_client.accounts.get(resource_group, aacs_name)\n", - "\n", - "\n", - "aacs_endpoint = aacs.properties.endpoint\n", - "aacs_resource_id = aacs.id\n", - "aacs_name = aacs.name\n", - "print(\n", - " f\"AACS name is {aacs.name}, use this name in UAI preparation notebook to create UAI.\"\n", - ")\n", - "print(f\"AACS endpoint is {aacs_endpoint}\")\n", - "print(f\"AACS ResourceId is {aacs_resource_id}\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3. Create Azure AI Content Safety enabled Llama 2 online endpoint" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 3.1 Check if Llama 2 model is available in the AML registry." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "reg_client = MLClient(\n", - " credential,\n", - " subscription_id=subscription_id,\n", - " resource_group_name=resource_group,\n", - " registry_name=registry_name,\n", - ")\n", - "version_list = list(\n", - " reg_client.models.list(model_name)\n", - ") # list available versions of the model\n", - "llama_model = None\n", - "\n", - "# If specific inference environments are tagged for the model\n", - "inference_envs_exist = False\n", - "\n", - "if len(version_list) == 0:\n", - " raise Exception(f\"No model named {model_name} found in registry\")\n", - "else:\n", - " model_version = version_list[0].version\n", - " llama_model = reg_client.models.get(model_name, model_version)\n", - " if (\n", - " \"inference_supported_envs\" in llama_model.tags\n", - " and len(llama_model.tags[\"inference_supported_envs\"]) >= 1\n", - " ):\n", - " inference_envs_exist = True\n", - " print(\n", - " f\"Using model name: {llama_model.name}, version: {llama_model.version}, id: {llama_model.id} for inferencing\"\n", - " )" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 3.2 Check if UAI is used" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "uai_id = \"\"\n", - "uai_client_id = \"\"\n", - "if uai_name != \"\":\n", - " from azure.mgmt.msi import ManagedServiceIdentityClient\n", - " from azure.mgmt.msi.models import Identity\n", - "\n", - " msi_client = ManagedServiceIdentityClient(\n", - " subscription_id=subscription_id,\n", - " credential=credential,\n", - " )\n", - " uai_resource = msi_client.user_assigned_identities.get(resource_group, uai_name)\n", - " uai_id = uai_resource.id\n", - " uai_client_id = uai_resource.client_id" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 3.3 Create Llama 2 online endpoint\n", - "This step may take a few minutes." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Create endpoint" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.ml.entities import (\n", - " ManagedOnlineEndpoint,\n", - " IdentityConfiguration,\n", - " ManagedIdentityConfiguration,\n", - ")\n", - "\n", - "# Check if the endpoint already exists in the workspace\n", - "try:\n", - " endpoint = ml_client.online_endpoints.get(endpoint_name)\n", - " print(\"---Endpoint already exists---\")\n", - "except:\n", - " # Create an online endpoint if it doesn't exist\n", - "\n", - " # Define the endpoint\n", - " endpoint = ManagedOnlineEndpoint(\n", - " name=endpoint_name,\n", - " description=\"Test endpoint for model\",\n", - " identity=IdentityConfiguration(\n", - " type=\"user_assigned\",\n", - " user_assigned_identities=[ManagedIdentityConfiguration(resource_id=uai_id)],\n", - " )\n", - " if uai_id != \"\"\n", - " else None,\n", - " )\n", - "\n", - " # Trigger the endpoint creation\n", - " try:\n", - " ml_client.begin_create_or_update(endpoint).wait()\n", - " print(\"\\n---Endpoint created successfully---\\n\")\n", - " except Exception as err:\n", - " raise RuntimeError(\n", - " f\"Endpoint creation failed. Detailed Response:\\n{err}\"\n", - " ) from err" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 3.4 Setup Deployment Parameters\n", - "\n", - "We utilize an optimized __foundation-model-inference__ container for model scoring. This container is designed to deliver high throughput and low latency. In this section, we introduce several environment variables that can be adjusted to customize a deployment for either high throughput or low latency scenarios.\n", - "\n", - "- __ENGINE_NAME__: Used to choose the inferencing framework to use in the scoring script. For Llama-2 models, if ENGINE_NAME = 'mii' the container will inference with the new [DeepSpeed-FastGen](https://github.com/microsoft/DeepSpeed/tree/master/blogs/deepspeed-fastgen). Alternatively if ENGINE_NAME = 'vllm' the container will inference with [vLLM](https://vllm.readthedocs.io/en/latest/), which is also the default.\n", - "- __WORKER_COUNT__: The number of workers to use for inferencing. This is used as a proxy for the number of concurrent requests that the server should handle.\n", - "- __TENSOR_PARALLEL__: The number of GPUs to use for tensor parallelism.\n", - "- __NUM_REPLICAS__: The number of model instances to load for the deployment. This is used to increase throughput by loading multiple models on multiple GPUs, if the model is small enough to fit.\n", - "\n", - "`NUM_REPLICAS` and `TENSOR_PARALLEL` work hand-in-hand to determine the most optimal configuration to increase the throughput for the deployment without degrading too much on the latency. The total number of GPUs used for inference will be `NUM_REPLICAS` * `TENSOR_PARALLEL`. For example, if `NUM_REPLICAS` = 2 and `TENSOR_PARALLEL` = 2, then 4 GPUs will be used for inference.\n", - "\n", - "Ensure that the model you are deploying is small enough to fit on the number of GPUs you are using, specified by `TENSOR_PARALLEL`. For instance, if there are 4 GPUs available, and `TENSOR_PARALLEL` = 2, then the model must be small enough to fit on 2 GPUs. If the model is too large, then the deployment will fail. \n", - "\n", - "__NOTE__: \n", - "- `NUM_REPLICAS` is currently only supported by the vLLM engine.\n", - "- DeepSpeed MII Engine is only supported on A100 / H100 GPUs.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "REQUEST_TIMEOUT_MS = 90000 # the timeout for each request in milliseconds\n", - "MAX_CONCURRENT_REQUESTS = (\n", - " 128 # the maximum number of concurrent requests supported by the endpoint\n", - ")\n", - "\n", - "acs_env_vars = {\n", - " \"CONTENT_SAFETY_ACCOUNT_NAME\": aacs_name,\n", - " \"CONTENT_SAFETY_ENDPOINT\": aacs_endpoint,\n", - " \"CONTENT_SAFETY_THRESHOLD\": content_severity_threshold,\n", - " \"SUBSCRIPTION_ID\": subscription_id,\n", - " \"RESOURCE_GROUP_NAME\": resource_group,\n", - " \"UAI_CLIENT_ID\": uai_client_id,\n", - "}\n", - "\n", - "fm_container_default_env_vars = {\n", - " \"WORKER_COUNT\": MAX_CONCURRENT_REQUESTS,\n", - " \"TENSOR_PARALLEL\": 2,\n", - " \"NUM_REPLICAS\": 2,\n", - "}\n", - "\n", - "deployment_env_vars = {**fm_container_default_env_vars, **acs_env_vars}\n", - "\n", - "# Uncomment the following lines to use DeepSpeed FastGen engine (experimental)\n", - "# mii_fastgen_env_vars = {\n", - "# \"ENGINE_NAME\": \"mii\",\n", - "# \"WORKER_COUNT\": MAX_CONCURRENT_REQUESTS,\n", - "# }\n", - "# deployment_env_vars = {**mii_fastgen_env_vars, **acs_env_vars}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 3.5 Deploy Llama 2 model\n", - "This step may take a few minutes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.ml.entities import (\n", - " OnlineRequestSettings,\n", - " CodeConfiguration,\n", - " ManagedOnlineDeployment,\n", - " ProbeSettings,\n", - ")\n", - "\n", - "# For inference environments vLLM and DS MII, the scoring script is baked into the container\n", - "code_configuration = (\n", - " CodeConfiguration(code=\"./llama-files/score/default/\", scoring_script=\"score.py\")\n", - " if not inference_envs_exist\n", - " else None\n", - ")\n", - "\n", - "deployment = ManagedOnlineDeployment(\n", - " name=deployment_name,\n", - " endpoint_name=endpoint_name,\n", - " model=llama_model.id,\n", - " instance_type=sku_name,\n", - " instance_count=1,\n", - " code_configuration=code_configuration,\n", - " environment_variables=deployment_env_vars,\n", - " request_settings=OnlineRequestSettings(\n", - " request_timeout_ms=REQUEST_TIMEOUT_MS,\n", - " max_concurrent_requests_per_instance=MAX_CONCURRENT_REQUESTS,\n", - " ),\n", - " liveness_probe=ProbeSettings(\n", - " failure_threshold=30,\n", - " success_threshold=1,\n", - " period=100,\n", - " initial_delay=500,\n", - " ),\n", - " readiness_probe=ProbeSettings(\n", - " failure_threshold=30,\n", - " success_threshold=1,\n", - " period=100,\n", - " initial_delay=500,\n", - " ),\n", - ")\n", - "\n", - "# Trigger the deployment creation\n", - "try:\n", - " ml_client.begin_create_or_update(deployment).wait()\n", - " print(\"\\n---Deployment created successfully---\\n\")\n", - "except Exception as err:\n", - " raise RuntimeError(\n", - " f\"Deployment creation failed. Detailed Response:\\n{err}\"\n", - " ) from err" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4. Test the Safety Enabled Llama 2 online endpoint." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "test_src_dir = \"./safety-llama-test\"\n", - "os.makedirs(test_src_dir, exist_ok=True)\n", - "print(f\"test script directory: {test_src_dir}\")\n", - "sample_data = os.path.join(test_src_dir, \"sample-request.json\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### Choose request from following 4 cells based on the Llama 2 model type you chosen and the scenario you want to test" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "## For text-generation models (without -chat suffix)\n", - "## Successful response\n", - "\n", - "import json\n", - "\n", - "with open(sample_data, \"w\") as f:\n", - " json.dump(\n", - " {\n", - " \"input_data\": {\n", - " \"input_string\": [\n", - " \"Hello\",\n", - " \"My name is John and I have a dog.\",\n", - " ],\n", - " \"parameters\": {\n", - " \"temperature\": 0.6,\n", - " \"top_p\": 0.6,\n", - " \"max_new_tokens\": 256,\n", - " \"do_sample\": True,\n", - " },\n", - " }\n", - " },\n", - " f,\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "## For text-generation models (without -chat suffix)\n", - "## Blocked request/response due to hateful content\n", - "\n", - "import json\n", - "\n", - "with open(sample_data, \"w\") as f:\n", - " json.dump(\n", - " {\n", - " \"input_data\": {\n", - " \"input_string\": [\n", - " \"The dog was given a eutanasa injection due to their severed leg bleding profusely from deep lacarations to the lower extremities, exposing tisssue and nerve.\", # This line contains hateful message and endpoint will return empty string as response\n", - " ],\n", - " \"parameters\": {\n", - " \"temperature\": 0.6,\n", - " \"top_p\": 0.6,\n", - " \"max_new_tokens\": 256,\n", - " \"do_sample\": True,\n", - " },\n", - " }\n", - " },\n", - " f,\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "## For chat-complete models (with \"-chat\" suffix)\n", - "## Successful request/response\n", - "\n", - "import json\n", - "\n", - "with open(sample_data, \"w\") as f:\n", - " json.dump(\n", - " {\n", - " \"input_data\": {\n", - " \"input_string\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What is the tallest building in the world?\",\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"As of 2021, the Burj Khalifa in Dubai, United Arab Emirates is the tallest building in the world, standing at a height of 828 meters (2,722 feet). It was completed in 2010 and has 163 floors. The Burj Khalifa is not only the tallest building in the world but also holds several other records, such as the highest occupied floor, highest outdoor observation deck, elevator with the longest travel distance, and the tallest freestanding structure in the world.\",\n", - " },\n", - " {\"role\": \"user\", \"content\": \"and in Africa?\"},\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"In Africa, the tallest building is the Carlton Centre, located in Johannesburg, South Africa. It stands at a height of 50 floors and 223 meters (730 feet). The CarltonDefault Centre was completed in 1973 and was the tallest building in Africa for many years until the construction of the Leonardo, a 55-story skyscraper in Sandton, Johannesburg, which was completed in 2019 and stands at a height of 230 meters (755 feet). Other notable tall buildings in Africa include the Ponte City Apartments in Johannesburg, the John Hancock Center in Lagos, Nigeria, and the Alpha II Building in Abidjan, Ivory Coast\",\n", - " },\n", - " {\"role\": \"user\", \"content\": \"and in Europe?\"},\n", - " ],\n", - " \"parameters\": {\n", - " \"temperature\": 0.9,\n", - " \"top_p\": 0.6,\n", - " \"do_sample\": True,\n", - " \"max_new_tokens\": 100,\n", - " },\n", - " }\n", - " },\n", - " f,\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "## For chat-complete models (with \"-chat\" suffix)\n", - "## Blocked request due to hateful content\n", - "\n", - "import json\n", - "\n", - "with open(sample_data, \"w\") as f:\n", - " json.dump(\n", - " {\n", - " \"input_data\": {\n", - " \"input_string\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"A 51-year-old man was found dead in his car. There were blood stains on the dashboard and windscreen. At autopsy, a deep, oblique, long incised injury was found on the front of the neck. It turns out that he died by suicide.\",\n", - " },\n", - " ],\n", - " \"parameters\": {\n", - " \"temperature\": 0.9,\n", - " \"top_p\": 0.6,\n", - " \"do_sample\": True,\n", - " \"max_new_tokens\": 100,\n", - " },\n", - " }\n", - " },\n", - " f,\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ml_client.online_endpoints.invoke(\n", - " endpoint_name=endpoint_name,\n", - " deployment_name=deployment_name,\n", - " request_file=sample_data,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernel_info": { - "name": "python3" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - }, - "microsoft": { - "ms_spell_check": { - "ms_spell_check_language": "en" - } - }, - "nteract": { - "version": "nteract-front-end@1.0.0" - } - }, - "nbformat": 4, - "nbformat_minor": 1 + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# How to create an Azure AI Content Safety enabled Llama 2 online endpoint (Preview)\n", + "### This notebook will walk you through the steps to create an __Azure AI Content Safety__ enabled __Llama 2__ online endpoint.\n", + "### This notebook is under preview\n", + "### The steps are:\n", + "1. Create an __Azure AI Content Safety__ resource for moderating the request from user and response from the __Llama 2__ online endpoint.\n", + "2. Create a new __Azure AI Content Safety__ enabled __Llama 2__ online endpoint with a custom score.py which will integrate with the __Azure AI Content Safety__ resource to moderate the response from the __Llama 2__ model and the request from the user, but to make the custom score.py to successfully authenticated to the __Azure AI Content Safety__ resource, is to create a User Assigned Identity (UAI) and assign appropriate roles to the UAI. Then, the custom score.py can obtain the access token of the UAI from the AAD server to access the Azure AI Content Safety resource. Use [this notebook](aacs-prepare-uai.ipynb) to create UAI account for step 3 below" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Prerequisites\n", + "#### 1.1 Check List:\n", + "- [x] You have created a new Python virtual environment for this notebook.\n", + "- [x] The identity you are using to execute this notebook(yourself or your VM) need to have the __Contributor__ role on the resource group where the AML Workspace your specified is located, because this notebook will create an Azure AI Content Safety resource using that identity." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.2 Assign variables for the workspace and deployment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The public registry name contains Llama 2 models\n", + "registry_name = \"azureml-meta\"\n", + "\n", + "# Name of the Llama 2 model to be deployed\n", + "# available_llama_models_text_generation = [\"Llama-2-7b\", \"Llama-2-13b\", \"Llama-2-70b\"]\n", + "# available_llama_models_chat_complete = [\"Llama-2-7b-chat\", \"Llama-2-13b-chat\", \"Llama-2-70b-chat\"]\n", + "model_name = \"Llama-2-7b\"\n", + "\n", + "endpoint_name = f\"{model_name}-test-ep\" # Replace with your endpoint name\n", + "deployment_name = \"llama\" # Replace with your deployment name, lower case only!!!\n", + "sku_name = \"STANDARD_NC4AS_T4_V3\" # Name of the sku(instance type) Check the model-list(can be found in the parent folder(inference)) to get the most optimal sku for your model (Default: Standard_DS2_v2)\n", + "\n", + "# The severity level that will trigger response be blocked\n", + "# Please reference Azure AI content documentation for more details\n", + "# https://learn.microsoft.com/en-us/azure/cognitive-services/content-safety/concepts/harm-categories\n", + "content_severity_threshold = \"2\"\n", + "\n", + "# UAI to be used for endpoint if you choose to use UAI as authentication method\n", + "uai_name = \"\" # default to \"aacs-uai\" in prepare uai notebook" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.3 Install Dependencies(as needed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# uncomment the following lines to install the required packages\n", + "# %pip install azure-identity==1.13.0\n", + "# %pip install azure-mgmt-cognitiveservices==13.4.0\n", + "# %pip install azure-ai-ml>=1.23.1\n", + "# %pip install azure-mgmt-msi==7.0.0\n", + "# %pip install azure-mgmt-authorization==3.0.0" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.4 Get credential" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", + "\n", + "try:\n", + " credential = DefaultAzureCredential()\n", + " # Check if given credential can get token successfully.\n", + " credential.get_token(\"https://management.azure.com/.default\")\n", + "except Exception as ex:\n", + " # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work\n", + " credential = InteractiveBrowserCredential()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.5 Configure workspace " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.ml import MLClient\n", + "\n", + "try:\n", + " ml_client = MLClient.from_config(credential=credential)\n", + "except Exception as ex:\n", + " # enter details of your AML workspace\n", + " subscription_id = \"\"\n", + " resource_group = \"\"\n", + " workspace = \"\"\n", + "\n", + " # get a handle to the workspace\n", + " ml_client = MLClient(credential, subscription_id, resource_group, workspace)\n", + "\n", + "subscription_id = ml_client.subscription_id\n", + "resource_group = ml_client.resource_group_name\n", + "workspace = ml_client.workspace_name\n", + "\n", + "print(f\"Connected to workspace {workspace}\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.6 Assign variables for Azure Content Safety\n", + "Currently, Azure AI Content Safety is in a limited set of regions:\n", + "\n", + "\n", + "__NOTE__: before you choose the region to deploy the Azure AI Content Safety, please be aware that your data will be transferred to the region you choose and by selecting a region outside your current location, you may be allowing the transmission of your data to regions outside your jurisdiction. It is important to note that data protection and privacy laws may vary between jurisdictions. Before proceeding, we strongly advise you to familiarize yourself with the local laws and regulations governing data transfer and ensure that you are legally permitted to transmit your data to an overseas location for processing. By continuing with the selection of a different region, you acknowledge that you have understood and accepted any potential risks associated with such data transmission. Please proceed with caution." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.mgmt.cognitiveservices import CognitiveServicesManagementClient\n", + "\n", + "acs_client = CognitiveServicesManagementClient(credential, subscription_id)\n", + "\n", + "\n", + "# settings for the Azure AI Content Safety resource\n", + "# we will choose existing AACS resource if it exists, otherwise create a new one\n", + "# name of azure ai content safety resource, has to be unique\n", + "import time\n", + "\n", + "aacs_name = f\"{endpoint_name}-aacs-{str(time.time()).replace('.','')}\"\n", + "available_aacs_locations = [\"east us\", \"west europe\"]\n", + "\n", + "# create a new Cognitive Services Account\n", + "kind = \"ContentSafety\"\n", + "aacs_sku_name = \"S0\"\n", + "aacs_location = available_aacs_locations[0]\n", + "\n", + "\n", + "print(\"Available SKUs:\")\n", + "aacs_skus = acs_client.resource_skus.list()\n", + "print(\"SKU Name\\tSKU Tier\\tLocations\")\n", + "for sku in aacs_skus:\n", + " if sku.kind == \"ContentSafety\":\n", + " locations = \",\".join(sku.locations)\n", + " print(sku.name + \"\\t\" + sku.tier + \"\\t\" + locations)\n", + "\n", + "print(\n", + " f\"Choose a new Azure AI Content Safety resource in {aacs_location} with SKU {aacs_sku_name}\"\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Create Azure AI Content Safety" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.mgmt.cognitiveservices.models import Account, Sku, AccountProperties\n", + "\n", + "\n", + "parameters = Account(\n", + " sku=Sku(name=aacs_sku_name),\n", + " kind=kind,\n", + " location=aacs_location,\n", + " properties=AccountProperties(\n", + " custom_sub_domain_name=aacs_name, public_network_access=\"Enabled\"\n", + " ),\n", + ")\n", + "# How many seconds to wait between checking the status of an async operation.\n", + "wait_time = 10\n", + "\n", + "\n", + "def find_acs(accounts):\n", + " return next(\n", + " x\n", + " for x in accounts\n", + " if x.kind == \"ContentSafety\"\n", + " and x.location == aacs_location\n", + " and x.sku.name == aacs_sku_name\n", + " )\n", + "\n", + "\n", + "try:\n", + " # check if AACS exists\n", + " aacs = acs_client.accounts.get(resource_group, aacs_name)\n", + " print(f\"Found existing Azure AI content safety Account {aacs.name}.\")\n", + "except:\n", + " try:\n", + " # check if there is an existing AACS resource within same resource group\n", + " aacs = find_acs(acs_client.accounts.list_by_resource_group(resource_group))\n", + " print(\n", + " f\"Found existing Azure AI content safety Account {aacs.name} in resource group {resource_group}.\"\n", + " )\n", + " except:\n", + " print(f\"Creating Azure AI content safety Account {aacs_name}.\")\n", + " acs_client.accounts.begin_create(resource_group, aacs_name, parameters).wait()\n", + " print(\"Resource created.\")\n", + " aacs = acs_client.accounts.get(resource_group, aacs_name)\n", + "\n", + "\n", + "aacs_endpoint = aacs.properties.endpoint\n", + "aacs_resource_id = aacs.id\n", + "aacs_name = aacs.name\n", + "print(\n", + " f\"AACS name is {aacs.name}, use this name in UAI preparation notebook to create UAI.\"\n", + ")\n", + "print(f\"AACS endpoint is {aacs_endpoint}\")\n", + "print(f\"AACS ResourceId is {aacs_resource_id}\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Create Azure AI Content Safety enabled Llama 2 online endpoint" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 3.1 Check if Llama 2 model is available in the AML registry." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "reg_client = MLClient(\n", + " credential,\n", + " subscription_id=subscription_id,\n", + " resource_group_name=resource_group,\n", + " registry_name=registry_name,\n", + ")\n", + "version_list = list(\n", + " reg_client.models.list(model_name)\n", + ") # list available versions of the model\n", + "llama_model = None\n", + "\n", + "# If specific inference environments are tagged for the model\n", + "inference_envs_exist = False\n", + "\n", + "if len(version_list) == 0:\n", + " raise Exception(f\"No model named {model_name} found in registry\")\n", + "else:\n", + " model_version = version_list[0].version\n", + " llama_model = reg_client.models.get(model_name, model_version)\n", + " if (\n", + " \"inference_supported_envs\" in llama_model.tags\n", + " and len(llama_model.tags[\"inference_supported_envs\"]) >= 1\n", + " ):\n", + " inference_envs_exist = True\n", + " print(\n", + " f\"Using model name: {llama_model.name}, version: {llama_model.version}, id: {llama_model.id} for inferencing\"\n", + " )" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 3.2 Check if UAI is used" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "uai_id = \"\"\n", + "uai_client_id = \"\"\n", + "if uai_name != \"\":\n", + " from azure.mgmt.msi import ManagedServiceIdentityClient\n", + " from azure.mgmt.msi.models import Identity\n", + "\n", + " msi_client = ManagedServiceIdentityClient(\n", + " subscription_id=subscription_id,\n", + " credential=credential,\n", + " )\n", + " uai_resource = msi_client.user_assigned_identities.get(resource_group, uai_name)\n", + " uai_id = uai_resource.id\n", + " uai_client_id = uai_resource.client_id" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 3.3 Create Llama 2 online endpoint\n", + "This step may take a few minutes." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Create endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.ml.entities import (\n", + " ManagedOnlineEndpoint,\n", + " IdentityConfiguration,\n", + " ManagedIdentityConfiguration,\n", + ")\n", + "\n", + "# Check if the endpoint already exists in the workspace\n", + "try:\n", + " endpoint = ml_client.online_endpoints.get(endpoint_name)\n", + " print(\"---Endpoint already exists---\")\n", + "except:\n", + " # Create an online endpoint if it doesn't exist\n", + "\n", + " # Define the endpoint\n", + " endpoint = ManagedOnlineEndpoint(\n", + " name=endpoint_name,\n", + " description=\"Test endpoint for model\",\n", + " identity=IdentityConfiguration(\n", + " type=\"user_assigned\",\n", + " user_assigned_identities=[ManagedIdentityConfiguration(resource_id=uai_id)],\n", + " )\n", + " if uai_id != \"\"\n", + " else None,\n", + " )\n", + "\n", + " # Trigger the endpoint creation\n", + " try:\n", + " ml_client.begin_create_or_update(endpoint).wait()\n", + " print(\"\\n---Endpoint created successfully---\\n\")\n", + " except Exception as err:\n", + " raise RuntimeError(\n", + " f\"Endpoint creation failed. Detailed Response:\\n{err}\"\n", + " ) from err" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 3.4 Setup Deployment Parameters\n", + "\n", + "We utilize an optimized __foundation-model-inference__ container for model scoring. This container is designed to deliver high throughput and low latency. In this section, we introduce several environment variables that can be adjusted to customize a deployment for either high throughput or low latency scenarios.\n", + "\n", + "- __ENGINE_NAME__: Used to choose the inferencing framework to use in the scoring script. For Llama-2 models, if ENGINE_NAME = 'mii' the container will inference with the new [DeepSpeed-FastGen](https://github.com/microsoft/DeepSpeed/tree/master/blogs/deepspeed-fastgen). Alternatively if ENGINE_NAME = 'vllm' the container will inference with [vLLM](https://vllm.readthedocs.io/en/latest/), which is also the default.\n", + "- __WORKER_COUNT__: The number of workers to use for inferencing. This is used as a proxy for the number of concurrent requests that the server should handle.\n", + "- __TENSOR_PARALLEL__: The number of GPUs to use for tensor parallelism.\n", + "- __NUM_REPLICAS__: The number of model instances to load for the deployment. This is used to increase throughput by loading multiple models on multiple GPUs, if the model is small enough to fit.\n", + "\n", + "`NUM_REPLICAS` and `TENSOR_PARALLEL` work hand-in-hand to determine the most optimal configuration to increase the throughput for the deployment without degrading too much on the latency. The total number of GPUs used for inference will be `NUM_REPLICAS` * `TENSOR_PARALLEL`. For example, if `NUM_REPLICAS` = 2 and `TENSOR_PARALLEL` = 2, then 4 GPUs will be used for inference.\n", + "\n", + "Ensure that the model you are deploying is small enough to fit on the number of GPUs you are using, specified by `TENSOR_PARALLEL`. For instance, if there are 4 GPUs available, and `TENSOR_PARALLEL` = 2, then the model must be small enough to fit on 2 GPUs. If the model is too large, then the deployment will fail. \n", + "\n", + "__NOTE__: \n", + "- `NUM_REPLICAS` is currently only supported by the vLLM engine.\n", + "- DeepSpeed MII Engine is only supported on A100 / H100 GPUs.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "REQUEST_TIMEOUT_MS = 90000 # the timeout for each request in milliseconds\n", + "MAX_CONCURRENT_REQUESTS = (\n", + " 128 # the maximum number of concurrent requests supported by the endpoint\n", + ")\n", + "\n", + "acs_env_vars = {\n", + " \"CONTENT_SAFETY_ACCOUNT_NAME\": aacs_name,\n", + " \"CONTENT_SAFETY_ENDPOINT\": aacs_endpoint,\n", + " \"CONTENT_SAFETY_THRESHOLD\": content_severity_threshold,\n", + " \"SUBSCRIPTION_ID\": subscription_id,\n", + " \"RESOURCE_GROUP_NAME\": resource_group,\n", + " \"UAI_CLIENT_ID\": uai_client_id,\n", + "}\n", + "\n", + "fm_container_default_env_vars = {\n", + " \"WORKER_COUNT\": MAX_CONCURRENT_REQUESTS,\n", + " \"TENSOR_PARALLEL\": 2,\n", + " \"NUM_REPLICAS\": 2,\n", + "}\n", + "\n", + "deployment_env_vars = {**fm_container_default_env_vars, **acs_env_vars}\n", + "\n", + "# Uncomment the following lines to use DeepSpeed FastGen engine (experimental)\n", + "# mii_fastgen_env_vars = {\n", + "# \"ENGINE_NAME\": \"mii\",\n", + "# \"WORKER_COUNT\": MAX_CONCURRENT_REQUESTS,\n", + "# }\n", + "# deployment_env_vars = {**mii_fastgen_env_vars, **acs_env_vars}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### 3.5 Deploy Llama 2 model\n", + "This step may take a few minutes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.ml.entities import (\n", + " OnlineRequestSettings,\n", + " CodeConfiguration,\n", + " ManagedOnlineDeployment,\n", + " ProbeSettings,\n", + ")\n", + "\n", + "# For inference environments vLLM and DS MII, the scoring script is baked into the container\n", + "code_configuration = (\n", + " CodeConfiguration(code=\"./llama-files/score/default/\", scoring_script=\"score.py\")\n", + " if not inference_envs_exist\n", + " else None\n", + ")\n", + "\n", + "deployment = ManagedOnlineDeployment(\n", + " name=deployment_name,\n", + " endpoint_name=endpoint_name,\n", + " model=llama_model.id,\n", + " instance_type=sku_name,\n", + " instance_count=1,\n", + " code_configuration=code_configuration,\n", + " environment_variables=deployment_env_vars,\n", + " request_settings=OnlineRequestSettings(\n", + " request_timeout_ms=REQUEST_TIMEOUT_MS,\n", + " max_concurrent_requests_per_instance=MAX_CONCURRENT_REQUESTS,\n", + " ),\n", + " liveness_probe=ProbeSettings(\n", + " failure_threshold=30,\n", + " success_threshold=1,\n", + " period=100,\n", + " initial_delay=500,\n", + " ),\n", + " readiness_probe=ProbeSettings(\n", + " failure_threshold=30,\n", + " success_threshold=1,\n", + " period=100,\n", + " initial_delay=500,\n", + " ),\n", + ")\n", + "\n", + "# Trigger the deployment creation\n", + "try:\n", + " ml_client.begin_create_or_update(deployment).wait()\n", + " print(\"\\n---Deployment created successfully---\\n\")\n", + "except Exception as err:\n", + " raise RuntimeError(\n", + " f\"Deployment creation failed. Detailed Response:\\n{err}\"\n", + " ) from err" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4. Test the Safety Enabled Llama 2 online endpoint." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "test_src_dir = \"./safety-llama-test\"\n", + "os.makedirs(test_src_dir, exist_ok=True)\n", + "print(f\"test script directory: {test_src_dir}\")\n", + "sample_data = os.path.join(test_src_dir, \"sample-request.json\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Choose request from following 4 cells based on the Llama 2 model type you chosen and the scenario you want to test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## For text-generation models (without -chat suffix)\n", + "## Successful response\n", + "\n", + "import json\n", + "\n", + "with open(sample_data, \"w\") as f:\n", + " json.dump(\n", + " {\n", + " \"input_data\": {\n", + " \"input_string\": [\n", + " \"Hello\",\n", + " \"My name is John and I have a dog.\",\n", + " ],\n", + " \"parameters\": {\n", + " \"temperature\": 0.6,\n", + " \"top_p\": 0.6,\n", + " \"max_new_tokens\": 256,\n", + " \"do_sample\": True,\n", + " },\n", + " }\n", + " },\n", + " f,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## For text-generation models (without -chat suffix)\n", + "## Blocked request/response due to hateful content\n", + "\n", + "import json\n", + "\n", + "with open(sample_data, \"w\") as f:\n", + " json.dump(\n", + " {\n", + " \"input_data\": {\n", + " \"input_string\": [\n", + " \"The dog was given a eutanasa injection due to their severed leg bleding profusely from deep lacarations to the lower extremities, exposing tisssue and nerve.\", # This line contains hateful message and endpoint will return empty string as response\n", + " ],\n", + " \"parameters\": {\n", + " \"temperature\": 0.6,\n", + " \"top_p\": 0.6,\n", + " \"max_new_tokens\": 256,\n", + " \"do_sample\": True,\n", + " },\n", + " }\n", + " },\n", + " f,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## For chat-complete models (with \"-chat\" suffix)\n", + "## Successful request/response\n", + "\n", + "import json\n", + "\n", + "with open(sample_data, \"w\") as f:\n", + " json.dump(\n", + " {\n", + " \"input_data\": {\n", + " \"input_string\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What is the tallest building in the world?\",\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"As of 2021, the Burj Khalifa in Dubai, United Arab Emirates is the tallest building in the world, standing at a height of 828 meters (2,722 feet). It was completed in 2010 and has 163 floors. The Burj Khalifa is not only the tallest building in the world but also holds several other records, such as the highest occupied floor, highest outdoor observation deck, elevator with the longest travel distance, and the tallest freestanding structure in the world.\",\n", + " },\n", + " {\"role\": \"user\", \"content\": \"and in Africa?\"},\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"In Africa, the tallest building is the Carlton Centre, located in Johannesburg, South Africa. It stands at a height of 50 floors and 223 meters (730 feet). The CarltonDefault Centre was completed in 1973 and was the tallest building in Africa for many years until the construction of the Leonardo, a 55-story skyscraper in Sandton, Johannesburg, which was completed in 2019 and stands at a height of 230 meters (755 feet). Other notable tall buildings in Africa include the Ponte City Apartments in Johannesburg, the John Hancock Center in Lagos, Nigeria, and the Alpha II Building in Abidjan, Ivory Coast\",\n", + " },\n", + " {\"role\": \"user\", \"content\": \"and in Europe?\"},\n", + " ],\n", + " \"parameters\": {\n", + " \"temperature\": 0.9,\n", + " \"top_p\": 0.6,\n", + " \"do_sample\": True,\n", + " \"max_new_tokens\": 100,\n", + " },\n", + " }\n", + " },\n", + " f,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## For chat-complete models (with \"-chat\" suffix)\n", + "## Blocked request due to hateful content\n", + "\n", + "import json\n", + "\n", + "with open(sample_data, \"w\") as f:\n", + " json.dump(\n", + " {\n", + " \"input_data\": {\n", + " \"input_string\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"A 51-year-old man was found dead in his car. There were blood stains on the dashboard and windscreen. At autopsy, a deep, oblique, long incised injury was found on the front of the neck. It turns out that he died by suicide.\",\n", + " },\n", + " ],\n", + " \"parameters\": {\n", + " \"temperature\": 0.9,\n", + " \"top_p\": 0.6,\n", + " \"do_sample\": True,\n", + " \"max_new_tokens\": 100,\n", + " },\n", + " }\n", + " },\n", + " f,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ml_client.online_endpoints.invoke(\n", + " endpoint_name=endpoint_name,\n", + " deployment_name=deployment_name,\n", + " request_file=sample_data,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernel_info": { + "name": "python3" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + }, + "microsoft": { + "ms_spell_check": { + "ms_spell_check_language": "en" + } + }, + "nteract": { + "version": "nteract-front-end@1.0.0" + } + }, + "nbformat": 4, + "nbformat_minor": 1 } diff --git a/sdk/python/foundation-models/system/inference/text-generation/text-generation-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/text-generation/text-generation-batch-endpoint.ipynb index 286ec37faa..b1e45e46cb 100644 --- a/sdk/python/foundation-models/system/inference/text-generation/text-generation-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/text-generation/text-generation-batch-endpoint.ipynb @@ -499,9 +499,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/text-generation/text-generation-online-endpoint-dolly.ipynb b/sdk/python/foundation-models/system/inference/text-generation/text-generation-online-endpoint-dolly.ipynb index ff0d175220..f6e6d9ce94 100644 --- a/sdk/python/foundation-models/system/inference/text-generation/text-generation-online-endpoint-dolly.ipynb +++ b/sdk/python/foundation-models/system/inference/text-generation/text-generation-online-endpoint-dolly.ipynb @@ -271,9 +271,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/text-generation/text-generation-online-endpoint-oss.ipynb b/sdk/python/foundation-models/system/inference/text-generation/text-generation-online-endpoint-oss.ipynb index 630bc512bf..e092bd83c3 100644 --- a/sdk/python/foundation-models/system/inference/text-generation/text-generation-online-endpoint-oss.ipynb +++ b/sdk/python/foundation-models/system/inference/text-generation/text-generation-online-endpoint-oss.ipynb @@ -279,9 +279,9 @@ ], "metadata": { "kernelspec": { - "display_name": "fm_v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/text-generation/text-generation-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/text-generation/text-generation-online-endpoint.ipynb index 79e3831d0e..f943b36991 100644 --- a/sdk/python/foundation-models/system/inference/text-generation/text-generation-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/text-generation/text-generation-online-endpoint.ipynb @@ -269,9 +269,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/text-to-image/aacs-prepare-uai.ipynb b/sdk/python/foundation-models/system/inference/text-to-image/aacs-prepare-uai.ipynb index 1bbe6a865f..9234bd1056 100644 --- a/sdk/python/foundation-models/system/inference/text-to-image/aacs-prepare-uai.ipynb +++ b/sdk/python/foundation-models/system/inference/text-to-image/aacs-prepare-uai.ipynb @@ -422,6 +422,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/text-to-image/image-text-to-image-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/text-to-image/image-text-to-image-batch-endpoint.ipynb index 7518b57227..30f7a9126b 100644 --- a/sdk/python/foundation-models/system/inference/text-to-image/image-text-to-image-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/text-to-image/image-text-to-image-batch-endpoint.ipynb @@ -484,6 +484,11 @@ "metadata": { "language_info": { "name": "ipython" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/text-to-image/image-text-to-image-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/text-to-image/image-text-to-image-online-endpoint.ipynb index 95ee694d16..027a4f9382 100644 --- a/sdk/python/foundation-models/system/inference/text-to-image/image-text-to-image-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/text-to-image/image-text-to-image-online-endpoint.ipynb @@ -310,10 +310,15 @@ } ], "metadata": { - "language_info": { - "name": "ipython" - } - }, + "language_info": { + "name": "ipython" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" + } + }, "nbformat": 4, "nbformat_minor": 2 } diff --git a/sdk/python/foundation-models/system/inference/text-to-image/safe-image-text-to-image-batch-deployment.ipynb b/sdk/python/foundation-models/system/inference/text-to-image/safe-image-text-to-image-batch-deployment.ipynb index cc2986fb70..4d1c29bda0 100644 --- a/sdk/python/foundation-models/system/inference/text-to-image/safe-image-text-to-image-batch-deployment.ipynb +++ b/sdk/python/foundation-models/system/inference/text-to-image/safe-image-text-to-image-batch-deployment.ipynb @@ -760,6 +760,11 @@ "metadata": { "language_info": { "name": "ipython" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/text-to-image/safe-image-text-to-image-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/text-to-image/safe-image-text-to-image-online-endpoint.ipynb index dcf2e9fddb..36fd1132b7 100644 --- a/sdk/python/foundation-models/system/inference/text-to-image/safe-image-text-to-image-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/text-to-image/safe-image-text-to-image-online-endpoint.ipynb @@ -743,6 +743,11 @@ "metadata": { "language_info": { "name": "ipython" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-batch-deployment.ipynb b/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-batch-deployment.ipynb index c4466528b2..98ef76986a 100644 --- a/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-batch-deployment.ipynb +++ b/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-batch-deployment.ipynb @@ -749,6 +749,11 @@ "metadata": { "language_info": { "name": "ipython" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-inpainting-batch-deployment.ipynb b/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-inpainting-batch-deployment.ipynb index 7c899e9eb1..d11bac2a73 100644 --- a/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-inpainting-batch-deployment.ipynb +++ b/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-inpainting-batch-deployment.ipynb @@ -772,6 +772,11 @@ "metadata": { "language_info": { "name": "ipython" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-inpainting-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-inpainting-online-endpoint.ipynb index 90c6b3675d..8a1b7b81ff 100644 --- a/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-inpainting-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-inpainting-online-endpoint.ipynb @@ -780,7 +780,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" } diff --git a/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-online-deployment.ipynb b/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-online-deployment.ipynb index d520becaf0..370e544c0f 100644 --- a/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-online-deployment.ipynb +++ b/sdk/python/foundation-models/system/inference/text-to-image/safe-text-to-image-online-deployment.ipynb @@ -730,6 +730,11 @@ "metadata": { "language_info": { "name": "ipython" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-batch-endpoint.ipynb index 7fe713e803..a08b7cd819 100644 --- a/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-batch-endpoint.ipynb @@ -480,6 +480,11 @@ "metadata": { "language_info": { "name": "ipython" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-inpainting-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-inpainting-batch-endpoint.ipynb index 5b2d99be73..6e27f1e713 100644 --- a/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-inpainting-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-inpainting-batch-endpoint.ipynb @@ -553,6 +553,11 @@ "metadata": { "language_info": { "name": "ipython" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-inpainting-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-inpainting-online-endpoint.ipynb index 3330e24314..10acd05b2d 100644 --- a/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-inpainting-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-inpainting-online-endpoint.ipynb @@ -373,6 +373,11 @@ "metadata": { "language_info": { "name": "ipython" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-online-endpoint.ipynb index bfecb0578e..34a43340db 100644 --- a/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/text-to-image/text-to-image-online-endpoint.ipynb @@ -401,6 +401,11 @@ "metadata": { "language_info": { "name": "ipython" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/token-classification/token-classification-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/token-classification/token-classification-batch-endpoint.ipynb index 3ab9656cb6..f416280ffc 100644 --- a/sdk/python/foundation-models/system/inference/token-classification/token-classification-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/token-classification/token-classification-batch-endpoint.ipynb @@ -547,9 +547,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/token-classification/token-classification-online-endpoint-oss.ipynb b/sdk/python/foundation-models/system/inference/token-classification/token-classification-online-endpoint-oss.ipynb index c6d5cd72ad..f36fd75802 100644 --- a/sdk/python/foundation-models/system/inference/token-classification/token-classification-online-endpoint-oss.ipynb +++ b/sdk/python/foundation-models/system/inference/token-classification/token-classification-online-endpoint-oss.ipynb @@ -300,9 +300,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/token-classification/token-classification-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/token-classification/token-classification-online-endpoint.ipynb index e09f50f8ca..54fcff31bc 100644 --- a/sdk/python/foundation-models/system/inference/token-classification/token-classification-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/token-classification/token-classification-online-endpoint.ipynb @@ -289,9 +289,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/translation/translation-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/translation/translation-batch-endpoint.ipynb index 2aca7249fa..6637ce4f47 100644 --- a/sdk/python/foundation-models/system/inference/translation/translation-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/translation/translation-batch-endpoint.ipynb @@ -505,9 +505,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/translation/translation-online-endpoint-oss.ipynb b/sdk/python/foundation-models/system/inference/translation/translation-online-endpoint-oss.ipynb index e79fcab36f..042a8f5bd8 100644 --- a/sdk/python/foundation-models/system/inference/translation/translation-online-endpoint-oss.ipynb +++ b/sdk/python/foundation-models/system/inference/translation/translation-online-endpoint-oss.ipynb @@ -293,9 +293,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/translation/translation-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/translation/translation-online-endpoint.ipynb index 9c9c961123..8173c7bf6a 100644 --- a/sdk/python/foundation-models/system/inference/translation/translation-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/translation/translation-online-endpoint.ipynb @@ -282,9 +282,9 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/video-multi-object-tracking/video-multi-object-tracking-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/video-multi-object-tracking/video-multi-object-tracking-online-endpoint.ipynb index f8d1a12e5c..aee4da7ba1 100644 --- a/sdk/python/foundation-models/system/inference/video-multi-object-tracking/video-multi-object-tracking-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/video-multi-object-tracking/video-multi-object-tracking-online-endpoint.ipynb @@ -374,9 +374,9 @@ ], "metadata": { "kernelspec": { - "display_name": "py38", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/visual-chat-completion/image-text-to-text-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/visual-chat-completion/image-text-to-text-online-endpoint.ipynb index 81441ccc84..8f71ee3ebd 100644 --- a/sdk/python/foundation-models/system/inference/visual-chat-completion/image-text-to-text-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/visual-chat-completion/image-text-to-text-online-endpoint.ipynb @@ -351,6 +351,11 @@ "metadata": { "language_info": { "name": "python" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK V2", + "language": "python", + "name": "python310-sdkv2" } }, "nbformat": 4, diff --git a/sdk/python/foundation-models/system/inference/visual-question-answering/visual-question-answering-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/visual-question-answering/visual-question-answering-batch-endpoint.ipynb index 49cc039cd7..c3a68249cc 100644 --- a/sdk/python/foundation-models/system/inference/visual-question-answering/visual-question-answering-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/visual-question-answering/visual-question-answering-batch-endpoint.ipynb @@ -481,9 +481,9 @@ ], "metadata": { "kernelspec": { - "display_name": "sdkv2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "sdkv2" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/visual-question-answering/visual-question-answering-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/visual-question-answering/visual-question-answering-online-endpoint.ipynb index 9f0471f2d0..ee2a088c39 100644 --- a/sdk/python/foundation-models/system/inference/visual-question-answering/visual-question-answering-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/visual-question-answering/visual-question-answering-online-endpoint.ipynb @@ -337,9 +337,9 @@ ], "metadata": { "kernelspec": { - "display_name": "sdkv2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "sdkv2" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/zero-shot-image-classification/zero-shot-image-classification-batch-endpoint.ipynb b/sdk/python/foundation-models/system/inference/zero-shot-image-classification/zero-shot-image-classification-batch-endpoint.ipynb index 29ddef08c3..ca0f0c31af 100644 --- a/sdk/python/foundation-models/system/inference/zero-shot-image-classification/zero-shot-image-classification-batch-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/zero-shot-image-classification/zero-shot-image-classification-batch-endpoint.ipynb @@ -490,9 +490,9 @@ ], "metadata": { "kernelspec": { - "display_name": "rc_133", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/foundation-models/system/inference/zero-shot-image-classification/zero-shot-image-classification-online-endpoint.ipynb b/sdk/python/foundation-models/system/inference/zero-shot-image-classification/zero-shot-image-classification-online-endpoint.ipynb index 4aa8f95832..123cb90b66 100644 --- a/sdk/python/foundation-models/system/inference/zero-shot-image-classification/zero-shot-image-classification-online-endpoint.ipynb +++ b/sdk/python/foundation-models/system/inference/zero-shot-image-classification/zero-shot-image-classification-online-endpoint.ipynb @@ -342,9 +342,9 @@ ], "metadata": { "kernelspec": { - "display_name": "rc_133", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/promptflow/create_faiss_index.ipynb b/sdk/python/generative-ai/promptflow/create_faiss_index.ipynb index 3751f3b660..fb7efaad07 100644 --- a/sdk/python/generative-ai/promptflow/create_faiss_index.ipynb +++ b/sdk/python/generative-ai/promptflow/create_faiss_index.ipynb @@ -214,9 +214,9 @@ ], "metadata": { "kernelspec": { - "display_name": "amlsubmitjobs", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/promptflow/deploy-flow/deploy-flow.ipynb b/sdk/python/generative-ai/promptflow/deploy-flow/deploy-flow.ipynb index 935d21d4df..6da0b1a7d2 100644 --- a/sdk/python/generative-ai/promptflow/deploy-flow/deploy-flow.ipynb +++ b/sdk/python/generative-ai/promptflow/deploy-flow/deploy-flow.ipynb @@ -199,9 +199,9 @@ ], "metadata": { "kernelspec": { - "display_name": "pf-examples", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/code_first/flows/chat-with-index/src/test.ipynb b/sdk/python/generative-ai/rag/code_first/flows/chat-with-index/src/test.ipynb index edc8adff5e..1caea20495 100644 --- a/sdk/python/generative-ai/rag/code_first/flows/chat-with-index/src/test.ipynb +++ b/sdk/python/generative-ai/rag/code_first/flows/chat-with-index/src/test.ipynb @@ -32,9 +32,9 @@ ], "metadata": { "kernelspec": { - "display_name": "pfsdk", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/azure_cognitive_search/acs_mlindex_with_langchain.ipynb b/sdk/python/generative-ai/rag/notebooks/azure_cognitive_search/acs_mlindex_with_langchain.ipynb index a12e76789a..475d7d0140 100644 --- a/sdk/python/generative-ai/rag/notebooks/azure_cognitive_search/acs_mlindex_with_langchain.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/azure_cognitive_search/acs_mlindex_with_langchain.ipynb @@ -973,9 +973,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/azure_cognitive_search/s3_to_acs_mlindex_with_langchain.ipynb b/sdk/python/generative-ai/rag/notebooks/azure_cognitive_search/s3_to_acs_mlindex_with_langchain.ipynb index acf14d658a..5a0bebe5d5 100644 --- a/sdk/python/generative-ai/rag/notebooks/azure_cognitive_search/s3_to_acs_mlindex_with_langchain.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/azure_cognitive_search/s3_to_acs_mlindex_with_langchain.ipynb @@ -774,9 +774,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk/crack_pdfs_with_azure_document_intelligence.ipynb b/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk/crack_pdfs_with_azure_document_intelligence.ipynb index c58bc5a159..3e25240dc4 100644 --- a/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk/crack_pdfs_with_azure_document_intelligence.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk/crack_pdfs_with_azure_document_intelligence.ipynb @@ -412,9 +412,9 @@ ], "metadata": { "kernelspec": { - "display_name": "project-baker310", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/crack_and_chunk_with_doc_intel_parallel_component.ipynb b/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/crack_and_chunk_with_doc_intel_parallel_component.ipynb index 5429c0ab63..4fbef54e13 100644 --- a/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/crack_and_chunk_with_doc_intel_parallel_component.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/crack_and_chunk_with_doc_intel_parallel_component.ipynb @@ -347,9 +347,9 @@ "name": "python3" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/d_parallel_crack_pdfs_with_azure_document_intelligence.ipynb b/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/d_parallel_crack_pdfs_with_azure_document_intelligence.ipynb index 96d8f488c9..a96a099e63 100644 --- a/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/d_parallel_crack_pdfs_with_azure_document_intelligence.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/d_parallel_crack_pdfs_with_azure_document_intelligence.ipynb @@ -559,9 +559,9 @@ "name": "python3" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/e_parallel_crack_pdfs_with_azure_document_intelligence.ipynb b/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/e_parallel_crack_pdfs_with_azure_document_intelligence.ipynb index c23ba74bfa..c32b7cafba 100644 --- a/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/e_parallel_crack_pdfs_with_azure_document_intelligence.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/e_parallel_crack_pdfs_with_azure_document_intelligence.ipynb @@ -579,9 +579,9 @@ "name": "python3" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/f_parallel_crack_pdfs_with_azure_document_intelligence.ipynb b/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/f_parallel_crack_pdfs_with_azure_document_intelligence.ipynb index 380cd4e2d6..d9722d0a2e 100644 --- a/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/f_parallel_crack_pdfs_with_azure_document_intelligence.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/f_parallel_crack_pdfs_with_azure_document_intelligence.ipynb @@ -633,9 +633,9 @@ "name": "python3" }, "kernelspec": { - "name": "python3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "display_name": "Python 3 (ipykernel)" + "name": "python310-sdkv2" }, "language_info": { "name": "python", diff --git a/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/local_parallel_crack_pdfs_with_azure_document_intelligence.ipynb b/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/local_parallel_crack_pdfs_with_azure_document_intelligence.ipynb index f4aadef2dd..2dbde7e571 100644 --- a/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/local_parallel_crack_pdfs_with_azure_document_intelligence.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/custom_crack_and_chunk_parallel/local_parallel_crack_pdfs_with_azure_document_intelligence.ipynb @@ -587,9 +587,9 @@ "name": "python3" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/db_copilot_with_rag.ipynb b/sdk/python/generative-ai/rag/notebooks/db_copilot_with_rag.ipynb index 713e2c237f..2fec5be7ec 100644 --- a/sdk/python/generative-ai/rag/notebooks/db_copilot_with_rag.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/db_copilot_with_rag.ipynb @@ -1153,9 +1153,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/faiss/faiss_mlindex_with_langchain.ipynb b/sdk/python/generative-ai/rag/notebooks/faiss/faiss_mlindex_with_langchain.ipynb index 3de719832f..4b96db637a 100644 --- a/sdk/python/generative-ai/rag/notebooks/faiss/faiss_mlindex_with_langchain.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/faiss/faiss_mlindex_with_langchain.ipynb @@ -719,9 +719,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/faiss/scheduled_update_faiss_index.ipynb b/sdk/python/generative-ai/rag/notebooks/faiss/scheduled_update_faiss_index.ipynb index 71d59672c6..17676337ad 100644 --- a/sdk/python/generative-ai/rag/notebooks/faiss/scheduled_update_faiss_index.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/faiss/scheduled_update_faiss_index.ipynb @@ -663,9 +663,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/faiss/url_to_faiss_incremental_embeddings_with_tabular_data.ipynb b/sdk/python/generative-ai/rag/notebooks/faiss/url_to_faiss_incremental_embeddings_with_tabular_data.ipynb index cb1ac10fe5..bda4c4d424 100644 --- a/sdk/python/generative-ai/rag/notebooks/faiss/url_to_faiss_incremental_embeddings_with_tabular_data.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/faiss/url_to_faiss_incremental_embeddings_with_tabular_data.ipynb @@ -997,7 +997,7 @@ "name": "python310-sdkv2" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/generative-ai/rag/notebooks/mlindex_with_testgen.ipynb b/sdk/python/generative-ai/rag/notebooks/mlindex_with_testgen.ipynb index 1f4fc0c5e4..3650fca5fc 100644 --- a/sdk/python/generative-ai/rag/notebooks/mlindex_with_testgen.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/mlindex_with_testgen.ipynb @@ -626,9 +626,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/mlindex_with_testgen_autoprompt.ipynb b/sdk/python/generative-ai/rag/notebooks/mlindex_with_testgen_autoprompt.ipynb index c6a2132070..454c7833f4 100644 --- a/sdk/python/generative-ai/rag/notebooks/mlindex_with_testgen_autoprompt.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/mlindex_with_testgen_autoprompt.ipynb @@ -688,9 +688,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/pinecone/pinecone_mlindex_with_langchain.ipynb b/sdk/python/generative-ai/rag/notebooks/pinecone/pinecone_mlindex_with_langchain.ipynb index 4fcec5a5c0..de05c2d5ca 100644 --- a/sdk/python/generative-ai/rag/notebooks/pinecone/pinecone_mlindex_with_langchain.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/pinecone/pinecone_mlindex_with_langchain.ipynb @@ -1021,9 +1021,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/pinecone/s3_to_pinecone_mlindex_with_langchain.ipynb b/sdk/python/generative-ai/rag/notebooks/pinecone/s3_to_pinecone_mlindex_with_langchain.ipynb index 6b286d81f4..3643ee64ef 100644 --- a/sdk/python/generative-ai/rag/notebooks/pinecone/s3_to_pinecone_mlindex_with_langchain.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/pinecone/s3_to_pinecone_mlindex_with_langchain.ipynb @@ -815,9 +815,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/generative-ai/rag/notebooks/qa_data_generation.ipynb b/sdk/python/generative-ai/rag/notebooks/qa_data_generation.ipynb index ad43d01fce..9c8a86aef8 100644 --- a/sdk/python/generative-ai/rag/notebooks/qa_data_generation.ipynb +++ b/sdk/python/generative-ai/rag/notebooks/qa_data_generation.ipynb @@ -577,9 +577,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/automl-forecasting-distributed-tcn.ipynb b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/automl-forecasting-distributed-tcn.ipynb index 9c91e8c9d6..c2732579fe 100644 --- a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/automl-forecasting-distributed-tcn.ipynb +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/automl-forecasting-distributed-tcn.ipynb @@ -1314,7 +1314,7 @@ "name": "python310-sdkv2" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-batch-inference.ipynb b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-batch-inference.ipynb index d350a88a06..6be9d8017e 100644 --- a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-batch-inference.ipynb +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-batch-inference.ipynb @@ -681,9 +681,9 @@ "friendly_name": "Forecasting away from training data", "index_order": 3, "kernelspec": { - "display_name": "sdkv2-test1", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-local-inference.ipynb b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-local-inference.ipynb index e0d50b2737..32d1f0a2bc 100644 --- a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-local-inference.ipynb +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-local-inference.ipynb @@ -462,9 +462,9 @@ "friendly_name": "Forecasting away from training data", "index_order": 3, "kernelspec": { - "display_name": "model-inference", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "project_environment" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-training.ipynb b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-training.ipynb index ab970e0479..8e2077f364 100644 --- a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-training.ipynb +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-forecast-function/auto-ml-forecasting-function-gap-training.ipynb @@ -596,9 +596,9 @@ "friendly_name": "Forecasting away from training data", "index_order": 3, "kernelspec": { - "display_name": "sdkv2-test1", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-energy-demand/automl-forecasting-task-energy-demand-advanced.ipynb b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-energy-demand/automl-forecasting-task-energy-demand-advanced.ipynb index f4a96a629b..1a82d9a58d 100644 --- a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-energy-demand/automl-forecasting-task-energy-demand-advanced.ipynb +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-energy-demand/automl-forecasting-task-energy-demand-advanced.ipynb @@ -10,7 +10,7 @@ "**Requirements** - In order to benefit from this tutorial, you will need:\n", "- A basic understanding of Machine Learning\n", "- An Azure account with an active subscription. [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F)\n", - "- An Azure ML workspace. [Check this notebook for creating a workspace](../../../resources/workspace/workspace.ipynb) \n", + "- An Azure ML workspace. [Check this notebook for creating a workspace](../../../resources/workspace/workspace.ipynb) \n", "- A python environment\n", "- Installed Azure Machine Learning Python SDK v2 - [install instructions](../../../README.md) - check the getting started section\n", "\n", @@ -283,7 +283,7 @@ }, "outputs": [], "source": [ - "# general job parameters\n", + "# general job parameters\n", "max_trials = 5\n", "exp_name = \"dpv2-forecasting-experiment\"" ] @@ -310,7 +310,7 @@ "source": [ "# Create the AutoML forecasting job with the related factory-function.\n", "\n", - "forecasting_job = automl.forecasting(\n", + "forecasting_job = automl.forecasting(\n", " # name=\"dpv2-forecasting-job-02\",\n", " experiment_name=exp_name,\n", " training_data=my_training_data_input,\n", diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-image-instance-segmentation-task-fridge-items/odFridgeObjects_jsonl2coco.ipynb b/sdk/python/jobs/automl-standalone-jobs/automl-image-instance-segmentation-task-fridge-items/odFridgeObjects_jsonl2coco.ipynb index 912fe2def6..ec420def62 100644 --- a/sdk/python/jobs/automl-standalone-jobs/automl-image-instance-segmentation-task-fridge-items/odFridgeObjects_jsonl2coco.ipynb +++ b/sdk/python/jobs/automl-standalone-jobs/automl-image-instance-segmentation-task-fridge-items/odFridgeObjects_jsonl2coco.ipynb @@ -131,9 +131,9 @@ ], "metadata": { "kernelspec": { - "display_name": "rc_133", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-regression-task-hardware-performance/automl-regression-task-hardware-performance.ipynb b/sdk/python/jobs/automl-standalone-jobs/automl-regression-task-hardware-performance/automl-regression-task-hardware-performance.ipynb index e853726098..e59aa0c1e3 100644 --- a/sdk/python/jobs/automl-standalone-jobs/automl-regression-task-hardware-performance/automl-regression-task-hardware-performance.ipynb +++ b/sdk/python/jobs/automl-standalone-jobs/automl-regression-task-hardware-performance/automl-regression-task-hardware-performance.ipynb @@ -9,7 +9,7 @@ "**Requirements** - In order to benefit from this tutorial, you will need:\n", "- A basic understanding of Machine Learning\n", "- An Azure account with an active subscription. [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F)\n", - "- An Azure ML workspace. [Check this notebook for creating a workspace](../../../resources/workspace/workspace.ipynb) \n", + "- An Azure ML workspace. [Check this notebook for creating a workspace](../../../resources/workspace/workspace.ipynb) \n", "- A python environment\n", "- Installed Azure Machine Learning Python SDK v2 - [install instructions](../../../README.md) - check the getting started section\n", "\n", @@ -221,7 +221,7 @@ "source": [ "# Create the AutoML regression job with the related factory-function.\n", "\n", - "regression_job = automl.regression(\n", + "regression_job = automl.regression(\n", " experiment_name=exp_name,\n", " training_data=my_training_data_input,\n", " target_column_name=\"ERP\",\n", diff --git a/sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion/chat_completion_with_model_as_platform.ipynb b/sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion/chat_completion_with_model_as_platform.ipynb index 1064fb34c1..0e37d8652c 100644 --- a/sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion/chat_completion_with_model_as_platform.ipynb +++ b/sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion/chat_completion_with_model_as_platform.ipynb @@ -550,7 +550,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion/gpt-oss-20b/gpt-oss-20b-chat-completion.ipynb b/sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion/gpt-oss-20b/gpt-oss-20b-chat-completion.ipynb index d651c8674d..2c73845f42 100644 --- a/sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion/gpt-oss-20b/gpt-oss-20b-chat-completion.ipynb +++ b/sdk/python/jobs/finetuning/standalone/model-as-a-platform/chat-completion/gpt-oss-20b/gpt-oss-20b-chat-completion.ipynb @@ -811,9 +811,9 @@ ], "metadata": { "kernelspec": { - "display_name": ".conda", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/jobs/finetuning/standalone/model-as-a-platform/text-generation/text_generation_with_model_as_a_platform.ipynb b/sdk/python/jobs/finetuning/standalone/model-as-a-platform/text-generation/text_generation_with_model_as_a_platform.ipynb index 6c75fee839..af5a66d004 100644 --- a/sdk/python/jobs/finetuning/standalone/model-as-a-platform/text-generation/text_generation_with_model_as_a_platform.ipynb +++ b/sdk/python/jobs/finetuning/standalone/model-as-a-platform/text-generation/text_generation_with_model_as_a_platform.ipynb @@ -525,7 +525,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/jobs/finetuning/standalone/model-as-a-service/chat-completion/chat_completion_with_model_as_service.ipynb b/sdk/python/jobs/finetuning/standalone/model-as-a-service/chat-completion/chat_completion_with_model_as_service.ipynb index 2568ccf482..37fe626fd9 100644 --- a/sdk/python/jobs/finetuning/standalone/model-as-a-service/chat-completion/chat_completion_with_model_as_service.ipynb +++ b/sdk/python/jobs/finetuning/standalone/model-as-a-service/chat-completion/chat_completion_with_model_as_service.ipynb @@ -649,9 +649,9 @@ ], "metadata": { "kernelspec": { - "display_name": "ss1", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/jobs/finetuning/standalone/model-as-a-service/text-generation/text_generation_with_model_as_a_service.ipynb b/sdk/python/jobs/finetuning/standalone/model-as-a-service/text-generation/text_generation_with_model_as_a_service.ipynb index 27a2fa1241..a402f9e5b0 100644 --- a/sdk/python/jobs/finetuning/standalone/model-as-a-service/text-generation/text_generation_with_model_as_a_service.ipynb +++ b/sdk/python/jobs/finetuning/standalone/model-as-a-service/text-generation/text_generation_with_model_as_a_service.ipynb @@ -503,7 +503,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/jobs/grpo/grpo_from_aml_pipeline.ipynb b/sdk/python/jobs/grpo/grpo_from_aml_pipeline.ipynb index f96fbd9a16..63f46e85c7 100644 --- a/sdk/python/jobs/grpo/grpo_from_aml_pipeline.ipynb +++ b/sdk/python/jobs/grpo/grpo_from_aml_pipeline.ipynb @@ -412,9 +412,9 @@ "name": "python38-azureml" }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/jobs/grpo/launch_grpo_command_job-med-mcqa-commented.ipynb b/sdk/python/jobs/grpo/launch_grpo_command_job-med-mcqa-commented.ipynb index d7d3a52aeb..f0c045cf76 100644 --- a/sdk/python/jobs/grpo/launch_grpo_command_job-med-mcqa-commented.ipynb +++ b/sdk/python/jobs/grpo/launch_grpo_command_job-med-mcqa-commented.ipynb @@ -560,9 +560,9 @@ "name": "python38-azureml" }, "kernelspec": { - "display_name": "aip-tutorials", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml/pipeline_with_components_from_yaml_serverless.ipynb b/sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml/pipeline_with_components_from_yaml_serverless.ipynb index e874c4428e..6ccb8c47f9 100644 --- a/sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml/pipeline_with_components_from_yaml_serverless.ipynb +++ b/sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml/pipeline_with_components_from_yaml_serverless.ipynb @@ -146,7 +146,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {"name": "build-pipeline"}, + "metadata": { + "name": "build-pipeline" + }, "outputs": [], "source": [ "# Construct pipeline\n", @@ -272,4 +274,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-hts-pipeline/aml-demand-forecast-hts.ipynb b/sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-hts-pipeline/aml-demand-forecast-hts.ipynb index 4c7c07ec3a..3d4b4c7fa1 100644 --- a/sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-hts-pipeline/aml-demand-forecast-hts.ipynb +++ b/sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-hts-pipeline/aml-demand-forecast-hts.ipynb @@ -1354,7 +1354,7 @@ "name": "python310-sdkv2" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-mm-pipeline/aml-demand-forecast-mm-pipeline.ipynb b/sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-mm-pipeline/aml-demand-forecast-mm-pipeline.ipynb index f225be9135..2564cb03d4 100644 --- a/sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-mm-pipeline/aml-demand-forecast-mm-pipeline.ipynb +++ b/sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-mm-pipeline/aml-demand-forecast-mm-pipeline.ipynb @@ -1454,7 +1454,7 @@ "name": "python310-sdkv2" }, "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/jobs/pipelines/1l_flow_in_pipeline/flow_in_pipeline.ipynb b/sdk/python/jobs/pipelines/1l_flow_in_pipeline/flow_in_pipeline.ipynb index 20aec64aad..18691e73ee 100644 --- a/sdk/python/jobs/pipelines/1l_flow_in_pipeline/flow_in_pipeline.ipynb +++ b/sdk/python/jobs/pipelines/1l_flow_in_pipeline/flow_in_pipeline.ipynb @@ -213,9 +213,9 @@ "description": "Create pipeline using components to run a distributed job with tensorflow" }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/jobs/pipelines/2c_nyc_taxi_data_regression/nyc_taxi_data_regression.ipynb b/sdk/python/jobs/pipelines/2c_nyc_taxi_data_regression/nyc_taxi_data_regression.ipynb index eb9a374ea9..8709eb060d 100644 --- a/sdk/python/jobs/pipelines/2c_nyc_taxi_data_regression/nyc_taxi_data_regression.ipynb +++ b/sdk/python/jobs/pipelines/2c_nyc_taxi_data_regression/nyc_taxi_data_regression.ipynb @@ -107,7 +107,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "name": "build-pipeline-with-pipeline-output" + "name": "build-pipeline-with-pipeline-output" }, "outputs": [], "source": [ diff --git a/sdk/python/jobs/pipelines/2e_image_classification_keras_minist_convnet/image_classification_keras_minist_convnet.ipynb b/sdk/python/jobs/pipelines/2e_image_classification_keras_minist_convnet/image_classification_keras_minist_convnet.ipynb index ce066a13b9..794c19f18b 100644 --- a/sdk/python/jobs/pipelines/2e_image_classification_keras_minist_convnet/image_classification_keras_minist_convnet.ipynb +++ b/sdk/python/jobs/pipelines/2e_image_classification_keras_minist_convnet/image_classification_keras_minist_convnet.ipynb @@ -379,9 +379,9 @@ "description": "Create pipeline to train cnn image classification model with keras" }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/jobs/pipelines/README.md b/sdk/python/jobs/pipelines/README.md index 5495646352..1f989bc112 100644 --- a/sdk/python/jobs/pipelines/README.md +++ b/sdk/python/jobs/pipelines/README.md @@ -28,10 +28,12 @@ Test Status is for branch - **_main_** |jobs|pipelines|[pipeline_with_spark_nodes](1i_pipeline_with_spark_nodes/pipeline_with_spark_nodes.ipynb)|Create pipeline with spark node|[![pipeline_with_spark_nodes](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1i_pipeline_with_spark_nodes-pipeline_with_spark_nodes.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1i_pipeline_with_spark_nodes-pipeline_with_spark_nodes.yml)| |jobs|pipelines|[nyc_taxi_data_regression_with_pipeline_component](1j_pipeline_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component.ipynb)|Create pipeline with CommandComponents from local YAML file|[![nyc_taxi_data_regression_with_pipeline_component](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component-nyc_taxi_data_regression_with_pipeline_component.yml)| |jobs|pipelines|[pipeline_with_train_eval_pipeline_component](1j_pipeline_with_pipeline_component/pipeline_with_train_eval_pipeline_component/pipeline_with_train_eval_pipeline_component.ipynb)|Create pipeline with CommandComponents from local YAML file|[![pipeline_with_train_eval_pipeline_component](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-pipeline_with_train_eval_pipeline_component.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1j_pipeline_with_pipeline_component-pipeline_with_train_eval_pipeline_component-pipeline_with_train_eval_pipeline_component.yml)| -|jobs|pipelines|[automl-forecasting-demand-hierarchical-timeseries-in-pipeline](1k_demand_forecasting_with_pipeline_components/automl-forecasting-demand-hierarchical-timeseries-in-pipeline/automl-forecasting-demand-hierarchical-timeseries-in-pipeline.ipynb)|*no description*|[![automl-forecasting-demand-hierarchical-timeseries-in-pipeline](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecasting_with_pipeline_components-automl-forecasting-demand-hierarchical-timeseries-in-pipeline-automl-forecasting-demand-hierarchical-timeseries-in-pipeline.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecasting_with_pipeline_components-automl-forecasting-demand-hierarchical-timeseries-in-pipeline-automl-forecasting-demand-hierarchical-timeseries-in-pipeline.yml)| -|jobs|pipelines|[automl-forecasting-demand-many-models-in-pipeline](1k_demand_forecasting_with_pipeline_components/automl-forecasting-demand-many-models-in-pipeline/automl-forecasting-demand-many-models-in-pipeline.ipynb)|*no description*|[![automl-forecasting-demand-many-models-in-pipeline](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecasting_with_pipeline_components-automl-forecasting-demand-many-models-in-pipeline-automl-forecasting-demand-many-models-in-pipeline.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecasting_with_pipeline_components-automl-forecasting-demand-many-models-in-pipeline-automl-forecasting-demand-many-models-in-pipeline.yml)| +|jobs|pipelines|[aml-demand-forecast-hts](1k_demand_forecast_pipeline/aml-demand-forecast-hts-pipeline/aml-demand-forecast-hts.ipynb)|*no description*|[![aml-demand-forecast-hts](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-hts-pipeline-aml-demand-forecast-hts.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-hts-pipeline-aml-demand-forecast-hts.yml)| +|jobs|pipelines|[aml-demand-forecast-mm-pipeline](1k_demand_forecast_pipeline/aml-demand-forecast-mm-pipeline/aml-demand-forecast-mm-pipeline.ipynb)|*no description*|[![aml-demand-forecast-mm-pipeline](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-mm-pipeline-aml-demand-forecast-mm-pipeline.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1k_demand_forecast_pipeline-aml-demand-forecast-mm-pipeline-aml-demand-forecast-mm-pipeline.yml)| +|jobs|pipelines|[flow_in_pipeline](1l_flow_in_pipeline/flow_in_pipeline.ipynb)|Create pipeline using components to run a distributed job with tensorflow|[![flow_in_pipeline](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1l_flow_in_pipeline-flow_in_pipeline.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1l_flow_in_pipeline-flow_in_pipeline.yml)| |jobs|pipelines|[train_mnist_with_tensorflow](2a_train_mnist_with_tensorflow/train_mnist_with_tensorflow.ipynb)|Create pipeline using components to run a distributed job with tensorflow|[![train_mnist_with_tensorflow](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2a_train_mnist_with_tensorflow-train_mnist_with_tensorflow.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2a_train_mnist_with_tensorflow-train_mnist_with_tensorflow.yml)| |jobs|pipelines|[train_cifar_10_with_pytorch](2b_train_cifar_10_with_pytorch/train_cifar_10_with_pytorch.ipynb)|Get data, train and evaluate a model in pipeline with Components|[![train_cifar_10_with_pytorch](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2b_train_cifar_10_with_pytorch-train_cifar_10_with_pytorch.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2b_train_cifar_10_with_pytorch-train_cifar_10_with_pytorch.yml)| |jobs|pipelines|[nyc_taxi_data_regression](2c_nyc_taxi_data_regression/nyc_taxi_data_regression.ipynb)|Build pipeline with components for 5 jobs - prep data, transform data, train model, predict results and evaluate model performance|[![nyc_taxi_data_regression](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2c_nyc_taxi_data_regression-nyc_taxi_data_regression.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2c_nyc_taxi_data_regression-nyc_taxi_data_regression.yml)| |jobs|pipelines|[image_classification_with_densenet](2d_image_classification_with_densenet/image_classification_with_densenet.ipynb)|Create pipeline to train cnn image classification model|[![image_classification_with_densenet](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2d_image_classification_with_densenet-image_classification_with_densenet.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2d_image_classification_with_densenet-image_classification_with_densenet.yml)| |jobs|pipelines|[image_classification_keras_minist_convnet](2e_image_classification_keras_minist_convnet/image_classification_keras_minist_convnet.ipynb)|Create pipeline to train cnn image classification model with keras|[![image_classification_keras_minist_convnet](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2e_image_classification_keras_minist_convnet-image_classification_keras_minist_convnet.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-2e_image_classification_keras_minist_convnet-image_classification_keras_minist_convnet.yml)| +|jobs|pipelines|[pipeline_with_step_sequence_dummy_dependencies](stepsequence/pipeline_with_step_sequence_dummy_dependencies.ipynb)|*no description*|[![pipeline_with_step_sequence_dummy_dependencies](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-stepsequence-pipeline_with_step_sequence_dummy_dependencies.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-stepsequence-pipeline_with_step_sequence_dummy_dependencies.yml)| diff --git a/sdk/python/jobs/pipelines/stepsequence/pipeline_with_step_sequence_dummy_dependencies.ipynb b/sdk/python/jobs/pipelines/stepsequence/pipeline_with_step_sequence_dummy_dependencies.ipynb index d1a226bc38..696e6ea60a 100644 --- a/sdk/python/jobs/pipelines/stepsequence/pipeline_with_step_sequence_dummy_dependencies.ipynb +++ b/sdk/python/jobs/pipelines/stepsequence/pipeline_with_step_sequence_dummy_dependencies.ipynb @@ -453,7 +453,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK v2", + "display_name": "Python 3.10 - SDK V2", "language": "python", "name": "python310-sdkv2" }, diff --git a/sdk/python/jobs/spark/automation/run_interactive_session_notebook.ipynb b/sdk/python/jobs/spark/automation/run_interactive_session_notebook.ipynb index d5a237598d..d8b7fcbd43 100644 --- a/sdk/python/jobs/spark/automation/run_interactive_session_notebook.ipynb +++ b/sdk/python/jobs/spark/automation/run_interactive_session_notebook.ipynb @@ -69,9 +69,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb b/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb index 96a756b5da..730f866104 100644 --- a/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb +++ b/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb @@ -879,9 +879,9 @@ "name": "azureml_py310_sdkv2_vnet" }, "kernelspec": { - "display_name": "azureml_py310_sdkv2_vnet", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "azureml_py310_sdkv2_vnet" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/readme.py b/sdk/python/readme.py index 43da97dd2b..642d4c9eb4 100644 --- a/sdk/python/readme.py +++ b/sdk/python/readme.py @@ -255,7 +255,7 @@ def write_notebook_workflow( - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" - name: pip install notebook reqs run: pip install --no-cache-dir -r sdk/python/dev-requirements.txt{mlflow_import}{forecast_import} - name: azure login @@ -279,7 +279,7 @@ def write_notebook_workflow( continue-on-error: true - name: validate readme run: | - python check-readme.py "{github_workspace}" "{github_workspace}/sdk/python/{posix_folder}" + python check-readme.py "{github_workspace}/sdk/python/{posix_folder}" working-directory: infra/bootstrapping continue-on-error: false - name: setup-cli diff --git a/sdk/python/resources/connections/connections.ipynb b/sdk/python/resources/connections/connections.ipynb index 64539f328d..023cc05dba 100644 --- a/sdk/python/resources/connections/connections.ipynb +++ b/sdk/python/resources/connections/connections.ipynb @@ -57,7 +57,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {"name": "details"}, + "metadata": { + "name": "details" + }, "outputs": [], "source": [ "# Enter details of your AML workspace\n", @@ -69,7 +71,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {"name": "ml_client"}, + "metadata": { + "name": "ml_client" + }, "outputs": [], "source": [ "# get a handle to the workspace\n", diff --git a/sdk/python/resources/workspace/workspace-managed-network.ipynb b/sdk/python/resources/workspace/workspace-managed-network.ipynb index 14c40a9eb6..fa36289faa 100644 --- a/sdk/python/resources/workspace/workspace-managed-network.ipynb +++ b/sdk/python/resources/workspace/workspace-managed-network.ipynb @@ -349,9 +349,9 @@ "description": "Create Azure ML workspace" }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/responsible-ai/tabular/responsibleaidashboard-ed-student-attrition-clf/responsibleaidashboard-ed-student-attrition-clf.ipynb b/sdk/python/responsible-ai/tabular/responsibleaidashboard-ed-student-attrition-clf/responsibleaidashboard-ed-student-attrition-clf.ipynb index a7248c01bd..2e02bc1bb6 100644 --- a/sdk/python/responsible-ai/tabular/responsibleaidashboard-ed-student-attrition-clf/responsibleaidashboard-ed-student-attrition-clf.ipynb +++ b/sdk/python/responsible-ai/tabular/responsibleaidashboard-ed-student-attrition-clf/responsibleaidashboard-ed-student-attrition-clf.ipynb @@ -1910,9 +1910,9 @@ "name": "python38-azureml" }, "kernelspec": { - "display_name": "Python 3.8 - AzureML", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python38-azureml" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/responsible-ai/tabular/responsibleaidashboard-finance-loan-classification/responsibleaidashboard-finance-loan-classification.ipynb b/sdk/python/responsible-ai/tabular/responsibleaidashboard-finance-loan-classification/responsibleaidashboard-finance-loan-classification.ipynb index 1d1a6ba812..627b2cecd9 100644 --- a/sdk/python/responsible-ai/tabular/responsibleaidashboard-finance-loan-classification/responsibleaidashboard-finance-loan-classification.ipynb +++ b/sdk/python/responsible-ai/tabular/responsibleaidashboard-finance-loan-classification/responsibleaidashboard-finance-loan-classification.ipynb @@ -1888,9 +1888,9 @@ "name": "python310-sdkv2" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/sdk/python/responsible-ai/tabular/responsibleaidashboard-healthcare-covid-classification/responsibleaidashboard-healthcare-covid-classification.ipynb b/sdk/python/responsible-ai/tabular/responsibleaidashboard-healthcare-covid-classification/responsibleaidashboard-healthcare-covid-classification.ipynb index 02dbf3ab7d..33df55e386 100644 --- a/sdk/python/responsible-ai/tabular/responsibleaidashboard-healthcare-covid-classification/responsibleaidashboard-healthcare-covid-classification.ipynb +++ b/sdk/python/responsible-ai/tabular/responsibleaidashboard-healthcare-covid-classification/responsibleaidashboard-healthcare-covid-classification.ipynb @@ -1937,9 +1937,9 @@ "name": "python310-sdkv2" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/tutorials/azureml-in-a-day/azureml-in-a-day.ipynb b/tutorials/azureml-in-a-day/azureml-in-a-day.ipynb index 860e7c33d3..113e31d39d 100644 --- a/tutorials/azureml-in-a-day/azureml-in-a-day.ipynb +++ b/tutorials/azureml-in-a-day/azureml-in-a-day.ipynb @@ -1,786 +1,786 @@ { - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# AzureML in a Day\n", - "\n", - "Learn how a data scientist uses Azure Machine Learning (Azure ML) to train a model, then use the model for prediction. This tutorial will help you become familiar with the core concepts of Azure ML and their most common usage. \n", - "\n", - "You'll learn how to submit a *command job* to run your *training script*, configured with the *job environment* necessary to run the script.\n", - "\n", - "The training script handles the data preparation, then trains and registers a model. Once you have the model, you'll *deploy* it as an *endpoint*, then call the endpoint for *inferencing*.\n", - "\n", - "The steps you'll take are:\n", - "\n", - "> * Connect to your Azure ML workspace\n", - "> * Create your job environment\n", - "> * Create your training script\n", - "> * Create and run your command job to run the training script, configured with the appropriate job environment\n", - "> * View the output of your training script\n", - "> * Deploy the newly-trained model as an endpoint\n", - "> * Call the Azure ML endpoint for inferencing" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisites\n", - "\n", - "* An Azure subscription. If you don't have an Azure subscription, [create a free account](https://aka.ms/AMLFree) before you begin.\n", - "* A working Azure ML workspace. A workspace can be created via Azure Portal, Azure CLI, or Python SDK. [Read more](https://docs.microsoft.com/azure/machine-learning/how-to-manage-workspace?tabs=python).\n", - "* An Azure Machine Learning [workspace]()\n", - "* A workspace and compute instance which you can create by completing the [Quickstart: Get started with Azure Machine Learning](https://docs.microsoft.com/azure/machine-learning/quickstart-create-resources#create-compute-instance)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Connect to the workspace\n", - "\n", - "Before you dive in the code, you'll need to connect to your Azure ML workspace. The workspace is the top-level resource for Azure Machine Learning, providing a centralized place to work with all the artifacts you create when you use Azure Machine Learning.\n", - "\n", - "We're using `DefaultAzureCredential` to get access to workspace. \n", - "`DefaultAzureCredential` is used to handle most Azure SDK authentication scenarios. \n", - "\n", - "Reference for more available credentials if it doesn't work for you: [configure credential example](../../configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/python/api/azure-identity/azure.identity?view=azure-python)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "credential" - }, - "outputs": [], - "source": [ - "# Handle to the workspace\n", - "from azure.ai.ml import MLClient\n", - "\n", - "# Authentication package\n", - "from azure.identity import DefaultAzureCredential\n", - "\n", - "credential = DefaultAzureCredential()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If you want to use a browser to login and authenticate, you can use the following code instead. In this example, you'll use the `DefaultAzureCredential`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Handle to the workspace\n", - "# from azure.ai.ml import MLClient\n", - "\n", - "# Authentication package\n", - "# from azure.identity import InteractiveBrowserCredential\n", - "# credential = InteractiveBrowserCredential()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the next cell, enter your Subscription ID, Resource Group name and Workspace name. To find these values:\n", - "\n", - "1. In the upper right Azure Machine Learning studio toolbar, select your workspace name.\n", - "1. Copy the value for workspace, resource group and subscription ID into the code. \n", - "1. You'll need to copy one value, close the area and paste, then come back for the next one.\n", - "\n", - "![image of workspace credentials](media\\find-credentials.png)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "ml_client" - }, - "outputs": [], - "source": [ - "# Get a handle to the workspace\n", - "ml_client = MLClient(\n", - " credential=credential,\n", - " subscription_id=\"\",\n", - " resource_group_name=\"\",\n", - " workspace_name=\"\",\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The result is a handler to the workspace that you'll use to manage other resources and jobs.\n", - "\n", - "> [!IMPORTANT]\n", - "> Creating MLClient will not connect to the workspace. The client initialization is lazy, it will wait for the first time it needs to make a call (in the notebook below, that will happen during job environment creation)." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create a job environment\n", - "\n", - "To run your AzureML job, you'll need an [environment](https://docs.microsoft.com/azure/machine-learning/concept-environments). An environment lists the software runtime and libraries that you want installed on the compute where you’ll be training. It's similar to your Python environment on your local machine.\n", - "\n", - "AzureML provides many curated or ready-made environments, which are useful for common training and inference scenarios. You can also create your own custom environments using a docker image, or a conda configuration.\n", - "\n", - "In this example, you'll create a custom conda environment for your jobs, using a conda yaml file.\n", - "\n", - "First, create a directory to store the file in." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "dependencies_dir" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "dependencies_dir = \"./dependencies\"\n", - "os.makedirs(dependencies_dir, exist_ok=True)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, create the file in the dependencies directory. The cell below uses IPython magic to write the file into the directory you just created." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "write_model" - }, - "outputs": [], - "source": [ - "%%writefile {dependencies_dir}/conda.yaml\n", - "name: model-env\n", - "channels:\n", - " - conda-forge\n", - "dependencies:\n", - " - python=3.10\n", - " - numpy=1.26.4\n", - " - pip=24.0\n", - " - scikit-learn=1.5.1\n", - " - scipy=1.10\n", - " - pandas=1.5.3\n", - " - pip:\n", - " - inference-schema[numpy-support]==1.3.0\n", - " - xlrd==2.0.1\n", - " - mlflow\n", - " - azureml-mlflow\n", - " - psutil\n", - " - tqdm\n", - " - ipykernel~=6.29\n", - " - matplotlib" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "The specification contains some usual packages, that you'll use in your job (numpy, pip).\n", - "\n", - "Reference this *yaml* file to create and register this custom environment in your workspace:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "custom_env_name" - }, - "outputs": [], - "source": [ - "from azure.ai.ml.entities import Environment\n", - "\n", - "custom_env_name = \"aml-scikit-learn\"\n", - "\n", - "pipeline_job_env = Environment(\n", - " name=custom_env_name,\n", - " description=\"Custom environment for Credit Card Defaults pipeline\",\n", - " tags={\"scikit-learn\": \"1.5.1\"},\n", - " conda_file=os.path.join(dependencies_dir, \"conda.yaml\"),\n", - " image=\"mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04:latest\",\n", - ")\n", - "pipeline_job_env = ml_client.environments.create_or_update(pipeline_job_env)\n", - "\n", - "print(\n", - " f\"Environment with name {pipeline_job_env.name} is registered to workspace, the environment version is {pipeline_job_env.version}\"\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## What is a command job?\n", - "\n", - "You'll create an Azure ML *command job* to train a model for credit default prediction. The command job is used to run a *training script* in a specified environment on serverless compute. You've already created the environment. Next you'll create the training script.\n", - "\n", - "The *training script* handles the data preparation, training and registering of the trained model. In this tutorial, you'll create a Python training script.\n", - "\n", - "Command jobs can be run from CLI, Python SDK, or studio interface. In this tutorial, you'll use the Azure ML Python SDK v2 to create and run the command job.\n", - "\n", - "After running the training job, you'll deploy the model, then use it to produce a prediction." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create training script\n", - "\n", - "Let's start by creating the training script - the *main.py* Python file.\n", - "\n", - "First create a source folder for the script:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "train_src_dir" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "train_src_dir = \"./src\"\n", - "os.makedirs(train_src_dir, exist_ok=True)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This script handles the preprocessing of the data, splitting it into test and train data. It then consumes this data to train a tree based model and return the output model. \n", - "\n", - "[MLFlow](https://learn.microsoft.com/azure/machine-learning/how-to-log-mlflow-models) will be used to log the parameters and metrics during our pipeline run. \n", - "\n", - "The cell below uses IPython magic to write the training script into the directory you just created." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "write_main" - }, - "outputs": [], - "source": [ - "%%writefile {train_src_dir}/main.py\n", - "import os\n", - "import argparse\n", - "import pandas as pd\n", - "import mlflow\n", - "import mlflow.sklearn\n", - "from sklearn.ensemble import GradientBoostingClassifier\n", - "from sklearn.metrics import classification_report\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "def main():\n", - " \"\"\"Main function of the script.\"\"\"\n", - "\n", - " # input and output arguments\n", - " parser = argparse.ArgumentParser()\n", - " parser.add_argument(\"--data\", type=str, help=\"path to input data\")\n", - " parser.add_argument(\"--test_train_ratio\", type=float, required=False, default=0.25)\n", - " parser.add_argument(\"--n_estimators\", required=False, default=100, type=int)\n", - " parser.add_argument(\"--learning_rate\", required=False, default=0.1, type=float)\n", - " parser.add_argument(\"--registered_model_name\", type=str, help=\"model name\")\n", - " args = parser.parse_args()\n", - " \n", - " # Start Logging\n", - " mlflow.start_run()\n", - "\n", - " # enable autologging\n", - " mlflow.sklearn.autolog()\n", - "\n", - " ###################\n", - " #\n", - " ###################\n", - " print(\" \".join(f\"{k}={v}\" for k, v in vars(args).items()))\n", - "\n", - " print(\"input data:\", args.data)\n", - " \n", - " credit_df = pd.read_csv(args.data, header=1, index_col=0)\n", - "\n", - " mlflow.log_metric(\"num_samples\", credit_df.shape[0])\n", - " mlflow.log_metric(\"num_features\", credit_df.shape[1] - 1)\n", - "\n", - " train_df, test_df = train_test_split(\n", - " credit_df,\n", - " test_size=args.test_train_ratio,\n", - " )\n", - " ####################\n", - " #\n", - " ####################\n", - "\n", - " ##################\n", - " #\n", - " ##################\n", - " # Extracting the label column\n", - " y_train = train_df.pop(\"default payment next month\")\n", - "\n", - " # convert the dataframe values to array\n", - " X_train = train_df.values\n", - "\n", - " # Extracting the label column\n", - " y_test = test_df.pop(\"default payment next month\")\n", - "\n", - " # convert the dataframe values to array\n", - " X_test = test_df.values\n", - "\n", - " print(f\"Training with data of shape {X_train.shape}\")\n", - "\n", - " clf = GradientBoostingClassifier(\n", - " n_estimators=args.n_estimators, learning_rate=args.learning_rate\n", - " )\n", - " clf.fit(X_train, y_train)\n", - "\n", - " y_pred = clf.predict(X_test)\n", - "\n", - " print(classification_report(y_test, y_pred))\n", - " ###################\n", - " #\n", - " ###################\n", - "\n", - " ##########################\n", - " #\n", - " ##########################\n", - " # Registering the model to the workspace\n", - " print(\"Registering the model via MLFlow\")\n", - " mlflow.sklearn.log_model(\n", - " sk_model=clf,\n", - " registered_model_name=args.registered_model_name,\n", - " artifact_path=args.registered_model_name,\n", - " )\n", - "\n", - " # Saving the model to a file\n", - " mlflow.sklearn.save_model(\n", - " sk_model=clf,\n", - " path=os.path.join(args.registered_model_name, \"trained_model\"),\n", - " )\n", - " ###########################\n", - " #\n", - " ###########################\n", - " \n", - " # Stop Logging\n", - " mlflow.end_run()\n", - "\n", - "if __name__ == \"__main__\":\n", - " main()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As you can see in this script, once the model is trained, the model file is saved and registered to the workspace. Now you can use the registered model in inferencing endpoints.\n", - "\n", - "## Configure the command\n", - "\n", - "Now that you have a script that can perform the desired tasks, you'll use the general purpose **command** that can run command line actions. This command line action can be directly calling system commands or by running a script. \n", - "\n", - "Here, you'll create input variables to specify the input data, split ratio, learning rate and registered model name. The command script will:\n", - "* Use the environment created earlier - you can use the `@latest` notation to indicate the latest version of the environment when the command is run.\n", - "* Configure some metadata like display name, experiment name etc. An *experiment* is a container for all the iterations you do on a certain project. All the jobs submitted under the same experiment name would be listed next to each other in Azure ML studio.\n", - "* Configure the command line action itself - `python main.py` in this case. The inputs/outputs are accessible in the command via the `${{ ... }}` notation.\n", - "* In this sample, we access the data from a file on the internet. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "registered_model_name" - }, - "outputs": [], - "source": [ - "from azure.ai.ml import command\n", - "from azure.ai.ml import Input\n", - "\n", - "registered_model_name = \"credit_defaults_model\"\n", - "\n", - "job = command(\n", - " inputs=dict(\n", - " data=Input(\n", - " type=\"uri_file\",\n", - " path=\"https://azuremlexamples.blob.core.windows.net/datasets/credit_card/default%20of%20credit%20card%20clients.csv\",\n", - " ),\n", - " test_train_ratio=0.2,\n", - " learning_rate=0.25,\n", - " registered_model_name=registered_model_name,\n", - " ),\n", - " code=\"./src/\", # location of source code\n", - " command=\"python main.py --data ${{inputs.data}} --test_train_ratio ${{inputs.test_train_ratio}} --learning_rate ${{inputs.learning_rate}} --registered_model_name ${{inputs.registered_model_name}}\",\n", - " environment=\"aml-scikit-learn@latest\",\n", - " experiment_name=\"train_model_credit_default_prediction\",\n", - " display_name=\"credit_default_prediction\",\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Submit the job \n", - "\n", - "It's now time to submit the job to run in AzureML. This time you'll use `create_or_update` on `ml_client.jobs`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "create_job" - }, - "outputs": [], - "source": [ - "ml_client.create_or_update(job)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## View job output and wait for job completion\n", - "\n", - "View the job in Azure ML studio by selecting the link in the output of the previous cell. \n", - "\n", - "The output of this job will look like this in Azure ML studio. Explore the tabs for various details like metrics, outputs etc. Once completed, the job will register a model in your workspace as a result of training. \n", - "\n", - "![Screenshot that shows the job overview](media/view-job.gif \"View the job in studio\")\n", - "\n", - "> [!IMPORTANT]\n", - "> Wait until the status of the job is complete before returning to this notebook to continue. The job will take 2 to 3 minutes to run. It could take longer (up to 10 minutes) if the compute has been scaled down to zero nodes and custom environment is still building.\n", - "\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Deploy the model as an online endpoint\n", - "\n", - "Now deploy your machine learning model as a web service in the Azure cloud, an [`online endpoint`](https://docs.microsoft.com/azure/machine-learning/concept-endpoints).\n", - "\n", - "To deploy a machine learning service, you usually need:\n", - "\n", - "* The model assets (file, metadata) that you want to deploy. You've already registered these assets in your training job.\n", - "* Some code to run as a service. The code executes the model on a given input request. This entry script receives data submitted to a deployed web service and passes it to the model, then returns the model's response to the client. The script is specific to your model. The entry script must understand the data that the model expects and returns. With an MLFlow model, as in this tutorial, this script is automatically created for you. Samples of scoring scripts can be found [here](https://github.com/Azure/azureml-examples/tree/sdk-preview/sdk/endpoints/online).\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create a new online endpoint\n", - "\n", - "Now that you have a registered model and an inference script, it's time to create your online endpoint. The endpoint name needs to be unique in the entire Azure region. For this tutorial, you'll create a unique name using [`UUID`](https://en.wikipedia.org/wiki/Universally_unique_identifier#:~:text=A%20universally%20unique%20identifier%20(UUID,%2C%20for%20practical%20purposes%2C%20unique.)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "online_endpoint_name" - }, - "outputs": [], - "source": [ - "import uuid\n", - "\n", - "# Creating a unique name for the endpoint\n", - "online_endpoint_name = \"credit-endpoint-\" + str(uuid.uuid4())[:8]" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> [!NOTE]\n", - "> Expect the endpoint creation to take approximately 6 to 8 minutes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "endpoint" - }, - "outputs": [], - "source": [ - "from azure.ai.ml.entities import (\n", - " ManagedOnlineEndpoint,\n", - " ManagedOnlineDeployment,\n", - " Model,\n", - " Environment,\n", - ")\n", - "\n", - "# create an online endpoint\n", - "endpoint = ManagedOnlineEndpoint(\n", - " name=online_endpoint_name,\n", - " description=\"this is an online endpoint\",\n", - " auth_mode=\"key\",\n", - " tags={\n", - " \"training_dataset\": \"credit_defaults\",\n", - " \"model_type\": \"sklearn.GradientBoostingClassifier\",\n", - " },\n", - ")\n", - "\n", - "endpoint = ml_client.online_endpoints.begin_create_or_update(endpoint).result()\n", - "\n", - "print(f\"Endpoint {endpoint.name} provisioning state: {endpoint.provisioning_state}\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Once you've created an endpoint, you can retrieve it as below:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "retrieve_endpoint" - }, - "outputs": [], - "source": [ - "endpoint = ml_client.online_endpoints.get(name=online_endpoint_name)\n", - "\n", - "print(\n", - " f'Endpoint \"{endpoint.name}\" with provisioning state \"{endpoint.provisioning_state}\" is retrieved'\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Deploy the model to the endpoint\n", - "\n", - "Once the endpoint is created, deploy the model with the entry script. Each endpoint can have multiple deployments. Direct traffic to these deployments can be specified using rules. Here you'll create a single deployment that handles 100% of the incoming traffic. We have chosen a color name for the deployment, for example, *blue*, *green*, *red* deployments, which is arbitrary.\n", - "\n", - "You can check the **Models** page on Azure ML studio, to identify the latest version of your registered model. Alternatively, the code below will retrieve the latest version number for you to use." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "latest_model_version" - }, - "outputs": [], - "source": [ - "# Let's pick the latest version of the model\n", - "latest_model_version = max(\n", - " [int(m.version) for m in ml_client.models.list(name=registered_model_name)]\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Deploy the latest version of the model. \n", - "\n", - "> [!NOTE]\n", - "> Expect this deployment to take approximately 6 to 8 minutes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "blue_deployment" - }, - "outputs": [], - "source": [ - "# picking the model to deploy. Here we use the latest version of our registered model\n", - "model = ml_client.models.get(name=registered_model_name, version=latest_model_version)\n", - "\n", - "\n", - "# create an online deployment.\n", - "blue_deployment = ManagedOnlineDeployment(\n", - " name=\"blue\",\n", - " endpoint_name=online_endpoint_name,\n", - " model=model,\n", - " instance_type=\"Standard_DS3_v2\",\n", - " instance_count=1,\n", - ")\n", - "\n", - "blue_deployment = ml_client.begin_create_or_update(blue_deployment).result()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Test with a sample query\n", - "\n", - "Now that the model is deployed to the endpoint, you can run inference with it.\n", - "\n", - "Create a sample request file following the design expected in the run method in the score script." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "deploy_dir" - }, - "outputs": [], - "source": [ - "deploy_dir = \"./deploy\"\n", - "os.makedirs(deploy_dir, exist_ok=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "write_sample" - }, - "outputs": [], - "source": [ - "%%writefile {deploy_dir}/sample-request.json\n", - "{\n", - " \"input_data\": {\n", - " \"columns\": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22],\n", - " \"index\": [0, 1],\n", - " \"data\": [\n", - " [20000,2,2,1,24,2,2,-1,-1,-2,-2,3913,3102,689,0,0,0,0,689,0,0,0,0],\n", - " [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, 9, 8]\n", - " ]\n", - " }\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "test" - }, - "outputs": [], - "source": [ - "# test the blue deployment with some sample data\n", - "ml_client.online_endpoints.invoke(\n", - " endpoint_name=online_endpoint_name,\n", - " request_file=\"./deploy/sample-request.json\",\n", - " deployment_name=\"blue\",\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Clean up resources\n", - "\n", - "If you're not going to use the endpoint, delete it to stop using the resource. Make sure no other deployments are using an endpoint before you delete it.\n", - "\n", - "\n", - "> [!NOTE]\n", - "> Expect this step to take approximately 6 to 8 minutes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "delete_endpoint" - }, - "outputs": [], - "source": [ - "ml_client.online_endpoints.begin_delete(name=online_endpoint_name)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Next Steps\n", - "\n", - "Learn about creating a multi step pipeline for this script [Create production ML pipelines in a Jupyter notebook](https://github.com/Azure/azureml-examples/blob/main/tutorials/e2e-ds-experience/e2e-ml-workflow.ipynb)." - ] - } - ], - "metadata": { - "description": { - "description": "Learn how a data scientist uses Azure Machine Learning (Azure ML) to train a model, then use the model for prediction. This tutorial will help you become familiar with the core concepts of Azure ML and their most common usage." - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# AzureML in a Day\n", + "\n", + "Learn how a data scientist uses Azure Machine Learning (Azure ML) to train a model, then use the model for prediction. This tutorial will help you become familiar with the core concepts of Azure ML and their most common usage. \n", + "\n", + "You'll learn how to submit a *command job* to run your *training script*, configured with the *job environment* necessary to run the script.\n", + "\n", + "The training script handles the data preparation, then trains and registers a model. Once you have the model, you'll *deploy* it as an *endpoint*, then call the endpoint for *inferencing*.\n", + "\n", + "The steps you'll take are:\n", + "\n", + "> * Connect to your Azure ML workspace\n", + "> * Create your job environment\n", + "> * Create your training script\n", + "> * Create and run your command job to run the training script, configured with the appropriate job environment\n", + "> * View the output of your training script\n", + "> * Deploy the newly-trained model as an endpoint\n", + "> * Call the Azure ML endpoint for inferencing" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "\n", + "* An Azure subscription. If you don't have an Azure subscription, [create a free account](https://aka.ms/AMLFree) before you begin.\n", + "* A working Azure ML workspace. A workspace can be created via Azure Portal, Azure CLI, or Python SDK. [Read more](https://docs.microsoft.com/azure/machine-learning/how-to-manage-workspace?tabs=python).\n", + "* An Azure Machine Learning [workspace]()\n", + "* A workspace and compute instance which you can create by completing the [Quickstart: Get started with Azure Machine Learning](https://docs.microsoft.com/azure/machine-learning/quickstart-create-resources#create-compute-instance)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Connect to the workspace\n", + "\n", + "Before you dive in the code, you'll need to connect to your Azure ML workspace. The workspace is the top-level resource for Azure Machine Learning, providing a centralized place to work with all the artifacts you create when you use Azure Machine Learning.\n", + "\n", + "We're using `DefaultAzureCredential` to get access to workspace. \n", + "`DefaultAzureCredential` is used to handle most Azure SDK authentication scenarios. \n", + "\n", + "Reference for more available credentials if it doesn't work for you: [configure credential example](../../configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/python/api/azure-identity/azure.identity?view=azure-python)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "credential" + }, + "outputs": [], + "source": [ + "# Handle to the workspace\n", + "from azure.ai.ml import MLClient\n", + "\n", + "# Authentication package\n", + "from azure.identity import DefaultAzureCredential\n", + "\n", + "credential = DefaultAzureCredential()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you want to use a browser to login and authenticate, you can use the following code instead. In this example, you'll use the `DefaultAzureCredential`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Handle to the workspace\n", + "# from azure.ai.ml import MLClient\n", + "\n", + "# Authentication package\n", + "# from azure.identity import InteractiveBrowserCredential\n", + "# credential = InteractiveBrowserCredential()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the next cell, enter your Subscription ID, Resource Group name and Workspace name. To find these values:\n", + "\n", + "1. In the upper right Azure Machine Learning studio toolbar, select your workspace name.\n", + "1. Copy the value for workspace, resource group and subscription ID into the code. \n", + "1. You'll need to copy one value, close the area and paste, then come back for the next one.\n", + "\n", + "![image of workspace credentials](media\\find-credentials.png)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "ml_client" + }, + "outputs": [], + "source": [ + "# Get a handle to the workspace\n", + "ml_client = MLClient(\n", + " credential=credential,\n", + " subscription_id=\"\",\n", + " resource_group_name=\"\",\n", + " workspace_name=\"\",\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The result is a handler to the workspace that you'll use to manage other resources and jobs.\n", + "\n", + "> [!IMPORTANT]\n", + "> Creating MLClient will not connect to the workspace. The client initialization is lazy, it will wait for the first time it needs to make a call (in the notebook below, that will happen during job environment creation)." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create a job environment\n", + "\n", + "To run your AzureML job, you'll need an [environment](https://docs.microsoft.com/azure/machine-learning/concept-environments). An environment lists the software runtime and libraries that you want installed on the compute where you’ll be training. It's similar to your Python environment on your local machine.\n", + "\n", + "AzureML provides many curated or ready-made environments, which are useful for common training and inference scenarios. You can also create your own custom environments using a docker image, or a conda configuration.\n", + "\n", + "In this example, you'll create a custom conda environment for your jobs, using a conda yaml file.\n", + "\n", + "First, create a directory to store the file in." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "dependencies_dir" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "dependencies_dir = \"./dependencies\"\n", + "os.makedirs(dependencies_dir, exist_ok=True)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, create the file in the dependencies directory. The cell below uses IPython magic to write the file into the directory you just created." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "write_model" + }, + "outputs": [], + "source": [ + "%%writefile {dependencies_dir}/conda.yaml\n", + "name: model-env\n", + "channels:\n", + " - conda-forge\n", + "dependencies:\n", + " - python=3.10\n", + " - numpy=1.26.4\n", + " - pip=24.0\n", + " - scikit-learn=1.5.1\n", + " - scipy=1.10\n", + " - pandas=1.5.3\n", + " - pip:\n", + " - inference-schema[numpy-support]==1.3.0\n", + " - xlrd==2.0.1\n", + " - mlflow\n", + " - azureml-mlflow\n", + " - psutil\n", + " - tqdm\n", + " - ipykernel~=6.29\n", + " - matplotlib" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "The specification contains some usual packages, that you'll use in your job (numpy, pip).\n", + "\n", + "Reference this *yaml* file to create and register this custom environment in your workspace:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "custom_env_name" + }, + "outputs": [], + "source": [ + "from azure.ai.ml.entities import Environment\n", + "\n", + "custom_env_name = \"aml-scikit-learn\"\n", + "\n", + "pipeline_job_env = Environment(\n", + " name=custom_env_name,\n", + " description=\"Custom environment for Credit Card Defaults pipeline\",\n", + " tags={\"scikit-learn\": \"1.5.1\"},\n", + " conda_file=os.path.join(dependencies_dir, \"conda.yaml\"),\n", + " image=\"mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04:latest\",\n", + ")\n", + "pipeline_job_env = ml_client.environments.create_or_update(pipeline_job_env)\n", + "\n", + "print(\n", + " f\"Environment with name {pipeline_job_env.name} is registered to workspace, the environment version is {pipeline_job_env.version}\"\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## What is a command job?\n", + "\n", + "You'll create an Azure ML *command job* to train a model for credit default prediction. The command job is used to run a *training script* in a specified environment on serverless compute. You've already created the environment. Next you'll create the training script.\n", + "\n", + "The *training script* handles the data preparation, training and registering of the trained model. In this tutorial, you'll create a Python training script.\n", + "\n", + "Command jobs can be run from CLI, Python SDK, or studio interface. In this tutorial, you'll use the Azure ML Python SDK v2 to create and run the command job.\n", + "\n", + "After running the training job, you'll deploy the model, then use it to produce a prediction." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create training script\n", + "\n", + "Let's start by creating the training script - the *main.py* Python file.\n", + "\n", + "First create a source folder for the script:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "train_src_dir" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "train_src_dir = \"./src\"\n", + "os.makedirs(train_src_dir, exist_ok=True)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This script handles the preprocessing of the data, splitting it into test and train data. It then consumes this data to train a tree based model and return the output model. \n", + "\n", + "[MLFlow](https://learn.microsoft.com/azure/machine-learning/how-to-log-mlflow-models) will be used to log the parameters and metrics during our pipeline run. \n", + "\n", + "The cell below uses IPython magic to write the training script into the directory you just created." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "write_main" + }, + "outputs": [], + "source": [ + "%%writefile {train_src_dir}/main.py\n", + "import os\n", + "import argparse\n", + "import pandas as pd\n", + "import mlflow\n", + "import mlflow.sklearn\n", + "from sklearn.ensemble import GradientBoostingClassifier\n", + "from sklearn.metrics import classification_report\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "def main():\n", + " \"\"\"Main function of the script.\"\"\"\n", + "\n", + " # input and output arguments\n", + " parser = argparse.ArgumentParser()\n", + " parser.add_argument(\"--data\", type=str, help=\"path to input data\")\n", + " parser.add_argument(\"--test_train_ratio\", type=float, required=False, default=0.25)\n", + " parser.add_argument(\"--n_estimators\", required=False, default=100, type=int)\n", + " parser.add_argument(\"--learning_rate\", required=False, default=0.1, type=float)\n", + " parser.add_argument(\"--registered_model_name\", type=str, help=\"model name\")\n", + " args = parser.parse_args()\n", + " \n", + " # Start Logging\n", + " mlflow.start_run()\n", + "\n", + " # enable autologging\n", + " mlflow.sklearn.autolog()\n", + "\n", + " ###################\n", + " #\n", + " ###################\n", + " print(\" \".join(f\"{k}={v}\" for k, v in vars(args).items()))\n", + "\n", + " print(\"input data:\", args.data)\n", + " \n", + " credit_df = pd.read_csv(args.data, header=1, index_col=0)\n", + "\n", + " mlflow.log_metric(\"num_samples\", credit_df.shape[0])\n", + " mlflow.log_metric(\"num_features\", credit_df.shape[1] - 1)\n", + "\n", + " train_df, test_df = train_test_split(\n", + " credit_df,\n", + " test_size=args.test_train_ratio,\n", + " )\n", + " ####################\n", + " #\n", + " ####################\n", + "\n", + " ##################\n", + " #\n", + " ##################\n", + " # Extracting the label column\n", + " y_train = train_df.pop(\"default payment next month\")\n", + "\n", + " # convert the dataframe values to array\n", + " X_train = train_df.values\n", + "\n", + " # Extracting the label column\n", + " y_test = test_df.pop(\"default payment next month\")\n", + "\n", + " # convert the dataframe values to array\n", + " X_test = test_df.values\n", + "\n", + " print(f\"Training with data of shape {X_train.shape}\")\n", + "\n", + " clf = GradientBoostingClassifier(\n", + " n_estimators=args.n_estimators, learning_rate=args.learning_rate\n", + " )\n", + " clf.fit(X_train, y_train)\n", + "\n", + " y_pred = clf.predict(X_test)\n", + "\n", + " print(classification_report(y_test, y_pred))\n", + " ###################\n", + " #\n", + " ###################\n", + "\n", + " ##########################\n", + " #\n", + " ##########################\n", + " # Registering the model to the workspace\n", + " print(\"Registering the model via MLFlow\")\n", + " mlflow.sklearn.log_model(\n", + " sk_model=clf,\n", + " registered_model_name=args.registered_model_name,\n", + " artifact_path=args.registered_model_name,\n", + " )\n", + "\n", + " # Saving the model to a file\n", + " mlflow.sklearn.save_model(\n", + " sk_model=clf,\n", + " path=os.path.join(args.registered_model_name, \"trained_model\"),\n", + " )\n", + " ###########################\n", + " #\n", + " ###########################\n", + " \n", + " # Stop Logging\n", + " mlflow.end_run()\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see in this script, once the model is trained, the model file is saved and registered to the workspace. Now you can use the registered model in inferencing endpoints.\n", + "\n", + "## Configure the command\n", + "\n", + "Now that you have a script that can perform the desired tasks, you'll use the general purpose **command** that can run command line actions. This command line action can be directly calling system commands or by running a script. \n", + "\n", + "Here, you'll create input variables to specify the input data, split ratio, learning rate and registered model name. The command script will:\n", + "* Use the environment created earlier - you can use the `@latest` notation to indicate the latest version of the environment when the command is run.\n", + "* Configure some metadata like display name, experiment name etc. An *experiment* is a container for all the iterations you do on a certain project. All the jobs submitted under the same experiment name would be listed next to each other in Azure ML studio.\n", + "* Configure the command line action itself - `python main.py` in this case. The inputs/outputs are accessible in the command via the `${{ ... }}` notation.\n", + "* In this sample, we access the data from a file on the internet. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "registered_model_name" + }, + "outputs": [], + "source": [ + "from azure.ai.ml import command\n", + "from azure.ai.ml import Input\n", + "\n", + "registered_model_name = \"credit_defaults_model\"\n", + "\n", + "job = command(\n", + " inputs=dict(\n", + " data=Input(\n", + " type=\"uri_file\",\n", + " path=\"https://azuremlexamples.blob.core.windows.net/datasets/credit_card/default%20of%20credit%20card%20clients.csv\",\n", + " ),\n", + " test_train_ratio=0.2,\n", + " learning_rate=0.25,\n", + " registered_model_name=registered_model_name,\n", + " ),\n", + " code=\"./src/\", # location of source code\n", + " command=\"python main.py --data ${{inputs.data}} --test_train_ratio ${{inputs.test_train_ratio}} --learning_rate ${{inputs.learning_rate}} --registered_model_name ${{inputs.registered_model_name}}\",\n", + " environment=\"aml-scikit-learn@latest\",\n", + " experiment_name=\"train_model_credit_default_prediction\",\n", + " display_name=\"credit_default_prediction\",\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Submit the job \n", + "\n", + "It's now time to submit the job to run in AzureML. This time you'll use `create_or_update` on `ml_client.jobs`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "create_job" + }, + "outputs": [], + "source": [ + "ml_client.create_or_update(job)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## View job output and wait for job completion\n", + "\n", + "View the job in Azure ML studio by selecting the link in the output of the previous cell. \n", + "\n", + "The output of this job will look like this in Azure ML studio. Explore the tabs for various details like metrics, outputs etc. Once completed, the job will register a model in your workspace as a result of training. \n", + "\n", + "![Screenshot that shows the job overview](media/view-job.gif \"View the job in studio\")\n", + "\n", + "> [!IMPORTANT]\n", + "> Wait until the status of the job is complete before returning to this notebook to continue. The job will take 2 to 3 minutes to run. It could take longer (up to 10 minutes) if the compute has been scaled down to zero nodes and custom environment is still building.\n", + "\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deploy the model as an online endpoint\n", + "\n", + "Now deploy your machine learning model as a web service in the Azure cloud, an [`online endpoint`](https://docs.microsoft.com/azure/machine-learning/concept-endpoints).\n", + "\n", + "To deploy a machine learning service, you usually need:\n", + "\n", + "* The model assets (file, metadata) that you want to deploy. You've already registered these assets in your training job.\n", + "* Some code to run as a service. The code executes the model on a given input request. This entry script receives data submitted to a deployed web service and passes it to the model, then returns the model's response to the client. The script is specific to your model. The entry script must understand the data that the model expects and returns. With an MLFlow model, as in this tutorial, this script is automatically created for you. Samples of scoring scripts can be found [here](https://github.com/Azure/azureml-examples/tree/sdk-preview/sdk/endpoints/online).\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create a new online endpoint\n", + "\n", + "Now that you have a registered model and an inference script, it's time to create your online endpoint. The endpoint name needs to be unique in the entire Azure region. For this tutorial, you'll create a unique name using [`UUID`](https://en.wikipedia.org/wiki/Universally_unique_identifier#:~:text=A%20universally%20unique%20identifier%20(UUID,%2C%20for%20practical%20purposes%2C%20unique.)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "online_endpoint_name" + }, + "outputs": [], + "source": [ + "import uuid\n", + "\n", + "# Creating a unique name for the endpoint\n", + "online_endpoint_name = \"credit-endpoint-\" + str(uuid.uuid4())[:8]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> [!NOTE]\n", + "> Expect the endpoint creation to take approximately 6 to 8 minutes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "endpoint" + }, + "outputs": [], + "source": [ + "from azure.ai.ml.entities import (\n", + " ManagedOnlineEndpoint,\n", + " ManagedOnlineDeployment,\n", + " Model,\n", + " Environment,\n", + ")\n", + "\n", + "# create an online endpoint\n", + "endpoint = ManagedOnlineEndpoint(\n", + " name=online_endpoint_name,\n", + " description=\"this is an online endpoint\",\n", + " auth_mode=\"key\",\n", + " tags={\n", + " \"training_dataset\": \"credit_defaults\",\n", + " \"model_type\": \"sklearn.GradientBoostingClassifier\",\n", + " },\n", + ")\n", + "\n", + "endpoint = ml_client.online_endpoints.begin_create_or_update(endpoint).result()\n", + "\n", + "print(f\"Endpoint {endpoint.name} provisioning state: {endpoint.provisioning_state}\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Once you've created an endpoint, you can retrieve it as below:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "retrieve_endpoint" + }, + "outputs": [], + "source": [ + "endpoint = ml_client.online_endpoints.get(name=online_endpoint_name)\n", + "\n", + "print(\n", + " f'Endpoint \"{endpoint.name}\" with provisioning state \"{endpoint.provisioning_state}\" is retrieved'\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deploy the model to the endpoint\n", + "\n", + "Once the endpoint is created, deploy the model with the entry script. Each endpoint can have multiple deployments. Direct traffic to these deployments can be specified using rules. Here you'll create a single deployment that handles 100% of the incoming traffic. We have chosen a color name for the deployment, for example, *blue*, *green*, *red* deployments, which is arbitrary.\n", + "\n", + "You can check the **Models** page on Azure ML studio, to identify the latest version of your registered model. Alternatively, the code below will retrieve the latest version number for you to use." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "latest_model_version" + }, + "outputs": [], + "source": [ + "# Let's pick the latest version of the model\n", + "latest_model_version = max(\n", + " [int(m.version) for m in ml_client.models.list(name=registered_model_name)]\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Deploy the latest version of the model. \n", + "\n", + "> [!NOTE]\n", + "> Expect this deployment to take approximately 6 to 8 minutes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "blue_deployment" + }, + "outputs": [], + "source": [ + "# picking the model to deploy. Here we use the latest version of our registered model\n", + "model = ml_client.models.get(name=registered_model_name, version=latest_model_version)\n", + "\n", + "\n", + "# create an online deployment.\n", + "blue_deployment = ManagedOnlineDeployment(\n", + " name=\"blue\",\n", + " endpoint_name=online_endpoint_name,\n", + " model=model,\n", + " instance_type=\"Standard_DS3_v2\",\n", + " instance_count=1,\n", + ")\n", + "\n", + "blue_deployment = ml_client.begin_create_or_update(blue_deployment).result()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Test with a sample query\n", + "\n", + "Now that the model is deployed to the endpoint, you can run inference with it.\n", + "\n", + "Create a sample request file following the design expected in the run method in the score script." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "deploy_dir" + }, + "outputs": [], + "source": [ + "deploy_dir = \"./deploy\"\n", + "os.makedirs(deploy_dir, exist_ok=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "write_sample" + }, + "outputs": [], + "source": [ + "%%writefile {deploy_dir}/sample-request.json\n", + "{\n", + " \"input_data\": {\n", + " \"columns\": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22],\n", + " \"index\": [0, 1],\n", + " \"data\": [\n", + " [20000,2,2,1,24,2,2,-1,-1,-2,-2,3913,3102,689,0,0,0,0,689,0,0,0,0],\n", + " [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, 9, 8]\n", + " ]\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "test" + }, + "outputs": [], + "source": [ + "# test the blue deployment with some sample data\n", + "ml_client.online_endpoints.invoke(\n", + " endpoint_name=online_endpoint_name,\n", + " request_file=\"./deploy/sample-request.json\",\n", + " deployment_name=\"blue\",\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Clean up resources\n", + "\n", + "If you're not going to use the endpoint, delete it to stop using the resource. Make sure no other deployments are using an endpoint before you delete it.\n", + "\n", + "\n", + "> [!NOTE]\n", + "> Expect this step to take approximately 6 to 8 minutes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "delete_endpoint" + }, + "outputs": [], + "source": [ + "ml_client.online_endpoints.begin_delete(name=online_endpoint_name)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next Steps\n", + "\n", + "Learn about creating a multi step pipeline for this script [Create production ML pipelines in a Jupyter notebook](https://github.com/Azure/azureml-examples/blob/main/tutorials/e2e-ds-experience/e2e-ml-workflow.ipynb)." + ] + } + ], + "metadata": { + "description": { + "description": "Learn how a data scientist uses Azure Machine Learning (Azure ML) to train a model, then use the model for prediction. This tutorial will help you become familiar with the core concepts of Azure ML and their most common usage." + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK v2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/tutorials/e2e-ds-experience/e2e-ml-workflow.ipynb b/tutorials/e2e-ds-experience/e2e-ml-workflow.ipynb index 5fb2f881a5..7c400b745d 100644 --- a/tutorials/e2e-ds-experience/e2e-ml-workflow.ipynb +++ b/tutorials/e2e-ds-experience/e2e-ml-workflow.ipynb @@ -1,1326 +1,1326 @@ { - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Tutorial: Create production ML pipelines with Python SDK v2 (preview) in a Jupyter notebook\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Learning Objectives** - By the end of this two part tutorial, you should be able to use Azure Machine Learning (Azure ML) to productionize your ML project.\n", - "\n", - "This means you will be able to leverage the AzureML Python SDK to:\n", - "\n", - "- connect to your Azure ML workspace\n", - "- create Azure ML data assets\n", - "- create reusable Azure ML components\n", - "- create, validate and run Azure ML pipelines\n", - "- deploy the newly-trained model as an endpoint\n", - "- call the Azure ML endpoint for inferencing" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Motivations** - This tutorial is intended to introduce Azure ML to data scientists who want to scale up or publish their ML projects. By completing a familiar end-to-end project, which starts by loading the data and ends by creating and calling an online inference endpoint, the user should become familiar with the core concepts of Azure ML and their most common usage. Each step of this tutorial can be modified or performed in other ways that might have security or scalability advantages. We will cover some of those in the Part II of this tutorial, however, we suggest the reader use the provide links in each section to learn more on each topic." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Requirements** - In order to benefit from this tutorial, you need to have:\n", - "- basic understanding of Machine Learning projects workflow\n", - "- an Azure subscription. If you don't have an Azure subscription, [create a free account](https://aka.ms/AMLFree) before you begin.\n", - "- a working Azure ML workspace. A workspace can be created via Azure Portal, Azure CLI, or Python SDK. [Read more](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-manage-workspace?tabs=python).\n", - "- a Python environmnet\n", - "- [installed Azure Machine Learning Python SDK v2](https://github.com/Azure/azureml-examples/blob/sdk-preview/sdk/setup.sh)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Introduction\n", - "\n", - "In this tutorial, you'll create an Azure ML pipeline to train a model for credit default prediction. The pipeline handles the data preparation, training and registering the trained model. You'll then run the pipeline, deploy the model and use it.\n", - "\n", - "The image below shows the pipeline as you'll see it in the AzureML portal once submitted. It's a rather simple pipeline we'll use to walk you through the AzureML SDK v2.\n", - "\n", - "The two steps are first data preparation and second training. \n", - "\n", - "![Screenshot that shows the AML Pipeline](media/pipeline-overview.jpg \"Overview of the pipeline\")\n", - "\n", - "An AzureML pipeline that runs from local components, requires several dependent files. Fo better understanding of the project structure, we produce all these dependencies in the notebook cells. By the end of this tutorial, the project structure should look like:\n", - "```\n", - "e2e-ds-experience \n", - " components\n", - " data_prep\n", - " data_prep.py \n", - " train\n", - " train.py\n", - " train.yml \n", - " dependencies\n", - " conda.yaml\n", - " deploy\n", - " sample-request.json\n", - " media\n", - " metrics.jpg\n", - " pipeline-overview.jpg\n", - " user-logs.jpg\n", - " e2e-ml-workflow.ipynb\n", - "```\n", - "After running this notebook, you should be able to create the project direcetly in the IDE of your choice, instead.\n", - "\n", - "## Set up the pipeline resources\n", - "\n", - "The Azure ML framework can be used from CLI, Python SDK, or studio interface. In this example, you'll use the AzureML Python SDK v2 to create a pipeline. \n", - "\n", - "Before creating the pipeline, you'll set up the resources the pipeline will use:\n", - "\n", - "* The dataset for training\n", - "* The software environment to run the pipeline\n", - "\n", - "## Connect to the workspace\n", - "\n", - "Before we dive in the code, you'll need to connect to your Azure ML workspace. The workspace is the top-level resource for Azure Machine Learning, providing a centralized place to work with all the artifacts you create when you use Azure Machine Learning.\n", - "\n", - "We are using `DefaultAzureCredential` to get access to workspace. \n", - "`DefaultAzureCredential` should be capable of handling most Azure SDK authentication scenarios. \n", - "\n", - "Reference for more available credentials if it does not work for you: [configure credential example](../../configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "import-mlclient" - }, - "outputs": [], - "source": [ - "# Handle to the workspace\n", - "from azure.ai.ml import MLClient\n", - "\n", - "# Authentication package\n", - "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", - "\n", - "try:\n", - " credential = DefaultAzureCredential()\n", - " # Check if given credential can get token successfully.\n", - " credential.get_token(\"https://management.azure.com/.default\")\n", - "except Exception as ex:\n", - " # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work\n", - " credential = InteractiveBrowserCredential()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the next cell, enter your Subscription ID, Resource Group name and Workspace name. To find your Subscription ID:\n", - "1. In the upper right Azure Machine Learning Studio toolbar, select your workspace name.\n", - "1. At the bottom, select **View all properties in Azure Portal**\n", - "1. Copy the value from Azure Portal into the code." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "ml_client" - }, - "outputs": [], - "source": [ - "# Get a handle to the workspace\n", - "ml_client = MLClient(\n", - " credential=credential,\n", - " subscription_id=\"\",\n", - " resource_group_name=\"\",\n", - " workspace_name=\"\",\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The result is a handler to the workspace that you'll use to manage other resources and jobs.\n", - "\n", - "> [!IMPORTANT]\n", - "> Creating MLClient will not connect to the workspace. The client initialization is lazy, it will wait for the first time it needs to make a call (in the notebook below, that will happen during dataset registration).\n", - "\n", - "## Register data from an external url\n", - "\n", - "The data you use for training is usually in one of the locations below:\n", - "\n", - "* Local machine\n", - "* Web\n", - "* Big Data Storage services (for example, Azure Blob, Azure Data Lake Storage, SQL)\n", - " \n", - "Azure ML uses a [`Data`](https://docs.microsoft.com/azure/machine-learning/how-to-create-register-data-assets?tabs=Python-SDK) object to register a reusable definition of data, and consume data within a pipeline. In the section below, you'll consume some data from web url as one example. `Data` assets ets from other sources can be created as well." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "credit_data" - }, - "outputs": [], - "source": [ - "from azure.ai.ml.entities import Data\n", - "from azure.ai.ml.constants import AssetTypes\n", - "\n", - "web_path = \"https://azuremlexamples.blob.core.windows.net/datasets/credit_card/default%20of%20credit%20card%20clients.csv\"\n", - "\n", - "credit_data = Data(\n", - " name=\"creditcard_blob_csv_defaults\",\n", - " path=web_path,\n", - " type=AssetTypes.URI_FILE,\n", - " description=\"Dataset for credit card defaults\",\n", - " tags={\"source_type\": \"web\", \"source\": \"AzureML examples blob\"},\n", - " version=\"1.0.0\",\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This code just created a `Data` asset, it is ready to be consumed as an input by the pipeline that you'll define in the next sections. In addition, you can register the data to your workspace so it becomes reusable across pipelines.\n", - "\n", - "Registering the data asset will enable you to:\n", - "\n", - "* reuse and share the data asset in future pipelines\n", - "* use versions to track the modification to the data asset\n", - "* use the data asset from Azure ML designer, which is Azure ML's GUI for pipeline authoring\n", - "\n", - "Since this is the first time that you're making a call to the workspace, you may be asked to authenticate. Once the authentication is complete, you'll then see the dataset registration completion message." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "update-credit_data" - }, - "outputs": [], - "source": [ - "credit_data = ml_client.data.create_or_update(credit_data)\n", - "print(\n", - " f\"Dataset with name {credit_data.name} was registered to workspace, the dataset version is {credit_data.version}\"\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create a job environment for pipeline steps\n", - "\n", - "So far, you've created a development environment on the compute instance, your development machine. You'll also need an [environment](https://docs.microsoft.com/azure/machine-learning/concept-environments) to use for each step of the pipeline. Each step can have its own environment, or you can use some common environments for multiple steps.\n", - "\n", - "In this example, you'll create a conda environment for your jobs, using a conda yaml file.\n", - "First, create a directory to store the file in." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "dependencies_dir" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "dependencies_dir = \"./dependencies\"\n", - "os.makedirs(dependencies_dir, exist_ok=True)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, create the file in the dependencies directory." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "conda.yaml" - }, - "outputs": [], - "source": [ - "%%writefile {dependencies_dir}/conda.yaml\n", - "name: model-env\n", - "channels:\n", - " - conda-forge\n", - "dependencies:\n", - " - python=3.8\n", - " - numpy=1.21.2\n", - " - pip=21.2.4\n", - " - scikit-learn=0.24.2\n", - " - scipy=1.7.1\n", - " - pandas>=1.1,<1.2\n", - " - pip:\n", - " - inference-schema[numpy-support]==1.3.0\n", - " - xlrd==2.0.1\n", - " - azureml-mlflow==1.42.0\n", - " - mlflow==2.16.2" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The specification contains some usual packages, that you'll use in your pipeline (numpy, pip).\n", - "\n", - "\n", - "Use the *yaml* file to create and register this custom environment in your workspace:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "custom_env_name" - }, - "outputs": [], - "source": [ - "from azure.ai.ml.entities import Environment\n", - "\n", - "custom_env_name = \"aml-scikit-learn\"\n", - "\n", - "pipeline_job_env = Environment(\n", - " name=custom_env_name,\n", - " description=\"Custom environment for Credit Card Defaults pipeline\",\n", - " tags={\"scikit-learn\": \"0.24.2\"},\n", - " conda_file=os.path.join(dependencies_dir, \"conda.yaml\"),\n", - " image=\"mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest\",\n", - " version=\"0.1.8\",\n", - ")\n", - "pipeline_job_env = ml_client.environments.create_or_update(pipeline_job_env)\n", - "\n", - "print(\n", - " f\"Environment with name {pipeline_job_env.name} is registered to workspace, the environment version is {pipeline_job_env.version}\"\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Build the training pipeline\n", - "\n", - "Now that you have all assets required to run your pipeline, it's time to build the pipeline itself, using the Azure ML Python SDK v2.\n", - "\n", - "Azure ML pipelines are reusable ML workflows that usually consist of several components. The typical life of a component is:\n", - "\n", - "* Write the yaml specification of the component.\n", - "* Optionally, register the component with a name and version in your workspace, to make it reusable and shareable.\n", - "* Load that component from the pipeline code.\n", - "* Implement the pipeline using this component inputs, outputs and parameters.\n", - "* Submit the pipeline.\n", - "\n", - "## Create component 1: data prep (using programmatic definition)\n", - "\n", - "Let's start by creating the first component. This component handles the preprocessing of the data. The preprocessing task is performed in the *data_prep.py* python file.\n", - "\n", - "First create a source folder for the data_prep component:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "data_prep_src_dir" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "data_prep_src_dir = \"./components/data_prep\"\n", - "os.makedirs(data_prep_src_dir, exist_ok=True)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This script performs the simple task of splitting the data into train and test datasets. \n", - "\n", - "[MLFlow](https://mlflow.org/docs/latest/tracking.html) will be used to log the parameters and metrics during our pipeline run." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "def-main" - }, - "outputs": [], - "source": [ - "%%writefile {data_prep_src_dir}/data_prep.py\n", - "import os\n", - "import argparse\n", - "import pandas as pd\n", - "from sklearn.model_selection import train_test_split\n", - "import logging\n", - "import mlflow\n", - "\n", - "\n", - "def main():\n", - " \"\"\"Main function of the script.\"\"\"\n", - "\n", - " # input and output arguments\n", - " parser = argparse.ArgumentParser()\n", - " parser.add_argument(\"--data\", type=str, help=\"path to input data\")\n", - " parser.add_argument(\"--test_train_ratio\", type=float, required=False, default=0.25)\n", - " parser.add_argument(\"--train_data\", type=str, help=\"path to train data\")\n", - " parser.add_argument(\"--test_data\", type=str, help=\"path to test data\")\n", - " args = parser.parse_args()\n", - "\n", - " # Start Logging\n", - " mlflow.start_run()\n", - "\n", - " print(\" \".join(f\"{k}={v}\" for k, v in vars(args).items()))\n", - "\n", - " print(\"input data:\", args.data)\n", - "\n", - " credit_df = pd.read_csv(args.data, header=1, index_col=0)\n", - "\n", - " mlflow.log_metric(\"num_samples\", credit_df.shape[0])\n", - " mlflow.log_metric(\"num_features\", credit_df.shape[1] - 1)\n", - "\n", - " credit_train_df, credit_test_df = train_test_split(\n", - " credit_df,\n", - " test_size=args.test_train_ratio,\n", - " )\n", - "\n", - " # output paths are mounted as folder, therefore, we are adding a filename to the path\n", - " credit_train_df.to_csv(os.path.join(args.train_data, \"data.csv\"), index=False)\n", - "\n", - " credit_test_df.to_csv(os.path.join(args.test_data, \"data.csv\"), index=False)\n", - "\n", - " # Stop Logging\n", - " mlflow.end_run()\n", - "\n", - "\n", - "if __name__ == \"__main__\":\n", - " main()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now that you have a script that can perform the desired task, we can create an Azure ML Component from it. Azure ML support various types of components for performing ML tasks, such as running scripts, data transfer, etc.\n", - "\n", - "A component can be created by calling the component instantiators, or directly writing the defining yaml file. \n", - "\n", - "You'll use the general purpose **command** that can run command line actions. This command line action can be directly calling system commands or running a script. The inputs/outputs are accessible in the command via the `${{ ... }}` notation. For the second component of this tutorial you will use `yaml` definitions.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "data_prep_component" - }, - "outputs": [], - "source": [ - "from azure.ai.ml import command\n", - "from azure.ai.ml import Input, Output\n", - "\n", - "data_prep_component = command(\n", - " name=\"data_prep_credit_defaults\",\n", - " display_name=\"Data preparation for training\",\n", - " description=\"reads a .xl input, split the input to train and test\",\n", - " inputs={\n", - " \"data\": Input(type=\"uri_folder\"),\n", - " \"test_train_ratio\": Input(type=\"number\"),\n", - " },\n", - " outputs=dict(\n", - " train_data=Output(type=\"uri_folder\", mode=\"rw_mount\"),\n", - " test_data=Output(type=\"uri_folder\", mode=\"rw_mount\"),\n", - " ),\n", - " # The source folder of the component\n", - " code=data_prep_src_dir,\n", - " command=\"\"\"python data_prep.py \\\n", - " --data ${{inputs.data}} --test_train_ratio ${{inputs.test_train_ratio}} \\\n", - " --train_data ${{outputs.train_data}} --test_data ${{outputs.test_data}} \\\n", - " \"\"\",\n", - " environment=f\"{pipeline_job_env.name}:{pipeline_job_env.version}\",\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Optionally, register the component in the workspace for future re-use. **command()** is a component builder, in order to fetch the component itself, we need to call the **.component** property from it." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Now we register the component to the workspace\n", - "data_prep_component = ml_client.create_or_update(data_prep_component.component)\n", - "\n", - "# Create (register) the component in your workspace\n", - "print(\n", - " f\"Component {data_prep_component.name} with Version {data_prep_component.version} is registered\"\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create component 2: training (using yaml definition)\n", - "\n", - "The second component that you'll create will consume the training and test data, train a tree based model and return the output model. You'll use Azure ML logging capabilities to record and visualize the learning progress.\n", - "\n", - "You used the `command` class to create your first component. This time you'll use the yaml definition to define the second component. Each method has its own advantages. A yaml definition can actually be checked-in along the code, and would provide a readable history tracking. Also, the same yaml file can be used in the CLI for component deficnition. The programmatic method using `command` can be easier with built-in class documentation and code completion.\n", - "\n", - "\n", - "Create the directory for this component:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "train_src_dir" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "train_src_dir = \"./components/train\"\n", - "os.makedirs(train_src_dir, exist_ok=True)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create the training script in the directory:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "train.py" - }, - "outputs": [], - "source": [ - "%%writefile {train_src_dir}/train.py\n", - "import argparse\n", - "from sklearn.ensemble import GradientBoostingClassifier\n", - "from sklearn.metrics import classification_report\n", - "import os\n", - "import pandas as pd\n", - "import mlflow\n", - "\n", - "\n", - "def select_first_file(path):\n", - " \"\"\"Selects first file in folder, use under assumption there is only one file in folder\n", - " Args:\n", - " path (str): path to directory or file to choose\n", - " Returns:\n", - " str: full path of selected file\n", - " \"\"\"\n", - " files = os.listdir(path)\n", - " return os.path.join(path, files[0])\n", - "\n", - "\n", - "# Start Logging\n", - "mlflow.start_run()\n", - "\n", - "# enable autologging\n", - "mlflow.sklearn.autolog()\n", - "\n", - "os.makedirs(\"./outputs\", exist_ok=True)\n", - "\n", - "\n", - "def main():\n", - " \"\"\"Main function of the script.\"\"\"\n", - "\n", - " # input and output arguments\n", - " parser = argparse.ArgumentParser()\n", - " parser.add_argument(\"--train_data\", type=str, help=\"path to train data\")\n", - " parser.add_argument(\"--test_data\", type=str, help=\"path to test data\")\n", - " parser.add_argument(\"--n_estimators\", required=False, default=100, type=int)\n", - " parser.add_argument(\"--learning_rate\", required=False, default=0.1, type=float)\n", - " parser.add_argument(\"--registered_model_name\", type=str, help=\"model name\")\n", - " parser.add_argument(\"--model\", type=str, help=\"path to model file\")\n", - " args = parser.parse_args()\n", - "\n", - " # paths are mounted as folder, therefore, we are selecting the file from folder\n", - " train_df = pd.read_csv(select_first_file(args.train_data))\n", - "\n", - " # Extracting the label column\n", - " y_train = train_df.pop(\"default payment next month\")\n", - "\n", - " # convert the dataframe values to array\n", - " X_train = train_df.values\n", - "\n", - " # paths are mounted as folder, therefore, we are selecting the file from folder\n", - " test_df = pd.read_csv(select_first_file(args.test_data))\n", - "\n", - " # Extracting the label column\n", - " y_test = test_df.pop(\"default payment next month\")\n", - "\n", - " # convert the dataframe values to array\n", - " X_test = test_df.values\n", - "\n", - " print(f\"Training with data of shape {X_train.shape}\")\n", - "\n", - " clf = GradientBoostingClassifier(\n", - " n_estimators=args.n_estimators, learning_rate=args.learning_rate\n", - " )\n", - " clf.fit(X_train, y_train)\n", - "\n", - " y_pred = clf.predict(X_test)\n", - "\n", - " print(classification_report(y_test, y_pred))\n", - "\n", - " # Registering the model to the workspace\n", - " print(\"Registering the model via MLFlow\")\n", - " mlflow.sklearn.log_model(\n", - " sk_model=clf,\n", - " registered_model_name=args.registered_model_name,\n", - " artifact_path=args.registered_model_name,\n", - " extra_pip_requirements=[\n", - " \"azureml-ai-monitoring\",\n", - " \"azureml-inference-server-http\"\n", - " ],\n", - " )\n", - "\n", - " # Saving the model to a file\n", - " mlflow.sklearn.save_model(\n", - " sk_model=clf,\n", - " path=os.path.join(args.model, \"trained_model\"),\n", - " )\n", - "\n", - " # Stop Logging\n", - " mlflow.end_run()\n", - "\n", - "\n", - "if __name__ == \"__main__\":\n", - " main()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As you can see in this training script, once the model is trained, the model file is saved and registered to the workspace. Now you can use the registered model in inferencing endpoints.\n", - "\n", - "\n", - "For the environment of this step, you'll use one of the built-in (curated) Azure ML environments. The tag `azureml`, tells the system to use look for the name in curated environments.\n", - "\n", - "First, create the *yaml* file describing the component:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "train.yml" - }, - "outputs": [], - "source": [ - "%%writefile {train_src_dir}/train.yml\n", - "# \n", - "name: train_credit_defaults_model\n", - "display_name: Train Credit Defaults Model\n", - "# version: 1 # Not specifying a version will automatically update the version\n", - "type: command\n", - "inputs:\n", - " train_data: \n", - " type: uri_folder\n", - " test_data: \n", - " type: uri_folder\n", - " learning_rate:\n", - " type: number \n", - " registered_model_name:\n", - " type: string\n", - "outputs:\n", - " model:\n", - " type: uri_folder\n", - "code: .\n", - "environment:\n", - " # for this step, we'll use an AzureML curate environment\n", - " azureml://registries/azureml/environments/sklearn-1.5/labels/latest\n", - "command: >-\n", - " python train.py \n", - " --train_data ${{inputs.train_data}} \n", - " --test_data ${{inputs.test_data}} \n", - " --learning_rate ${{inputs.learning_rate}}\n", - " --registered_model_name ${{inputs.registered_model_name}} \n", - " --model ${{outputs.model}}\n", - "# \n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Once the `yaml` file and the script are ready, you can create your component using `load_component()`. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "train_component" - }, - "outputs": [], - "source": [ - "# importing the Component Package\n", - "from azure.ai.ml import load_component\n", - "\n", - "# Loading the component from the yml file\n", - "train_component = load_component(source=os.path.join(train_src_dir, \"train.yml\"))" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now create and register the component:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "update-train_component" - }, - "outputs": [], - "source": [ - "# Now we register the component to the workspace\n", - "train_component = ml_client.create_or_update(train_component)\n", - "\n", - "# Create (register) the component in your workspace\n", - "print(\n", - " f\"Component {train_component.name} with Version {train_component.version} is registered\"\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create the pipeline from components\n", - "\n", - "Now that both your components are defined and registered, you can start implementing the pipeline." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here, you'll use *input data*, *split ratio* and *registered model name* as input variables. Then call the components and connect them via their inputs /outputs identifiers. The outputs of each step can be accessed via the `.outputs` property." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - } - }, - "source": [ - "The python functions returned by `load_component()` work as any regular python function that we'll use within a pipeline to call each step.\n", - "\n", - "To code the pipeline, we use a specific `@dsl.pipeline` decorator that identifies the Azure ML pipelines. In the decorator, we can specify the pipeline description and default resources like compute (serverless is used here) and storage. Like a python function, pipelines can have inputs, you can then create multiple instances of a single pipeline with different inputs.\n", - "\n", - "Here, we used *input data*, *split ratio* and *registered model name* as input variables. We then call the components and connect them via their inputs /outputs identifiers. The outputs of each step can be accessed via the `.outputs` property." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "pipeline" - }, - "outputs": [], - "source": [ - "# the dsl decorator tells the sdk that we are defining an Azure ML pipeline\n", - "from azure.ai.ml import dsl, Input, Output\n", - "\n", - "\n", - "@dsl.pipeline(\n", - " compute=\"serverless\",\n", - " description=\"E2E data_perp-train pipeline\",\n", - ")\n", - "def credit_defaults_pipeline(\n", - " pipeline_job_data_input,\n", - " pipeline_job_test_train_ratio,\n", - " pipeline_job_learning_rate,\n", - " pipeline_job_registered_model_name,\n", - "):\n", - " # using data_prep_function like a python call with its own inputs\n", - " data_prep_job = data_prep_component(\n", - " data=pipeline_job_data_input,\n", - " test_train_ratio=pipeline_job_test_train_ratio,\n", - " )\n", - "\n", - " # using train_func like a python call with its own inputs\n", - " train_job = train_component(\n", - " train_data=data_prep_job.outputs.train_data, # note: using outputs from previous step\n", - " test_data=data_prep_job.outputs.test_data, # note: using outputs from previous step\n", - " learning_rate=pipeline_job_learning_rate, # note: using a pipeline input as parameter\n", - " registered_model_name=pipeline_job_registered_model_name,\n", - " )\n", - "\n", - " # a pipeline returns a dictionary of outputs\n", - " # keys will code for the pipeline output identifier\n", - " return {\n", - " \"pipeline_job_train_data\": data_prep_job.outputs.train_data,\n", - " \"pipeline_job_test_data\": data_prep_job.outputs.test_data,\n", - " }" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now use your pipeline definition to instantiate a pipeline with your dataset, split rate of choice and the name you picked for your model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "registered_model_name" - }, - "outputs": [], - "source": [ - "registered_model_name = \"credit_defaults_model\"\n", - "\n", - "# Let's instantiate the pipeline with the parameters of our choice\n", - "pipeline = credit_defaults_pipeline(\n", - " pipeline_job_data_input=Input(type=\"uri_file\", path=credit_data.path),\n", - " pipeline_job_test_train_ratio=0.25,\n", - " pipeline_job_learning_rate=0.05,\n", - " pipeline_job_registered_model_name=registered_model_name,\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Submit the job \n", - "\n", - "It's now time to submit the job to run in Azure ML. This time you'll use `create_or_update` on `ml_client.jobs`.\n", - "\n", - "Here you'll also pass an experiment name. An experiment is a container for all the iterations one does on a certain project. All the jobs submitted under the same experiment name would be listed next to each other in Azure ML studio.\n", - "\n", - "Once completed, the pipeline will register a model in your workspace as a result of training." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "returned_job" - }, - "outputs": [], - "source": [ - "import webbrowser\n", - "\n", - "# submit the pipeline job\n", - "pipeline_job = ml_client.jobs.create_or_update(\n", - " pipeline,\n", - " # Project's name\n", - " experiment_name=\"e2e_registered_components\",\n", - ")\n", - "# open the pipeline in web browser\n", - "webbrowser.open(pipeline_job.studio_url)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can track the progress of your pipeline, by using the link generated in the cell above or in this notebook using the following code:\n", - "```python\n", - " ml_client.jobs.stream(pipeline_job.name)\n", - "```\n", - "\n", - "When you select on each component, you'll see more information about the results of that component. \n", - "There are two important parts to look for at this stage:\n", - "* `Outputs+logs` > `user_logs` > `std_log.txt`\n", - "This section shows the script run sdtout." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* `Outputs+logs` > `Metric`\n", - "This section shows different logged metrics. In this example. mlflow `autologging`, has automatically logged the training metrics." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Deploy the model as an online endpoint\n", - "\n", - "Now deploy your machine learning model as a web service in the Azure cloud, an [`online endpoint`](https://docs.microsoft.com/azure/machine-learning/concept-endpoints).\n", - "\n", - "To deploy a machine learning service, you usually need:\n", - "\n", - "* The model assets (filed, metadata) that you want to deploy. You've already registered these assets in your training component.\n", - "* Some code to run as a service. The code executes the model on a given input request. This entry script receives data submitted to a deployed web service and passes it to the model, then returns the model's response to the client. The script is specific to your model. The entry script must understand the data that the model expects and returns. When using a MLFlow model, as in this tutorial, this script is automatically created for you. Samples of scoring scripts can be found [here](https://github.com/Azure/azureml-examples/tree/sdk-preview/sdk/endpoints/online).\n", - "\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create a new online endpoint\n", - "\n", - "Now that you have a registered model and an inference script, it's time to create your online endpoint. The endpoint name needs to be unique in the entire Azure region. For this tutorial, you'll create a unique name using [`UUID`](https://en.wikipedia.org/wiki/Universally_unique_identifier#:~:text=A%20universally%20unique%20identifier%20(UUID,%2C%20for%20practical%20purposes%2C%20unique.)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "online_endpoint_name" - }, - "outputs": [], - "source": [ - "import uuid\n", - "\n", - "# Creating a unique name for the endpoint\n", - "online_endpoint_name = \"credit-endpoint-\" + str(uuid.uuid4())[:8]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "endpoint" - }, - "outputs": [], - "source": [ - "from azure.ai.ml.entities import (\n", - " ManagedOnlineEndpoint,\n", - " ManagedOnlineDeployment,\n", - " Model,\n", - " Environment,\n", - ")\n", - "\n", - "# create an online endpoint\n", - "endpoint = ManagedOnlineEndpoint(\n", - " name=online_endpoint_name,\n", - " description=\"this is an online endpoint\",\n", - " auth_mode=\"key\",\n", - " tags={\n", - " \"training_dataset\": \"credit_defaults\",\n", - " \"model_type\": \"sklearn.GradientBoostingClassifier\",\n", - " },\n", - ")\n", - "\n", - "endpoint_result = ml_client.begin_create_or_update(endpoint).result()\n", - "\n", - "print(\n", - " f\"Endpint {endpoint_result.name} provisioning state: {endpoint_result.provisioning_state}\"\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Once you've created an endpoint, you can retrieve it as below:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "update-endpoint" - }, - "outputs": [], - "source": [ - "endpoint = ml_client.online_endpoints.get(name=online_endpoint_name)\n", - "\n", - "print(\n", - " f'Endpint \"{endpoint.name}\" with provisioning state \"{endpoint.provisioning_state}\" is retrieved'\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Deploy the model to the endpoint\n", - "\n", - "Once the endpoint is created, deploy the model with the entry script. Each endpoint can have multiple deployments and direct traffic to these deployments can be specified using rules. Here you'll create a single deployment that handles 100% of the incoming traffic. We have chosen a color name for the deployment, for example, *blue*, *green*, *red* deployments, which is arbitrary.\n", - "\n", - "You can check the *Models* page on the Azure ML studio, to identify the latest version of your registered model. Alternatively, the code below will retrieve the latest version number for you to use." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "latest_model_version" - }, - "outputs": [], - "source": [ - "# Let's pick the latest version of the model\n", - "latest_model_version = max(\n", - " [int(m.version) for m in ml_client.models.list(name=registered_model_name)]\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Deploy the latest version of the model. \n", - "\n", - "> [!NOTE]\n", - "> Expect this deployment to take approximately 6 to 8 minutes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "model" - }, - "outputs": [], - "source": [ - "# picking the model to deploy. Here we use the latest version of our registered model\n", - "model = ml_client.models.get(name=registered_model_name, version=latest_model_version)\n", - "\n", - "\n", - "# create an online deployment.\n", - "blue_deployment = ManagedOnlineDeployment(\n", - " name=\"blue\",\n", - " endpoint_name=online_endpoint_name,\n", - " model=model,\n", - " instance_type=\"Standard_F2s_v2\",\n", - " instance_count=1,\n", - ")\n", - "\n", - "blue_deployment_results = ml_client.online_deployments.begin_create_or_update(\n", - " blue_deployment\n", - ").result()\n", - "\n", - "print(\n", - " f\"Deployment {blue_deployment_results.name} provisioning state: {blue_deployment_results.provisioning_state}\"\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Test with a sample query\n", - "\n", - "Now that the model is deployed to the endpoint, you can run inference with it.\n", - "\n", - "Create a sample request file following the design expected in the run method in the score script." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "sample-request.json" - }, - "outputs": [], - "source": [ - "deploy_dir = \"./deploy\"\n", - "os.makedirs(deploy_dir, exist_ok=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "name": "write-sample-request" - }, - "outputs": [], - "source": [ - "%%writefile {deploy_dir}/sample-request.json\n", - "{\n", - " \"input_data\": {\n", - " \"columns\": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22],\n", - " \"index\": [0, 1],\n", - " \"data\": [\n", - " [20000,2,2,1,24,2,2,-1,-1,-2,-2,3913,3102,689,0,0,0,0,689,0,0,0,0],\n", - " [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, 9, 8]\n", - " ]\n", - " }\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "Python" - ], - "id": "" - }, - "name": "ml_client.online_endpoints.invoke" - }, - "outputs": [], - "source": [ - "# test the blue deployment with some sample data\n", - "ml_client.online_endpoints.invoke(\n", - " endpoint_name=online_endpoint_name,\n", - " request_file=\"./deploy/sample-request.json\",\n", - " deployment_name=\"blue\",\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Clean up resources\n", - "\n", - "If you're not going to use the endpoint, delete it to stop using the resource. Make sure no other deployments are using an endpoint before you delete it.\n", - "\n", - "\n", - "> [!NOTE]\n", - "> Expect this step to take approximately 6 to 8 minutes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "attributes": { - "classes": [ - "python " - ], - "id": "" - }, - "name": "ml_client.online_endpoints.begin_delete" - }, - "outputs": [], - "source": [ - "ml_client.online_endpoints.begin_delete(name=online_endpoint_name)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Next Steps\n", - "\n", - "Learn more about [Azure ML logging](https://github.com/Azure/azureml-examples/blob/sdk-preview/notebooks/mlflow/mlflow-v1-comparison.ipynb)." - ] - } - ], - "metadata": { - "celltoolbar": "Edit Metadata", - "description": { - "description": "Create production ML pipelines with Python SDK v2 in a Jupyter notebook" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.17" - }, - "nteract": { - "version": "nteract-front-end@1.0.0" - } + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tutorial: Create production ML pipelines with Python SDK v2 (preview) in a Jupyter notebook\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Learning Objectives** - By the end of this two part tutorial, you should be able to use Azure Machine Learning (Azure ML) to productionize your ML project.\n", + "\n", + "This means you will be able to leverage the AzureML Python SDK to:\n", + "\n", + "- connect to your Azure ML workspace\n", + "- create Azure ML data assets\n", + "- create reusable Azure ML components\n", + "- create, validate and run Azure ML pipelines\n", + "- deploy the newly-trained model as an endpoint\n", + "- call the Azure ML endpoint for inferencing" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Motivations** - This tutorial is intended to introduce Azure ML to data scientists who want to scale up or publish their ML projects. By completing a familiar end-to-end project, which starts by loading the data and ends by creating and calling an online inference endpoint, the user should become familiar with the core concepts of Azure ML and their most common usage. Each step of this tutorial can be modified or performed in other ways that might have security or scalability advantages. We will cover some of those in the Part II of this tutorial, however, we suggest the reader use the provide links in each section to learn more on each topic." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Requirements** - In order to benefit from this tutorial, you need to have:\n", + "- basic understanding of Machine Learning projects workflow\n", + "- an Azure subscription. If you don't have an Azure subscription, [create a free account](https://aka.ms/AMLFree) before you begin.\n", + "- a working Azure ML workspace. A workspace can be created via Azure Portal, Azure CLI, or Python SDK. [Read more](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-manage-workspace?tabs=python).\n", + "- a Python environmnet\n", + "- [installed Azure Machine Learning Python SDK v2](https://github.com/Azure/azureml-examples/blob/sdk-preview/sdk/setup.sh)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Introduction\n", + "\n", + "In this tutorial, you'll create an Azure ML pipeline to train a model for credit default prediction. The pipeline handles the data preparation, training and registering the trained model. You'll then run the pipeline, deploy the model and use it.\n", + "\n", + "The image below shows the pipeline as you'll see it in the AzureML portal once submitted. It's a rather simple pipeline we'll use to walk you through the AzureML SDK v2.\n", + "\n", + "The two steps are first data preparation and second training. \n", + "\n", + "![Screenshot that shows the AML Pipeline](media/pipeline-overview.jpg \"Overview of the pipeline\")\n", + "\n", + "An AzureML pipeline that runs from local components, requires several dependent files. Fo better understanding of the project structure, we produce all these dependencies in the notebook cells. By the end of this tutorial, the project structure should look like:\n", + "```\n", + "e2e-ds-experience \n", + " components\n", + " data_prep\n", + " data_prep.py \n", + " train\n", + " train.py\n", + " train.yml \n", + " dependencies\n", + " conda.yaml\n", + " deploy\n", + " sample-request.json\n", + " media\n", + " metrics.jpg\n", + " pipeline-overview.jpg\n", + " user-logs.jpg\n", + " e2e-ml-workflow.ipynb\n", + "```\n", + "After running this notebook, you should be able to create the project direcetly in the IDE of your choice, instead.\n", + "\n", + "## Set up the pipeline resources\n", + "\n", + "The Azure ML framework can be used from CLI, Python SDK, or studio interface. In this example, you'll use the AzureML Python SDK v2 to create a pipeline. \n", + "\n", + "Before creating the pipeline, you'll set up the resources the pipeline will use:\n", + "\n", + "* The dataset for training\n", + "* The software environment to run the pipeline\n", + "\n", + "## Connect to the workspace\n", + "\n", + "Before we dive in the code, you'll need to connect to your Azure ML workspace. The workspace is the top-level resource for Azure Machine Learning, providing a centralized place to work with all the artifacts you create when you use Azure Machine Learning.\n", + "\n", + "We are using `DefaultAzureCredential` to get access to workspace. \n", + "`DefaultAzureCredential` should be capable of handling most Azure SDK authentication scenarios. \n", + "\n", + "Reference for more available credentials if it does not work for you: [configure credential example](../../configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" }, - "nbformat": 4, - "nbformat_minor": 1 + "name": "import-mlclient" + }, + "outputs": [], + "source": [ + "# Handle to the workspace\n", + "from azure.ai.ml import MLClient\n", + "\n", + "# Authentication package\n", + "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", + "\n", + "try:\n", + " credential = DefaultAzureCredential()\n", + " # Check if given credential can get token successfully.\n", + " credential.get_token(\"https://management.azure.com/.default\")\n", + "except Exception as ex:\n", + " # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work\n", + " credential = InteractiveBrowserCredential()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the next cell, enter your Subscription ID, Resource Group name and Workspace name. To find your Subscription ID:\n", + "1. In the upper right Azure Machine Learning Studio toolbar, select your workspace name.\n", + "1. At the bottom, select **View all properties in Azure Portal**\n", + "1. Copy the value from Azure Portal into the code." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "ml_client" + }, + "outputs": [], + "source": [ + "# Get a handle to the workspace\n", + "ml_client = MLClient(\n", + " credential=credential,\n", + " subscription_id=\"\",\n", + " resource_group_name=\"\",\n", + " workspace_name=\"\",\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The result is a handler to the workspace that you'll use to manage other resources and jobs.\n", + "\n", + "> [!IMPORTANT]\n", + "> Creating MLClient will not connect to the workspace. The client initialization is lazy, it will wait for the first time it needs to make a call (in the notebook below, that will happen during dataset registration).\n", + "\n", + "## Register data from an external url\n", + "\n", + "The data you use for training is usually in one of the locations below:\n", + "\n", + "* Local machine\n", + "* Web\n", + "* Big Data Storage services (for example, Azure Blob, Azure Data Lake Storage, SQL)\n", + " \n", + "Azure ML uses a [`Data`](https://docs.microsoft.com/azure/machine-learning/how-to-create-register-data-assets?tabs=Python-SDK) object to register a reusable definition of data, and consume data within a pipeline. In the section below, you'll consume some data from web url as one example. `Data` assets ets from other sources can be created as well." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "credit_data" + }, + "outputs": [], + "source": [ + "from azure.ai.ml.entities import Data\n", + "from azure.ai.ml.constants import AssetTypes\n", + "\n", + "web_path = \"https://azuremlexamples.blob.core.windows.net/datasets/credit_card/default%20of%20credit%20card%20clients.csv\"\n", + "\n", + "credit_data = Data(\n", + " name=\"creditcard_blob_csv_defaults\",\n", + " path=web_path,\n", + " type=AssetTypes.URI_FILE,\n", + " description=\"Dataset for credit card defaults\",\n", + " tags={\"source_type\": \"web\", \"source\": \"AzureML examples blob\"},\n", + " version=\"1.0.0\",\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This code just created a `Data` asset, it is ready to be consumed as an input by the pipeline that you'll define in the next sections. In addition, you can register the data to your workspace so it becomes reusable across pipelines.\n", + "\n", + "Registering the data asset will enable you to:\n", + "\n", + "* reuse and share the data asset in future pipelines\n", + "* use versions to track the modification to the data asset\n", + "* use the data asset from Azure ML designer, which is Azure ML's GUI for pipeline authoring\n", + "\n", + "Since this is the first time that you're making a call to the workspace, you may be asked to authenticate. Once the authentication is complete, you'll then see the dataset registration completion message." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "update-credit_data" + }, + "outputs": [], + "source": [ + "credit_data = ml_client.data.create_or_update(credit_data)\n", + "print(\n", + " f\"Dataset with name {credit_data.name} was registered to workspace, the dataset version is {credit_data.version}\"\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create a job environment for pipeline steps\n", + "\n", + "So far, you've created a development environment on the compute instance, your development machine. You'll also need an [environment](https://docs.microsoft.com/azure/machine-learning/concept-environments) to use for each step of the pipeline. Each step can have its own environment, or you can use some common environments for multiple steps.\n", + "\n", + "In this example, you'll create a conda environment for your jobs, using a conda yaml file.\n", + "First, create a directory to store the file in." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "dependencies_dir" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "dependencies_dir = \"./dependencies\"\n", + "os.makedirs(dependencies_dir, exist_ok=True)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, create the file in the dependencies directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "conda.yaml" + }, + "outputs": [], + "source": [ + "%%writefile {dependencies_dir}/conda.yaml\n", + "name: model-env\n", + "channels:\n", + " - conda-forge\n", + "dependencies:\n", + " - python=3.8\n", + " - numpy=1.21.2\n", + " - pip=21.2.4\n", + " - scikit-learn=0.24.2\n", + " - scipy=1.7.1\n", + " - pandas>=1.1,<1.2\n", + " - pip:\n", + " - inference-schema[numpy-support]==1.3.0\n", + " - xlrd==2.0.1\n", + " - azureml-mlflow==1.42.0\n", + " - mlflow==2.16.2" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The specification contains some usual packages, that you'll use in your pipeline (numpy, pip).\n", + "\n", + "\n", + "Use the *yaml* file to create and register this custom environment in your workspace:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "custom_env_name" + }, + "outputs": [], + "source": [ + "from azure.ai.ml.entities import Environment\n", + "\n", + "custom_env_name = \"aml-scikit-learn\"\n", + "\n", + "pipeline_job_env = Environment(\n", + " name=custom_env_name,\n", + " description=\"Custom environment for Credit Card Defaults pipeline\",\n", + " tags={\"scikit-learn\": \"0.24.2\"},\n", + " conda_file=os.path.join(dependencies_dir, \"conda.yaml\"),\n", + " image=\"mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest\",\n", + " version=\"0.1.8\",\n", + ")\n", + "pipeline_job_env = ml_client.environments.create_or_update(pipeline_job_env)\n", + "\n", + "print(\n", + " f\"Environment with name {pipeline_job_env.name} is registered to workspace, the environment version is {pipeline_job_env.version}\"\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Build the training pipeline\n", + "\n", + "Now that you have all assets required to run your pipeline, it's time to build the pipeline itself, using the Azure ML Python SDK v2.\n", + "\n", + "Azure ML pipelines are reusable ML workflows that usually consist of several components. The typical life of a component is:\n", + "\n", + "* Write the yaml specification of the component.\n", + "* Optionally, register the component with a name and version in your workspace, to make it reusable and shareable.\n", + "* Load that component from the pipeline code.\n", + "* Implement the pipeline using this component inputs, outputs and parameters.\n", + "* Submit the pipeline.\n", + "\n", + "## Create component 1: data prep (using programmatic definition)\n", + "\n", + "Let's start by creating the first component. This component handles the preprocessing of the data. The preprocessing task is performed in the *data_prep.py* python file.\n", + "\n", + "First create a source folder for the data_prep component:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "data_prep_src_dir" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "data_prep_src_dir = \"./components/data_prep\"\n", + "os.makedirs(data_prep_src_dir, exist_ok=True)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This script performs the simple task of splitting the data into train and test datasets. \n", + "\n", + "[MLFlow](https://mlflow.org/docs/latest/tracking.html) will be used to log the parameters and metrics during our pipeline run." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "def-main" + }, + "outputs": [], + "source": [ + "%%writefile {data_prep_src_dir}/data_prep.py\n", + "import os\n", + "import argparse\n", + "import pandas as pd\n", + "from sklearn.model_selection import train_test_split\n", + "import logging\n", + "import mlflow\n", + "\n", + "\n", + "def main():\n", + " \"\"\"Main function of the script.\"\"\"\n", + "\n", + " # input and output arguments\n", + " parser = argparse.ArgumentParser()\n", + " parser.add_argument(\"--data\", type=str, help=\"path to input data\")\n", + " parser.add_argument(\"--test_train_ratio\", type=float, required=False, default=0.25)\n", + " parser.add_argument(\"--train_data\", type=str, help=\"path to train data\")\n", + " parser.add_argument(\"--test_data\", type=str, help=\"path to test data\")\n", + " args = parser.parse_args()\n", + "\n", + " # Start Logging\n", + " mlflow.start_run()\n", + "\n", + " print(\" \".join(f\"{k}={v}\" for k, v in vars(args).items()))\n", + "\n", + " print(\"input data:\", args.data)\n", + "\n", + " credit_df = pd.read_csv(args.data, header=1, index_col=0)\n", + "\n", + " mlflow.log_metric(\"num_samples\", credit_df.shape[0])\n", + " mlflow.log_metric(\"num_features\", credit_df.shape[1] - 1)\n", + "\n", + " credit_train_df, credit_test_df = train_test_split(\n", + " credit_df,\n", + " test_size=args.test_train_ratio,\n", + " )\n", + "\n", + " # output paths are mounted as folder, therefore, we are adding a filename to the path\n", + " credit_train_df.to_csv(os.path.join(args.train_data, \"data.csv\"), index=False)\n", + "\n", + " credit_test_df.to_csv(os.path.join(args.test_data, \"data.csv\"), index=False)\n", + "\n", + " # Stop Logging\n", + " mlflow.end_run()\n", + "\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that you have a script that can perform the desired task, we can create an Azure ML Component from it. Azure ML support various types of components for performing ML tasks, such as running scripts, data transfer, etc.\n", + "\n", + "A component can be created by calling the component instantiators, or directly writing the defining yaml file. \n", + "\n", + "You'll use the general purpose **command** that can run command line actions. This command line action can be directly calling system commands or running a script. The inputs/outputs are accessible in the command via the `${{ ... }}` notation. For the second component of this tutorial you will use `yaml` definitions.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "data_prep_component" + }, + "outputs": [], + "source": [ + "from azure.ai.ml import command\n", + "from azure.ai.ml import Input, Output\n", + "\n", + "data_prep_component = command(\n", + " name=\"data_prep_credit_defaults\",\n", + " display_name=\"Data preparation for training\",\n", + " description=\"reads a .xl input, split the input to train and test\",\n", + " inputs={\n", + " \"data\": Input(type=\"uri_folder\"),\n", + " \"test_train_ratio\": Input(type=\"number\"),\n", + " },\n", + " outputs=dict(\n", + " train_data=Output(type=\"uri_folder\", mode=\"rw_mount\"),\n", + " test_data=Output(type=\"uri_folder\", mode=\"rw_mount\"),\n", + " ),\n", + " # The source folder of the component\n", + " code=data_prep_src_dir,\n", + " command=\"\"\"python data_prep.py \\\n", + " --data ${{inputs.data}} --test_train_ratio ${{inputs.test_train_ratio}} \\\n", + " --train_data ${{outputs.train_data}} --test_data ${{outputs.test_data}} \\\n", + " \"\"\",\n", + " environment=f\"{pipeline_job_env.name}:{pipeline_job_env.version}\",\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "Optionally, register the component in the workspace for future re-use. **command()** is a component builder, in order to fetch the component itself, we need to call the **.component** property from it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Now we register the component to the workspace\n", + "data_prep_component = ml_client.create_or_update(data_prep_component.component)\n", + "\n", + "# Create (register) the component in your workspace\n", + "print(\n", + " f\"Component {data_prep_component.name} with Version {data_prep_component.version} is registered\"\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create component 2: training (using yaml definition)\n", + "\n", + "The second component that you'll create will consume the training and test data, train a tree based model and return the output model. You'll use Azure ML logging capabilities to record and visualize the learning progress.\n", + "\n", + "You used the `command` class to create your first component. This time you'll use the yaml definition to define the second component. Each method has its own advantages. A yaml definition can actually be checked-in along the code, and would provide a readable history tracking. Also, the same yaml file can be used in the CLI for component deficnition. The programmatic method using `command` can be easier with built-in class documentation and code completion.\n", + "\n", + "\n", + "Create the directory for this component:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "train_src_dir" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "train_src_dir = \"./components/train\"\n", + "os.makedirs(train_src_dir, exist_ok=True)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create the training script in the directory:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "train.py" + }, + "outputs": [], + "source": [ + "%%writefile {train_src_dir}/train.py\n", + "import argparse\n", + "from sklearn.ensemble import GradientBoostingClassifier\n", + "from sklearn.metrics import classification_report\n", + "import os\n", + "import pandas as pd\n", + "import mlflow\n", + "\n", + "\n", + "def select_first_file(path):\n", + " \"\"\"Selects first file in folder, use under assumption there is only one file in folder\n", + " Args:\n", + " path (str): path to directory or file to choose\n", + " Returns:\n", + " str: full path of selected file\n", + " \"\"\"\n", + " files = os.listdir(path)\n", + " return os.path.join(path, files[0])\n", + "\n", + "\n", + "# Start Logging\n", + "mlflow.start_run()\n", + "\n", + "# enable autologging\n", + "mlflow.sklearn.autolog()\n", + "\n", + "os.makedirs(\"./outputs\", exist_ok=True)\n", + "\n", + "\n", + "def main():\n", + " \"\"\"Main function of the script.\"\"\"\n", + "\n", + " # input and output arguments\n", + " parser = argparse.ArgumentParser()\n", + " parser.add_argument(\"--train_data\", type=str, help=\"path to train data\")\n", + " parser.add_argument(\"--test_data\", type=str, help=\"path to test data\")\n", + " parser.add_argument(\"--n_estimators\", required=False, default=100, type=int)\n", + " parser.add_argument(\"--learning_rate\", required=False, default=0.1, type=float)\n", + " parser.add_argument(\"--registered_model_name\", type=str, help=\"model name\")\n", + " parser.add_argument(\"--model\", type=str, help=\"path to model file\")\n", + " args = parser.parse_args()\n", + "\n", + " # paths are mounted as folder, therefore, we are selecting the file from folder\n", + " train_df = pd.read_csv(select_first_file(args.train_data))\n", + "\n", + " # Extracting the label column\n", + " y_train = train_df.pop(\"default payment next month\")\n", + "\n", + " # convert the dataframe values to array\n", + " X_train = train_df.values\n", + "\n", + " # paths are mounted as folder, therefore, we are selecting the file from folder\n", + " test_df = pd.read_csv(select_first_file(args.test_data))\n", + "\n", + " # Extracting the label column\n", + " y_test = test_df.pop(\"default payment next month\")\n", + "\n", + " # convert the dataframe values to array\n", + " X_test = test_df.values\n", + "\n", + " print(f\"Training with data of shape {X_train.shape}\")\n", + "\n", + " clf = GradientBoostingClassifier(\n", + " n_estimators=args.n_estimators, learning_rate=args.learning_rate\n", + " )\n", + " clf.fit(X_train, y_train)\n", + "\n", + " y_pred = clf.predict(X_test)\n", + "\n", + " print(classification_report(y_test, y_pred))\n", + "\n", + " # Registering the model to the workspace\n", + " print(\"Registering the model via MLFlow\")\n", + " mlflow.sklearn.log_model(\n", + " sk_model=clf,\n", + " registered_model_name=args.registered_model_name,\n", + " artifact_path=args.registered_model_name,\n", + " extra_pip_requirements=[\n", + " \"azureml-ai-monitoring\",\n", + " \"azureml-inference-server-http\"\n", + " ],\n", + " )\n", + "\n", + " # Saving the model to a file\n", + " mlflow.sklearn.save_model(\n", + " sk_model=clf,\n", + " path=os.path.join(args.model, \"trained_model\"),\n", + " )\n", + "\n", + " # Stop Logging\n", + " mlflow.end_run()\n", + "\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see in this training script, once the model is trained, the model file is saved and registered to the workspace. Now you can use the registered model in inferencing endpoints.\n", + "\n", + "\n", + "For the environment of this step, you'll use one of the built-in (curated) Azure ML environments. The tag `azureml`, tells the system to use look for the name in curated environments.\n", + "\n", + "First, create the *yaml* file describing the component:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "train.yml" + }, + "outputs": [], + "source": [ + "%%writefile {train_src_dir}/train.yml\n", + "# \n", + "name: train_credit_defaults_model\n", + "display_name: Train Credit Defaults Model\n", + "# version: 1 # Not specifying a version will automatically update the version\n", + "type: command\n", + "inputs:\n", + " train_data: \n", + " type: uri_folder\n", + " test_data: \n", + " type: uri_folder\n", + " learning_rate:\n", + " type: number \n", + " registered_model_name:\n", + " type: string\n", + "outputs:\n", + " model:\n", + " type: uri_folder\n", + "code: .\n", + "environment:\n", + " # for this step, we'll use an AzureML curate environment\n", + " azureml://registries/azureml/environments/sklearn-1.5/labels/latest\n", + "command: >-\n", + " python train.py \n", + " --train_data ${{inputs.train_data}} \n", + " --test_data ${{inputs.test_data}} \n", + " --learning_rate ${{inputs.learning_rate}}\n", + " --registered_model_name ${{inputs.registered_model_name}} \n", + " --model ${{outputs.model}}\n", + "# \n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Once the `yaml` file and the script are ready, you can create your component using `load_component()`. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "train_component" + }, + "outputs": [], + "source": [ + "# importing the Component Package\n", + "from azure.ai.ml import load_component\n", + "\n", + "# Loading the component from the yml file\n", + "train_component = load_component(source=os.path.join(train_src_dir, \"train.yml\"))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now create and register the component:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "update-train_component" + }, + "outputs": [], + "source": [ + "# Now we register the component to the workspace\n", + "train_component = ml_client.create_or_update(train_component)\n", + "\n", + "# Create (register) the component in your workspace\n", + "print(\n", + " f\"Component {train_component.name} with Version {train_component.version} is registered\"\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create the pipeline from components\n", + "\n", + "Now that both your components are defined and registered, you can start implementing the pipeline." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here, you'll use *input data*, *split ratio* and *registered model name* as input variables. Then call the components and connect them via their inputs /outputs identifiers. The outputs of each step can be accessed via the `.outputs` property." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + } + }, + "source": [ + "The python functions returned by `load_component()` work as any regular python function that we'll use within a pipeline to call each step.\n", + "\n", + "To code the pipeline, we use a specific `@dsl.pipeline` decorator that identifies the Azure ML pipelines. In the decorator, we can specify the pipeline description and default resources like compute (serverless is used here) and storage. Like a python function, pipelines can have inputs, you can then create multiple instances of a single pipeline with different inputs.\n", + "\n", + "Here, we used *input data*, *split ratio* and *registered model name* as input variables. We then call the components and connect them via their inputs /outputs identifiers. The outputs of each step can be accessed via the `.outputs` property." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "pipeline" + }, + "outputs": [], + "source": [ + "# the dsl decorator tells the sdk that we are defining an Azure ML pipeline\n", + "from azure.ai.ml import dsl, Input, Output\n", + "\n", + "\n", + "@dsl.pipeline(\n", + " compute=\"serverless\",\n", + " description=\"E2E data_perp-train pipeline\",\n", + ")\n", + "def credit_defaults_pipeline(\n", + " pipeline_job_data_input,\n", + " pipeline_job_test_train_ratio,\n", + " pipeline_job_learning_rate,\n", + " pipeline_job_registered_model_name,\n", + "):\n", + " # using data_prep_function like a python call with its own inputs\n", + " data_prep_job = data_prep_component(\n", + " data=pipeline_job_data_input,\n", + " test_train_ratio=pipeline_job_test_train_ratio,\n", + " )\n", + "\n", + " # using train_func like a python call with its own inputs\n", + " train_job = train_component(\n", + " train_data=data_prep_job.outputs.train_data, # note: using outputs from previous step\n", + " test_data=data_prep_job.outputs.test_data, # note: using outputs from previous step\n", + " learning_rate=pipeline_job_learning_rate, # note: using a pipeline input as parameter\n", + " registered_model_name=pipeline_job_registered_model_name,\n", + " )\n", + "\n", + " # a pipeline returns a dictionary of outputs\n", + " # keys will code for the pipeline output identifier\n", + " return {\n", + " \"pipeline_job_train_data\": data_prep_job.outputs.train_data,\n", + " \"pipeline_job_test_data\": data_prep_job.outputs.test_data,\n", + " }" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now use your pipeline definition to instantiate a pipeline with your dataset, split rate of choice and the name you picked for your model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "registered_model_name" + }, + "outputs": [], + "source": [ + "registered_model_name = \"credit_defaults_model\"\n", + "\n", + "# Let's instantiate the pipeline with the parameters of our choice\n", + "pipeline = credit_defaults_pipeline(\n", + " pipeline_job_data_input=Input(type=\"uri_file\", path=credit_data.path),\n", + " pipeline_job_test_train_ratio=0.25,\n", + " pipeline_job_learning_rate=0.05,\n", + " pipeline_job_registered_model_name=registered_model_name,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Submit the job \n", + "\n", + "It's now time to submit the job to run in Azure ML. This time you'll use `create_or_update` on `ml_client.jobs`.\n", + "\n", + "Here you'll also pass an experiment name. An experiment is a container for all the iterations one does on a certain project. All the jobs submitted under the same experiment name would be listed next to each other in Azure ML studio.\n", + "\n", + "Once completed, the pipeline will register a model in your workspace as a result of training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "returned_job" + }, + "outputs": [], + "source": [ + "import webbrowser\n", + "\n", + "# submit the pipeline job\n", + "pipeline_job = ml_client.jobs.create_or_update(\n", + " pipeline,\n", + " # Project's name\n", + " experiment_name=\"e2e_registered_components\",\n", + ")\n", + "# open the pipeline in web browser\n", + "webbrowser.open(pipeline_job.studio_url)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can track the progress of your pipeline, by using the link generated in the cell above or in this notebook using the following code:\n", + "```python\n", + " ml_client.jobs.stream(pipeline_job.name)\n", + "```\n", + "\n", + "When you select on each component, you'll see more information about the results of that component. \n", + "There are two important parts to look for at this stage:\n", + "* `Outputs+logs` > `user_logs` > `std_log.txt`\n", + "This section shows the script run sdtout." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* `Outputs+logs` > `Metric`\n", + "This section shows different logged metrics. In this example. mlflow `autologging`, has automatically logged the training metrics." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deploy the model as an online endpoint\n", + "\n", + "Now deploy your machine learning model as a web service in the Azure cloud, an [`online endpoint`](https://docs.microsoft.com/azure/machine-learning/concept-endpoints).\n", + "\n", + "To deploy a machine learning service, you usually need:\n", + "\n", + "* The model assets (filed, metadata) that you want to deploy. You've already registered these assets in your training component.\n", + "* Some code to run as a service. The code executes the model on a given input request. This entry script receives data submitted to a deployed web service and passes it to the model, then returns the model's response to the client. The script is specific to your model. The entry script must understand the data that the model expects and returns. When using a MLFlow model, as in this tutorial, this script is automatically created for you. Samples of scoring scripts can be found [here](https://github.com/Azure/azureml-examples/tree/sdk-preview/sdk/endpoints/online).\n", + "\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create a new online endpoint\n", + "\n", + "Now that you have a registered model and an inference script, it's time to create your online endpoint. The endpoint name needs to be unique in the entire Azure region. For this tutorial, you'll create a unique name using [`UUID`](https://en.wikipedia.org/wiki/Universally_unique_identifier#:~:text=A%20universally%20unique%20identifier%20(UUID,%2C%20for%20practical%20purposes%2C%20unique.)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "online_endpoint_name" + }, + "outputs": [], + "source": [ + "import uuid\n", + "\n", + "# Creating a unique name for the endpoint\n", + "online_endpoint_name = \"credit-endpoint-\" + str(uuid.uuid4())[:8]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "endpoint" + }, + "outputs": [], + "source": [ + "from azure.ai.ml.entities import (\n", + " ManagedOnlineEndpoint,\n", + " ManagedOnlineDeployment,\n", + " Model,\n", + " Environment,\n", + ")\n", + "\n", + "# create an online endpoint\n", + "endpoint = ManagedOnlineEndpoint(\n", + " name=online_endpoint_name,\n", + " description=\"this is an online endpoint\",\n", + " auth_mode=\"key\",\n", + " tags={\n", + " \"training_dataset\": \"credit_defaults\",\n", + " \"model_type\": \"sklearn.GradientBoostingClassifier\",\n", + " },\n", + ")\n", + "\n", + "endpoint_result = ml_client.begin_create_or_update(endpoint).result()\n", + "\n", + "print(\n", + " f\"Endpint {endpoint_result.name} provisioning state: {endpoint_result.provisioning_state}\"\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Once you've created an endpoint, you can retrieve it as below:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "update-endpoint" + }, + "outputs": [], + "source": [ + "endpoint = ml_client.online_endpoints.get(name=online_endpoint_name)\n", + "\n", + "print(\n", + " f'Endpint \"{endpoint.name}\" with provisioning state \"{endpoint.provisioning_state}\" is retrieved'\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deploy the model to the endpoint\n", + "\n", + "Once the endpoint is created, deploy the model with the entry script. Each endpoint can have multiple deployments and direct traffic to these deployments can be specified using rules. Here you'll create a single deployment that handles 100% of the incoming traffic. We have chosen a color name for the deployment, for example, *blue*, *green*, *red* deployments, which is arbitrary.\n", + "\n", + "You can check the *Models* page on the Azure ML studio, to identify the latest version of your registered model. Alternatively, the code below will retrieve the latest version number for you to use." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "latest_model_version" + }, + "outputs": [], + "source": [ + "# Let's pick the latest version of the model\n", + "latest_model_version = max(\n", + " [int(m.version) for m in ml_client.models.list(name=registered_model_name)]\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Deploy the latest version of the model. \n", + "\n", + "> [!NOTE]\n", + "> Expect this deployment to take approximately 6 to 8 minutes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "model" + }, + "outputs": [], + "source": [ + "# picking the model to deploy. Here we use the latest version of our registered model\n", + "model = ml_client.models.get(name=registered_model_name, version=latest_model_version)\n", + "\n", + "\n", + "# create an online deployment.\n", + "blue_deployment = ManagedOnlineDeployment(\n", + " name=\"blue\",\n", + " endpoint_name=online_endpoint_name,\n", + " model=model,\n", + " instance_type=\"Standard_F2s_v2\",\n", + " instance_count=1,\n", + ")\n", + "\n", + "blue_deployment_results = ml_client.online_deployments.begin_create_or_update(\n", + " blue_deployment\n", + ").result()\n", + "\n", + "print(\n", + " f\"Deployment {blue_deployment_results.name} provisioning state: {blue_deployment_results.provisioning_state}\"\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Test with a sample query\n", + "\n", + "Now that the model is deployed to the endpoint, you can run inference with it.\n", + "\n", + "Create a sample request file following the design expected in the run method in the score script." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "sample-request.json" + }, + "outputs": [], + "source": [ + "deploy_dir = \"./deploy\"\n", + "os.makedirs(deploy_dir, exist_ok=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "name": "write-sample-request" + }, + "outputs": [], + "source": [ + "%%writefile {deploy_dir}/sample-request.json\n", + "{\n", + " \"input_data\": {\n", + " \"columns\": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22],\n", + " \"index\": [0, 1],\n", + " \"data\": [\n", + " [20000,2,2,1,24,2,2,-1,-1,-2,-2,3913,3102,689,0,0,0,0,689,0,0,0,0],\n", + " [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, 9, 8]\n", + " ]\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "Python" + ], + "id": "" + }, + "name": "ml_client.online_endpoints.invoke" + }, + "outputs": [], + "source": [ + "# test the blue deployment with some sample data\n", + "ml_client.online_endpoints.invoke(\n", + " endpoint_name=online_endpoint_name,\n", + " request_file=\"./deploy/sample-request.json\",\n", + " deployment_name=\"blue\",\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Clean up resources\n", + "\n", + "If you're not going to use the endpoint, delete it to stop using the resource. Make sure no other deployments are using an endpoint before you delete it.\n", + "\n", + "\n", + "> [!NOTE]\n", + "> Expect this step to take approximately 6 to 8 minutes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "attributes": { + "classes": [ + "python " + ], + "id": "" + }, + "name": "ml_client.online_endpoints.begin_delete" + }, + "outputs": [], + "source": [ + "ml_client.online_endpoints.begin_delete(name=online_endpoint_name)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next Steps\n", + "\n", + "Learn more about [Azure ML logging](https://github.com/Azure/azureml-examples/blob/sdk-preview/notebooks/mlflow/mlflow-v1-comparison.ipynb)." + ] + } + ], + "metadata": { + "celltoolbar": "Edit Metadata", + "description": { + "description": "Create production ML pipelines with Python SDK v2 in a Jupyter notebook" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK v2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.17" + }, + "nteract": { + "version": "nteract-front-end@1.0.0" + } + }, + "nbformat": 4, + "nbformat_minor": 1 } diff --git a/tutorials/get-started-notebooks/quickstart.ipynb b/tutorials/get-started-notebooks/quickstart.ipynb index c1777d1314..d0ee2bc8c2 100644 --- a/tutorials/get-started-notebooks/quickstart.ipynb +++ b/tutorials/get-started-notebooks/quickstart.ipynb @@ -694,9 +694,9 @@ "name": "python310-sdkv2" }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.10 - SDK v2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "codemirror_mode": { diff --git a/tutorials/readme.py b/tutorials/readme.py index f19959f93a..40dd1c8224 100644 --- a/tutorials/readme.py +++ b/tutorials/readme.py @@ -162,7 +162,7 @@ def write_notebook_workflow( - name: setup python uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: "3.11" - name: pip install notebook reqs run: pip install -r sdk/python/dev-requirements.txt{mlflow_import}{forecast_import} - name: azure login @@ -186,7 +186,7 @@ def write_notebook_workflow( continue-on-error: true - name: validate readme run: | - python check-readme.py "{github_workspace}" "{github_workspace}/tutorials/{posix_folder}" + python check-readme.py "{github_workspace}/tutorials/{posix_folder}" working-directory: infra/bootstrapping continue-on-error: false - name: setup-cli