ci: display HTML test report URLs in check run summaries (#252) #1222
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Connectors | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| # TODO: Consider moving to run these only after the "PyTest (Fast)" workflow is successful. | |
| # workflow_run: | |
| # workflows: [PyTest (Fast)] | |
| # types: | |
| # - completed | |
| concurrency: | |
| # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow. | |
| # | |
| # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests | |
| # ensuring that only one run per pull request is active at a time. | |
| # | |
| # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches. | |
| # This allows us to run multiple workflow dispatches in parallel. | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| cdk_changes: | |
| name: Get Changes | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| statuses: write | |
| pull-requests: read | |
| steps: | |
| - name: Checkout Airbyte | |
| if: github.event_name != 'pull_request' | |
| uses: actions/checkout@v4 | |
| - id: changes | |
| uses: dorny/[email protected] | |
| with: | |
| filters: | | |
| src: | |
| - 'airbyte_cdk/**' | |
| - 'bin/**' | |
| - 'poetry.lock' | |
| - 'pyproject.toml' | |
| file-based: | |
| - 'airbyte_cdk/sources/file_based/**' | |
| vector-db-based: | |
| - 'airbyte_cdk/destinations/vector_db_based/**' | |
| sql: | |
| - 'airbyte_cdk/sql/**' | |
| outputs: | |
| # Source code modified: | |
| src: ${{ steps.changes.outputs.src }} | |
| # Extras modified: | |
| file-based: ${{ steps.changes.outputs.file-based }} | |
| vector-db-based: ${{ steps.changes.outputs.vector-db-based }} | |
| sql: ${{ steps.changes.outputs.sql }} | |
| connectors_ci: | |
| needs: cdk_changes | |
| # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR | |
| # Forked PRs are handled by the community_ci.yml workflow | |
| # If the condition is not met the job will be skipped (it will not fail) | |
| # runs-on: connector-test-large | |
| runs-on: ubuntu-22.04 # Ubuntu 24 doesn't work as of now | |
| timeout-minutes: 360 # 6 hours | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - connector: source-hardcoded-records | |
| cdk_extra: n/a | |
| - connector: source-shopify | |
| cdk_extra: n/a | |
| # Chargebee is being flaky: | |
| # - connector: source-chargebee | |
| # cdk_extra: n/a | |
| # These two are behind in CDK updates and can't be used as tests until they are updated: | |
| # - connector: source-s3 | |
| # cdk_extra: file-based | |
| # - connector: destination-pinecone | |
| # cdk_extra: vector-db-based | |
| - connector: destination-motherduck | |
| cdk_extra: sql | |
| # ZenDesk currently failing (as of 2024-12-02) | |
| # TODO: Re-enable once fixed | |
| # - connector: source-zendesk-support | |
| # cdk_extra: n/a | |
| # TODO: These are manifest connectors and won't work as expected until we | |
| # add `--use-local-cdk` support for manifest connectors. | |
| - connector: source-the-guardian-api | |
| cdk_extra: n/a | |
| - connector: source-pokeapi | |
| cdk_extra: n/a | |
| name: "${{matrix.connector}} Connector Job (skip=${{needs.cdk_changes.outputs['src'] == 'false' || needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})" | |
| permissions: | |
| checks: write | |
| contents: write # Required for creating commit statuses | |
| pull-requests: read | |
| steps: | |
| - name: Abort if extra not changed (${{matrix.cdk_extra}}) | |
| id: no_changes | |
| if: ${{ needs.cdk_changes.outputs['src'] == 'false' || matrix.cdk_extra != 'n/a' && needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' }} | |
| run: | | |
| echo "Aborting job." | |
| echo "Source code changed: ${{ needs.cdk_changes.outputs['src'] }}" | |
| if [ "${{ matrix.cdk_extra }}" != "n/a" ]; then | |
| echo "Extra not changed: ${{ matrix.cdk_extra }} = ${{ needs.cdk_changes.outputs[matrix.cdk_extra] }}" | |
| fi | |
| echo "> Skipped '${{matrix.connector}}' (no relevant changes)" >> $GITHUB_STEP_SUMMARY | |
| echo "status=cancelled" >> $GITHUB_OUTPUT | |
| exit 0 | |
| continue-on-error: true | |
| # Get the monorepo so we can test the connectors | |
| - name: Checkout CDK | |
| if: steps.no_changes.outputs.status != 'cancelled' | |
| uses: actions/checkout@v4 | |
| with: | |
| path: airbyte-python-cdk | |
| - name: Checkout Airbyte Monorepo | |
| uses: actions/checkout@v4 | |
| if: steps.no_changes.outputs.status != 'cancelled' | |
| with: | |
| repository: airbytehq/airbyte | |
| ref: master | |
| path: airbyte | |
| token: ${{ github.token }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| # Initial status will be created by the job itself | |
| - name: Test Connector | |
| if: steps.no_changes.outputs.status != 'cancelled' | |
| timeout-minutes: 90 | |
| env: | |
| GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" | |
| run: | | |
| cd airbyte | |
| make tools.airbyte-ci-dev.install | |
| airbyte-ci \ | |
| --ci-report-bucket-name=airbyte-ci-reports-multi \ | |
| connectors \ | |
| --name ${{matrix.connector}} \ | |
| --use-local-cdk \ | |
| test \ | |
| --fail-fast \ | |
| --skip-step qa_checks \ | |
| --skip-step connector_live_tests | |
| - name: Evaluate Test | |
| id: evaluate_output | |
| if: always() && steps.no_changes.outputs.status != 'cancelled' | |
| run: | | |
| # save job output json file as ci step output | |
| mkdir -p airbyte/airbyte-ci/connectors/pipelines/pipeline_reports | |
| echo "Debug: Searching for output.json in pipeline reports directory" | |
| find airbyte/airbyte-ci/connectors/pipelines/pipeline_reports -type f -ls | |
| json_output_file=$(find airbyte/airbyte-ci/connectors/pipelines/pipeline_reports -name 'output.json' -print -quit || true) | |
| echo "Debug: Found output.json at: ${json_output_file}" | |
| if [ -z "$json_output_file" ] || [ ! -f "$json_output_file" ]; then | |
| echo "No output.json found - test may have failed to generate report" | |
| echo "## Job Output for ${{matrix.connector}}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Test failed to generate report" >> $GITHUB_STEP_SUMMARY | |
| echo "::error::Test failed for connector '${{ matrix.connector }}' - no output report generated" | |
| exit 1 | |
| fi | |
| echo "Debug: Contents of output.json:" | |
| cat ${json_output_file} | |
| job_output=$(cat ${json_output_file}) | |
| success=$(echo ${job_output} | jq -r '.success') | |
| failed_step=$(echo ${job_output} | jq -r '.failed_steps | select(length > 0) | .[0] // "None"') | |
| run_duration=$(echo ${job_output} | jq -r '.run_duration') | |
| html_report_url=$(echo ${job_output} | jq -r '.html_report_url') | |
| echo "Debug: Extracted values from output.json:" | |
| echo "- Success: ${success}" | |
| echo "- Failed Step: ${failed_step}" | |
| echo "- Run Duration: ${run_duration}" | |
| echo "- HTML Report URL: ${html_report_url}" | |
| echo "## Job Output for ${{matrix.connector}}" >> $GITHUB_STEP_SUMMARY | |
| echo "- [HTML Report](${html_report_url})" >> $GITHUB_STEP_SUMMARY | |
| echo "- Success: ${success}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Test Duration: $(printf "%.0f" ${run_duration})s" >> $GITHUB_STEP_SUMMARY | |
| if [ "${success}" != "true" ]; then | |
| echo "- Failed Step: ${failed_step}" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo -e "\n[Download Job Output](${{steps.upload_job_output.outputs.artifact-url}})" >> $GITHUB_STEP_SUMMARY | |
| if [ "${success}" != "true" ]; then | |
| echo "::error::Test failed for connector '${{ matrix.connector }}' on step '${failed_step}'. " | |
| exit 1 | |
| fi | |
| echo "See the execution report for details: ${html_report_url}" | |
| echo "Debug: Setting outputs for other steps" | |
| echo "success=${success}" >> $GITHUB_OUTPUT | |
| echo "html_report_url=${html_report_url}" >> $GITHUB_OUTPUT | |
| echo "Debug: Current GITHUB_OUTPUT contents:" | |
| cat $GITHUB_OUTPUT | |
| # Update the test report check run with results | |
| - name: Create Test Report Status | |
| if: always() && steps.no_changes.outputs.status != 'cancelled' && steps.evaluate_output.outcome == 'success' | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | |
| run: | | |
| HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}" | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| repos/${{ github.repository }}/statuses/$HEAD_SHA \ | |
| -f state="${{ steps.evaluate_output.outputs.success == 'true' && 'success' || 'failure' }}" \ | |
| -f target_url="${{ steps.evaluate_output.outputs.html_report_url }}" \ | |
| -f description="Click Details to view the test report" \ | |
| -f context="${{ matrix.connector }} Test Report" | |
| # Create failure status if report generation failed | |
| - name: Create Report Generation Failed Status | |
| if: always() && steps.no_changes.outputs.status != 'cancelled' && steps.evaluate_output.outcome != 'success' | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | |
| run: | | |
| HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}" | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| repos/${{ github.repository }}/statuses/$HEAD_SHA \ | |
| -f state="failure" \ | |
| -f description="Test execution failed to generate a report" \ | |
| -f context="${{ matrix.connector }} Test Report" | |
| # Upload the job output to the artifacts | |
| - name: Upload Job Output | |
| id: upload_job_output | |
| if: always() && steps.no_changes.outputs.status != 'cancelled' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{matrix.connector}}-job-output | |
| path: airbyte/airbyte-ci/connectors/pipelines/pipeline_reports |