_test.yml

name: "> Test"

on:
  workflow_call:
    inputs:
      platform_environment:
        description: 'Environment to test against, that is staging or production'
        required: false
        default: 'staging'
        type: string
      commit_message:
        description: 'Commit message to check for skip/enable markers'
        required: false
        type: string
    secrets:
      AIGNOSTICS_CLIENT_ID_DEVICE_STAGING:
        required: false
      AIGNOSTICS_REFRESH_TOKEN_STAGING:
        required: false
      GCP_CREDENTIALS_STAGING:
        required: false
      AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION:
        required: false
      AIGNOSTICS_REFRESH_TOKEN_PRODUCTION:
        required: false
      GCP_CREDENTIALS_PRODUCTION:
        required: false
      CODECOV_TOKEN:
        required: false
      SONAR_TOKEN:
        required: false
      SENTRY_DSN:
        required: false
  
env:
  # https://gist.github.com/NodeJSmith/e7e37f2d3f162456869f015f842bcf15
  PYTHONIOENCODING: "utf8"
  AIGNOSTICS_PLATFORM_ENVIRONMENT: ${{ inputs.platform_environment }}

jobs:

  generate-matrix:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Set matrix based on commit message and PR labels
        id: set-matrix
        env:
          COMMIT_MESSAGE: ${{ inputs.commit_message }}
          PR_LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }}
        run: |
          if [[ "$COMMIT_MESSAGE" == *"skip:test:matrix-runner"* ]] || [[ "$PR_LABELS" == *"skip:test:matrix-runner"* ]]; then
            echo 'matrix={"runner":["ubuntu-latest"],"experimental":[false]}' >> $GITHUB_OUTPUT
          else
            echo 'matrix={"runner":["ubuntu-latest"],"experimental":[false],"include":[{"runner":"ubuntu-24.04-arm","experimental":true},{"runner":"macos-latest","experimental":true},{"runner":"macos-15-intel","experimental":true},{"runner":"windows-latest","experimental":true}]}' >> $GITHUB_OUTPUT
          fi

  test:
    needs: generate-matrix
    runs-on: ${{ matrix.runner }}
    continue-on-error: ${{ matrix.experimental }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }}
    permissions:
      attestations: write
      contents: read
      id-token: write
      packages: read
    steps:

      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0

      - name: Install uv
        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
        with:
          version-file: "pyproject.toml"
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Install dev tools
        if: ${{ matrix.runner == 'ubuntu-latest' || matrix.runner == 'ubuntu-24.04-arm' }}
        shell: bash
        run: .github/workflows/_install_dev_tools.bash

      - name: Install Python, venv and dependencies
        shell: bash
        run: uv sync --all-extras --frozen --link-mode=copy

      # Need xdisplay for testing QuPath app integration
      - name: Setup display
        uses: pyvista/setup-headless-display-action@5bc8de3bc71fcda7a96439571287a554901541a0 # v4.3
  
      - name: Release version check
        if: startsWith(github.ref, 'refs/tags/v')
        shell: bash
        run: |
          TAG_VERSION=${GITHUB_REF#refs/tags/v}
          TOML_VERSION=$(uv run --frozen python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])")
          if [ "$TAG_VERSION" != "$TOML_VERSION" ]; then
            echo "Release version mismatch: Tag $TAG_VERSION != pyproject.toml $TOML_VERSION"
            exit 1
          fi

      - name: Print development version info
        if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
        shell: bash
        run: |
          TOML_VERSION=$(uv run --frozen python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])")
          echo "Development build - Current version in pyproject.toml: $TOML_VERSION"

      - name: Create .env file
        uses: SpicyPizza/create-envfile@ace6d4f5d7802b600276c23ca417e669f1a06f6f # v2.0.3
        with:
          # The following 3 lines are correct even if vscode complains
          envkey_AIGNOSTICS_API_ROOT: ${{ inputs.platform_environment == 'staging' && 'https://platform-staging.aignostics.com' || 'https://platform.aignostics.com' }}
          envkey_AIGNOSTICS_CLIENT_ID_DEVICE: ${{ inputs.platform_environment == 'staging' && secrets.AIGNOSTICS_CLIENT_ID_DEVICE_STAGING || secrets.AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION }}
          envkey_AIGNOSTICS_REFRESH_TOKEN: ${{ inputs.platform_environment == 'staging' && secrets.AIGNOSTICS_REFRESH_TOKEN_STAGING || secrets.AIGNOSTICS_REFRESH_TOKEN_PRODUCTION }}
          envkey_AIGNOSTICS_SENTRY_ENABLED: true
          envkey_AIGNOSTICS_SENTRY_DSN: ${{ secrets.SENTRY_DSN}}
          envkey_AIGNOSTICS_SENTRY_DEBUG: false
          envkey_AIGNOSTICS_SENTRY_SEND_DEFAULT_PII: false
          envkey_AIGNOSTICS_SENTRY_SAMPLE_RATE: 0
          envkey_AIGNOSTICS_SENTRY_TRACES_SAMPLE_RATE: 0
          envkey_AIGNOSTICS_SENTRY_PROFILES_SAMPLE_RATE: 0
          fail_on_empty: false

      - name: Set up GCP credentials for bucket access
        shell: bash
        env:
          GCP_CREDENTIALS: ${{ inputs.platform_environment == 'staging' && secrets.GCP_CREDENTIALS_STAGING || secrets.GCP_CREDENTIALS_PRODUCTION }}
        run: |
          echo "$GCP_CREDENTIALS" | base64 -d > credentials.json
          echo "GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/credentials.json" >> $GITHUB_ENV

      - name: Validate installation (single Python version)
        shell: bash
        run: |
            OUTPUT=$(uv run --frozen --no-dev aignostics --help)
            if [[ "$OUTPUT" != *"built with love in Berlin"* ]]; then
            echo "Output does not contain 'built with love in Berlin'"
            exit 1
            fi

      - name: Test / Smoke (single Python version)
        shell: bash
        run: |
          uv run --frozen --no-dev aignostics --help
          uv run --frozen --all-extras aignostics system info
          uv run --frozen --all-extras aignostics system health
          uv run --frozen --all-extras aignostics user whoami --mask-secrets
          uv run --frozen --all-extras aignostics application list
          uv run --frozen --all-extras aignostics application run list --verbose --limit 1

      # All test steps use continue-on-error: true so that every test suite
      # always runs regardless of whether earlier suites failed. This ensures
      # Codecov always receives a complete coverage report (partial reports
      # cause the coverage badge to read lower than the true value).
      # A single "Assert no test failures" gate step at the end surfaces any
      # failures and fails the job — so the workflow still fails correctly.

      - name: Test / Unit (multiple Python versions)
        id: unit
        continue-on-error: true
        if: ${{ !cancelled() }}
        uses: ./.github/actions/run-tests
        with:
          test-type: unit
          make-target: test_unit_matrix
          skip-marker: skip:test:unit
          summary-title: All unit tests passed
          commit-message: ${{ inputs.commit_message }}

      - name: Test / Integration (multiple Python versions)
        id: integration
        continue-on-error: true
        if: ${{ !cancelled() }}
        uses: ./.github/actions/run-tests
        with:
          test-type: integration
          make-target: test_integration_matrix
          skip-marker: skip:test:integration
          summary-title: All integration tests passed
          commit-message: ${{ inputs.commit_message }}

      - name: Test / E2E / regular (multiple Python versions)
        id: e2e_regular
        continue-on-error: true
        if: ${{ !cancelled() }}
        uses: ./.github/actions/run-tests
        with:
          test-type: e2e
          make-target: test_e2e_matrix
          skip-marker: skip:test:e2e
          summary-title: All regular e2e tests passed
          commit-message: ${{ inputs.commit_message }}

      - name: Test / E2E / long running (single Python version)
        id: e2e_long_running
        continue-on-error: true
        if: ${{ !cancelled() }}
        uses: ./.github/actions/run-tests
        with:
          test-type: long-running
          make-target: test_long_running
          skip-marker: skip:test:long_running
          summary-title: All long running e2e tests passed
          commit-message: ${{ inputs.commit_message }}

      - name: Test / E2E / very long running (single Python version)
        id: e2e_very_long_running
        continue-on-error: true
        if: |
          !cancelled() && (
            contains(inputs.commit_message, 'enable:test:very_long_running') ||
            contains(github.event.pull_request.labels.*.name, 'enable:test:very_long_running') ||
            ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && startsWith(github.ref, 'refs/heads/release/v'))
          )
        uses: ./.github/actions/run-tests
        with:
          test-type: very-long-running
          make-target: test_very_long_running
          summary-title: All very long running e2e tests passed
          commit-message: ${{ inputs.commit_message }}

      - name: Upload test artifacts for inspection
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ always() && (env.GITHUB_WORKFLOW_RUNTIME != 'ACT') }}
        with:
          name: test-results-${{ matrix.runner }}
          path: |
            reports/mypy_junit.xml
            reports/junit_*.xml
            reports/coverage.xml
            reports/coverage.md
            reports/coverage_html
          retention-days: 7

      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
        if: |
          !cancelled() &&
          (env.GITHUB_WORKFLOW_RUNTIME != 'ACT') &&
          matrix.runner == 'ubuntu-latest' &&
          !contains(inputs.commit_message, 'skip:codecov') &&
          !contains(github.event.pull_request.labels.*.name, 'skip:codecov')
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          slug: aignostics/python-sdk

      - name: Upload test results to Codecov
        if: |
          !cancelled() &&
          (env.GITHUB_WORKFLOW_RUNTIME != 'ACT') &&
          matrix.runner == 'ubuntu-latest' &&
          !contains(inputs.commit_message, 'skip:codecov') &&
          !contains(github.event.pull_request.labels.*.name, 'skip:codecov')
        uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3 # v1.2.1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

      - name: SonarQube Scan
        if: ${{ !cancelled() && (env.GITHUB_WORKFLOW_RUNTIME != 'ACT') && matrix.runner == 'ubuntu-latest' }}
        uses: SonarSource/sonarqube-scan-action@22918119ff8e1ca75a623e15c8296b6ea4fbe28f # v8.2.1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

      - name: Assert no test failures
        # Single gate that fails the job if any test suite failed. Kept last so
        # that all reporting steps (Codecov, SonarQube, artifact upload) always
        # run to completion before the job is marked as failed.
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          failed=()
          [[ "${{ steps.unit.outcome }}"                  == "failure" ]] && failed+=("unit")
          [[ "${{ steps.integration.outcome }}"           == "failure" ]] && failed+=("integration")
          [[ "${{ steps.e2e_regular.outcome }}"           == "failure" ]] && failed+=("e2e_regular")
          [[ "${{ steps.e2e_long_running.outcome }}"      == "failure" ]] && failed+=("e2e_long_running")
          [[ "${{ steps.e2e_very_long_running.outcome }}" == "failure" ]] && failed+=("e2e_very_long_running")
          if [[ ${#failed[@]} -gt 0 ]]; then
            echo "The following test suites failed: ${failed[*]}"
            exit 1
          fi
          echo "All test suites passed."