test.yml

---
name: Test

on:
  push:
    branches: [main]
  pull_request: {}
  schedule:
    # * is a special character in YAML so you have to quote this string
    # Run at 1:00 every day
    - cron: 0 1 * * *
  workflow_dispatch: {}

# We share Vuforia credentials and therefore Vuforia databases across
# workflows. We therefore want to run only one workflow at a time.
concurrency: vuforia_credentials

permissions: {}

jobs:
  # CI tests with matrix
  ci-tests:
    runs-on: ubuntu-latest
    environment: vuforia
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.15']
        ci_pattern:
          - tests/mock_vws/query/test_content_type.py
          - tests/mock_vws/query/test_success.py
          - tests/mock_vws/query/test_incorrect_fields.py
          - tests/mock_vws/query/test_max_num_results.py
          - tests/mock_vws/query/test_result_order.py
          - tests/mock_vws/query/test_include_target_data.py
          - tests/mock_vws/query/test_accept_header.py
          - tests/mock_vws/query/test_active_flag.py
          - tests/mock_vws/query/test_bad_image.py
          - tests/mock_vws/query/test_maximum_image_file_size.py
          - tests/mock_vws/query/test_maximum_image_dimensions.py
          - tests/mock_vws/query/test_image_formats.py
          - tests/mock_vws/query/test_processing.py
          - tests/mock_vws/query/test_update.py
          - tests/mock_vws/query/test_deleted.py
          - tests/mock_vws/query/test_target_status_failed.py
          - tests/mock_vws/query/test_date_formats.py
          - tests/mock_vws/query/test_inactive_project.py
          - tests/mock_vws/query/test_truncated_body.py
          - tests/mock_vws/add_target/test_content_types.py
          - tests/mock_vws/add_target/test_missing_data.py
          - tests/mock_vws/add_target/test_width.py
          - tests/mock_vws/add_target/test_target_name.py
          - tests/mock_vws/add_target/test_image.py
          - tests/mock_vws/add_target/test_active_flag.py
          - tests/mock_vws/add_target/test_unexpected_data.py
          - tests/mock_vws/add_target/test_application_metadata.py
          - tests/mock_vws/add_target/test_inactive_project.py
          - tests/mock_vws/authorization_header/test_authorization_header.py
          - tests/mock_vws/authorization_header/test_malformed.py::test_one_part_no_space
          - tests/mock_vws/authorization_header/test_malformed.py::test_one_part_with_space
          - tests/mock_vws/authorization_header/test_malformed.py::test_missing_signature
          - tests/mock_vws/authorization_header/test_bad_key.py
          - tests/mock_vws/test_cloud_query_failure_response.py
          - tests/mock_vws/content_length/test_incorrect.py::test_not_integer
          - tests/mock_vws/content_length/test_incorrect.py::test_too_large
          - tests/mock_vws/content_length/test_incorrect.py::test_too_small
          - tests/mock_vws/header_size/test_oversized_header_line.py::test_header_too_large
          - tests/mock_vws/header_size/test_oversized_header_line.py::test_cookie_too_large
          - tests/mock_vws/header_size/test_oversized_header_line.py::test_large_header_within_limit
          - tests/mock_vws/database_summary/contract/test_database_summary.py::test_success
          - tests/mock_vws/database_summary/contract/test_database_summary.py::test_active_images
          - tests/mock_vws/database_summary/contract/test_database_summary.py::test_failed_images
          - tests/mock_vws/database_summary/contract/test_database_summary.py::test_inactive_images
          - tests/mock_vws/database_summary/contract/test_database_summary.py::test_inactive_failed
          - tests/mock_vws/database_summary/contract/test_database_summary.py::test_deleted
          - tests/mock_vws/database_summary/test_database_summary.py::test_processing_images
          - tests/mock_vws/database_summary/contract/test_quotas.py
          - tests/mock_vws/database_summary/contract/test_recos.py
          - tests/mock_vws/database_summary/contract/test_request_usage.py
          - tests/mock_vws/database_summary/contract/test_inactive_project.py
          - tests/mock_vws/date_header/test_format.py
          - tests/mock_vws/date_header/test_missing.py
          - tests/mock_vws/date_header/test_skewed_time.py::test_date_out_of_range_after
          - tests/mock_vws/date_header/test_skewed_time.py::test_date_out_of_range_before
          - tests/mock_vws/date_header/test_skewed_time.py::test_date_in_range_after
          - tests/mock_vws/date_header/test_skewed_time.py::test_date_in_range_before
          - tests/mock_vws/delete_target
          - tests/mock_vws/get_duplicates
          - tests/mock_vws/get_target
          - tests/mock_vws/invalid_given_id
          - tests/mock_vws/invalid_json/test_invalid_json.py::test_not_json
          - tests/mock_vws/invalid_json/test_invalid_json.py::test_not_an_object
          - tests/mock_vws/invalid_json/test_invalid_json.py::test_not_utf_8
          - tests/mock_vws/invalid_json/test_invalid_json.py::test_empty_body
          - tests/mock_vws/invalid_json/test_invalid_json.py::test_invalid_json_with_skewed_time
          - tests/mock_vws/target_list
          - tests/mock_vws/reco_counts_report
          - tests/mock_vws/test_target_raters.py
          - tests/mock_vws/target_summary
          - tests/mock_vws/unexpected_json
          - tests/mock_vws/update_target/test_active_flag.py
          - tests/mock_vws/update_target/test_application_metadata.py::test_base64_encoded
          - tests/mock_vws/update_target/test_application_metadata.py::test_invalid_type
          - tests/mock_vws/update_target/test_application_metadata.py::test_not_base64_encoded_processable
          - tests/mock_vws/update_target/test_application_metadata.py::test_not_base64_encoded_not_processable
          - tests/mock_vws/update_target/test_application_metadata.py::test_metadata_too_large
          - tests/mock_vws/update_target/test_image.py::test_image_valid
          - tests/mock_vws/update_target/test_image.py::test_bad_image_format_or_color_space
          - tests/mock_vws/update_target/test_image.py::test_corrupted
          - tests/mock_vws/update_target/test_image.py::test_truncated
          - tests/mock_vws/update_target/test_image.py::test_image_too_large
          - tests/mock_vws/update_target/test_image.py::test_not_base64_encoded_processable
          - tests/mock_vws/update_target/test_image.py::test_not_base64_encoded_not_processable
          - tests/mock_vws/update_target/test_image.py::test_not_image
          - tests/mock_vws/update_target/test_image.py::test_invalid_type
          - tests/mock_vws/update_target/test_image.py::test_rating_does_not_return_to_minus_one
          - tests/mock_vws/update_target/test_image.py::test_rating_can_change
          - tests/mock_vws/update_target/test_target_name.py::test_name_valid
          - tests/mock_vws/update_target/test_target_name.py::test_name_invalid
          - tests/mock_vws/update_target/test_target_name.py::test_existing_target_name
          - tests/mock_vws/update_target/test_target_name.py::test_same_name_given
          - tests/mock_vws/update_target/test_unexpected_data.py
          - tests/mock_vws/update_target/test_update.py
          - tests/mock_vws/update_target/test_width.py
          - tests/mock_vws/update_target/test_inactive_project.py
          - tests/mock_vws/test_model_target_generation_failure.py
          - tests/mock_vws/test_model_target_generation_warning.py
          - tests/mock_vws/test_model_target_failure_response.py
          - tests/mock_vws/test_model_target_training_allowance.py
          - tests/mock_vws/model_target_web_api
          - tests/mock_vws/test_model_target_retries.py
          - tests/mock_vws/vumark_generation_api
          - tests/mock_vws/test_vumark_generation_failure.py
          - tests/mock_vws/test_target_validators.py
          - tests/mock_vws/requests_mock
          - tests/mock_vws/respx_mock
          - tests/mock_vws/httpx2_mock
          - tests/mock_vws/flask_app
          - tests/mock_vws/test_healthcheck.py
          - tests/mock_vws/test_docker.py
          - README.rst
          - docs/

    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@v10.3.0
        with:
          enable-cache: true
          cache-dependency-glob: '**/pyproject.toml'

      - name: Set secrets file
        env:
          CI_PATTERN: ${{ matrix.ci_pattern }}
          ENCRYPTED_FILE: secrets.tar.gpg
          LARGE_SECRET_PASSPHRASE: ${{ secrets.PASSPHRASE_FOR_VUFORIA_SECRETS }}
          JOB_INDEX: ${{ strategy.job-index }}
        run: |
          # See the "CI Setup" document for details of how this was set up.
          ci/decrypt_secret.sh
          tar xvf "${HOME}"/secrets/secrets.tar
          # Matrix jobs are 0-based; only secrets 0-99 exist, so patterns
          # past index 99 wrap and share a database with an earlier job.
          # The ci_pattern list is therefore ordered so that every pattern
          # from index 100 on is one which never parametrises over the Real
          # Vuforia backend, and so never uses these credentials.
          SECRET_INDEX=$((JOB_INDEX % 100))
          cp "./ci_secrets/vuforia_secrets_${SECRET_INDEX}.env" ./vuforia_secrets.env

      # We have seen issues with running out of disk space on test_docker
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        if: matrix.ci_pattern == 'tests/mock_vws/test_docker.py'
        with:
          # All of these default to true (meaning they are removed).
          docker-images: false
          large-packages: false
          swap-storage: false
          tool-cache: false

          android: true
          dotnet: true
          haskell: true

      - name: Run tests
        run: |
          uv run --group=dev \
            coverage run -m pytest \
              -s \
              -vvv \
              --showlocals \
              --exitfirst \
              ${{ matrix.ci_pattern }}
        env:
          UV_PYTHON: ${{ matrix.python-version }}

      - name: Sanitize pattern for artifact name
        id: sanitize
        run: |
          SANITIZED_PATTERN=$(echo "${{ matrix.ci_pattern }}" | sed 's/::/-/g' | sed 's/:/-/g' | sed 's|/|-|g')
          echo "name=coverage-data-ci-${{ matrix.python-version }}-${SANITIZED_PATTERN}" >> "$GITHUB_OUTPUT"

      - name: Upload coverage data
        uses: actions/upload-artifact@v7
        with:
          name: ${{ steps.sanitize.outputs.name }}
          path: .coverage.*
          include-hidden-files: true
          if-no-files-found: error

  # Skip tests
  skip-tests:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.15']
        platform: [ubuntu-latest]

    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@v10.3.0
        with:
          enable-cache: true
          cache-dependency-glob: '**/pyproject.toml'

      - name: Set secrets file
        run: |
          cp ./vuforia_secrets.env.example ./vuforia_secrets.env

      - name: Run tests
        run: |
          uv run --group=dev \
            coverage run -m pytest \
              --skip-marker=requires_docker_build \
              --skip-backend=flask_in_process \
              --skip-backend=mock \
              --skip-backend=real \
              --capture=no \
              -vvv \
              --exitfirst \
              .
        env:
          UV_PYTHON: ${{ matrix.python-version }}

      - name: Upload coverage data
        uses: actions/upload-artifact@v7
        with:
          name: coverage-data-skip-tests-${{ matrix.python-version }}
          path: .coverage.*
          include-hidden-files: true
          if-no-files-found: error

  # Windows tests
  windows-tests:
    runs-on: windows-latest
    strategy:
      matrix:
        python-version: ['3.15']

    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@v10.3.0
        with:
          enable-cache: true
          cache-dependency-glob: '**/pyproject.toml'

      - name: Set secrets file
        run: |
          cp ./vuforia_secrets.env.example ./vuforia_secrets.env

      - name: Start Docker
        shell: pwsh
        run: |
          $service = Get-Service docker -ErrorAction Stop
          if ($service.Status -ne "Running") {
            Start-Service docker
          }
          $deadline = (Get-Date).AddMinutes(2)
          do {
            docker info
            if ($LASTEXITCODE -eq 0) {
              exit 0
            }
            Start-Sleep -Seconds 5
          } while ((Get-Date) -lt $deadline)
          docker info

      - name: Run tests
        shell: bash
        run: |
          # We use pytest-xdist to make this run much faster.
          # The downside is that we cannot use -s / --capture=no.
          #
          uv run --group=dev \
            coverage run -m pytest \
              --skip-backend=real \
              -vvv \
              --exitfirst \
              -n auto \
              .
        env:
          UV_PYTHON: ${{ matrix.python-version }}

      - name: Upload coverage data
        uses: actions/upload-artifact@v7
        with:
          name: coverage-data-windows-${{ matrix.python-version }}
          path: .coverage.*
          include-hidden-files: true
          if-no-files-found: error

  coverage:
    name: Combine & check coverage
    needs: [ci-tests, skip-tests, windows-tests]
    if: ${{ !cancelled() }}
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@v10.3.0
        with:
          enable-cache: true
          cache-dependency-glob: '**/pyproject.toml'
          python-version: '3.15'

      - uses: actions/download-artifact@v8
        with:
          pattern: coverage-data-*
          merge-multiple: true

      - name: Require 100% Coverage
        id: coverage
        run: |
          uv run --group=dev coverage combine
          uv run --group=dev coverage html --skip-covered --skip-empty

          # Report and write to summary, without failing yet.
          uv run --group=dev coverage report --format=markdown \
            >> "$GITHUB_STEP_SUMMARY" || true

          # Report again and fail if under 100%.
          uv run --group=dev coverage report

      - name: Upload HTML report if check failed
        uses: actions/upload-artifact@v7
        with:
          name: html-report
          path: htmlcov
        if: ${{ failure() }}

  # Final completion check
  completion-ci:
    needs: [ci-tests, skip-tests, windows-tests, coverage]
    runs-on: ubuntu-latest
    if: ${{ !cancelled() }}
    steps:
      - name: Check all jobs status
        run: |-
          if ! ${{ needs.ci-tests.result == 'success' }}; then
            echo "CI tests failed"
            exit 1
          fi
          if ! ${{ needs.skip-tests.result == 'success' }}; then
            echo "Skip tests failed"
            exit 1
          fi
          if ! ${{ needs.windows-tests.result == 'success' }}; then
            echo "Windows tests failed"
            exit 1
          fi
          if ! ${{ needs.coverage.result == 'success' }}; then
            echo "Coverage check failed"
            exit 1
          fi