test_evaluation_function.yml

name: Endpoint Validation Test

on:
  workflow_call:
    inputs:
      eval_function:
        description: 'Evaluation Function Name'
        required: true
        type: string
      sql_limit:
        description: 'Max number of records to fetch'
        required: false
        type: string
        default: 1000
      request_delay:
        description: 'Delay (seconds) between dispatching requests'
        required: false
        type: string
        default: '0'
      max_concurrency:
        description: 'Max concurrent requests (lower for GPT-backed functions)'
        required: false
        type: string
        default: '5'
      seed:
        description: 'Random seed for reproducible sampling (float in [-1.0, 1.0]). Leave blank to auto-generate.'
        required: false
        type: string
        default: ''
      pass_threshold:
        description: 'Minimum pass rate % (0-100) required for the test to succeed. Default 100 means any error fails the run.'
        required: false
        type: string
        default: '100'
      ids_to_exclude:
        description: 'Comma-separated submission UUIDs to add to the exclusion list before testing.'
        required: false
        type: string
        default: ''
      api_mode:
        description: 'API contract to test against: legacy or mued.'
        required: false
        type: string
        default: 'legacy'
    secrets:
      TEST_API_ENDPOINT:
        description: 'API Endpoint URL to test'
        required: false
      DB_USER:
        required: false
      DB_PASSWORD:
        required: false
      DB_HOST:
        required: false
      DB_PORT:
        required: false
      DB_NAME:
        required: false
      GCP_SERVICE_ACCOUNT_KEY:
        required: false
      GCP_PROJECT_ID:
        required: false
  workflow_dispatch:
    inputs:
      eval_function:
        description: 'Evaluation Function Name'
        required: true
        type: string
      endpoint:
        description: 'API Endpoint URL to test'
        required: true
        type: string
      ids_to_exclude:
        description: 'Comma-separated submission UUIDs to add to the exclusion list before testing.'
        required: false
        type: string
        default: ''
      sql_limit:
        description: 'Max number of records to fetch'
        required: false
        type: string
        default: '1000'
      request_delay:
        description: 'Delay (seconds) between dispatching requests'
        required: false
        type: string
        default: '0'
      max_concurrency:
        description: 'Max concurrent requests'
        required: false
        type: string
        default: '5'
      seed:
        description: 'Random seed for reproducible sampling (float in [-1.0, 1.0]). Leave blank to auto-generate.'
        required: false
        type: string
        default: ''
      pass_threshold:
        description: 'Minimum pass rate % (0-100) required for the test to succeed.'
        required: false
        type: string
        default: '100'
      api_mode:
        description: 'API contract to test against: legacy or mued.'
        required: false
        type: string
        default: 'legacy'

jobs:
  run_test:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          repository: 'lambda-feedback/Database-Testing'
          ref: main
          token: ${{ github.token }}
          path: Database-Testing
          submodules: true

      - name: Set up Python Environment
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install Dependencies
        working-directory: ./Database-Testing
        run: |
          set -euo pipefail
          pip install -r requirements.txt

      - name: Set up Node.js for muEd schema bundling
        if: inputs.api_mode == 'mued'
        uses: actions/setup-node@v4
        with:
          node-version: '24'

      - name: Bundle muEd OpenAPI schema
        if: inputs.api_mode == 'mued'
        working-directory: ./Database-Testing/vendor/mued-api
        run: |
          set -euo pipefail
          npm ci
          npm run bundle

      - name: Authenticate to GCP
        uses: google-github-actions/auth@v1
        with:
          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}

      - name: Add exclusions to Firestore
        if: inputs.ids_to_exclude != ''
        working-directory: ./Database-Testing
        env:
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
          GOOGLE_CREDENTIALS_JSON: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
        run: |
          set -euo pipefail
          IDS=$(echo "${{ inputs.ids_to_exclude }}" | tr ',' ' ')
          python3 manage_exclusions.py \
            --eval_function_name "${{ inputs.eval_function }}" \
            add --ids $IDS

      - name: Run Evaluation Function Against Database
        id: run_script
        working-directory: ./Database-Testing
        env:
          DB_USER: ${{ secrets.DB_USER }}
          DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
          DB_HOST: ${{ secrets.DB_HOST }}
          DB_PORT: ${{ secrets.DB_PORT }}
          DB_NAME: ${{ secrets.DB_NAME }}
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
          GOOGLE_CREDENTIALS_JSON: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
        run: |
          set -euo pipefail

          # Run the argparse-enabled script and capture exit code
          SEED_ARG=""
          if [ -n "${{ inputs.seed }}" ]; then
            SEED_ARG="--seed ${{ inputs.seed }}"
          fi

          if ! python3 test_evaluation_function.py \
            --endpoint "${{ secrets.TEST_API_ENDPOINT || inputs.endpoint }}" \
            --eval_function_name "${{ inputs.eval_function }}" \
            --sql_limit "${{ inputs.sql_limit }}" \
            --grade_params_json "" \
            --request_delay "${{ inputs.request_delay }}" \
            --max_concurrency "${{ inputs.max_concurrency }}" \
            --api_mode "${{ inputs.api_mode }}" \
            $SEED_ARG; then
            echo "::error::Python script failed with non-zero exit code"
            exit 1
          fi

          # Verify report_data.json exists (needed for summary)
          if [ ! -f report_data.json ]; then
            echo "::error::report_data.json not found!"
            exit 1
          fi

          REPORT_DATA="$(cat report_data.json)"
          
          # Check for error status in the JSON response
          STATUS="$(echo "$REPORT_DATA" | jq -r '.status // "unknown"')"
          if [ "$STATUS" = "failed" ]; then
            ERROR_MSG="$(echo "$REPORT_DATA" | jq -r '.error // "Unknown error"')"
            echo "::error::Script returned failed status: ${ERROR_MSG}"
            exit 1
          fi

          # Check pass rate against configurable threshold
          PASSES="$(echo "$REPORT_DATA" | jq -r '.pass_count // 0')"
          TESTED="$(echo "$REPORT_DATA" | jq -r '.tested_count // 0')"
          ERRORS="$(echo "$REPORT_DATA" | jq -r '.number_of_errors // 0')"
          THRESHOLD="${{ inputs.pass_threshold }}"

          if [ "$TESTED" -gt 0 ]; then
            PASS_RATE="$(echo "scale=4; $PASSES / $TESTED * 100" | bc -l)"
          else
            PASS_RATE="0"
          fi

          if [ "$(echo "$PASS_RATE < $THRESHOLD" | bc -l)" = "1" ]; then
            echo "::error file=test_evaluation_function.py::Pass rate ${PASS_RATE}% is below the ${THRESHOLD}% threshold. ${ERRORS} test(s) failed."
            exit 1
          fi

      - name: Create Job Summary Report
        if: always() # Run even if the previous step failed/found errors
        working-directory: ./Database-Testing
        run: |
          set -euo pipefail
          
          if [ ! -f report_data.json ]; then
            echo "## Evaluation Function Report" >> "$GITHUB_STEP_SUMMARY"
            echo "**Status:** Failed - No report data generated" >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi
          
          REPORT_DATA="$(cat report_data.json)"
          
          # Check if this is an error response
          STATUS="$(echo "$REPORT_DATA" | jq -r '.status // "unknown"')"
          if [ "$STATUS" = "failed" ]; then
            ERROR_MSG="$(echo "$REPORT_DATA" | jq -r '.error // "Unknown error"')"
            {
              echo "## Endpoint Validation Report"
              echo "---"
              echo "**Status:** Failed"
              echo "**Error:** ${ERROR_MSG}"
              echo "---"
            } >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi
          
          PASSES="$(echo "$REPORT_DATA" | jq -r '.pass_count // 0')"
          TOTAL="$(echo "$REPORT_DATA" | jq -r '.total_count // 0')"
          TESTED="$(echo "$REPORT_DATA" | jq -r '.tested_count // 0')"
          ERRORS="$(echo "$REPORT_DATA" | jq -r '.number_of_errors // 0')"
          FEEDBACK_WARNINGS="$(echo "$REPORT_DATA" | jq -r '.number_of_feedback_warnings // 0')"
          PARSING_WARNINGS="$(echo "$REPORT_DATA" | jq -r '.number_of_parsing_warnings // 0')"
          FIRESTORE_LINK="$(echo "$REPORT_DATA" | jq -r '.firestore_link // ""')"
          FIRESTORE_DOC_ID="$(echo "$REPORT_DATA" | jq -r '.firestore_doc_id // ""')"
          SEED="$(echo "$REPORT_DATA" | jq -r '.seed // "N/A"')"
          THRESHOLD="${{ inputs.pass_threshold }}"

          if [ "$TESTED" -gt 0 ]; then
            PASS_RATE="$(echo "scale=2; $PASSES / $TESTED * 100" | bc -l)"
          else
            PASS_RATE="0"
          fi

          STATUS_EMOJI="[PASS]"
          if [ "$(echo "$PASS_RATE < $THRESHOLD" | bc -l)" = "1" ]; then
              STATUS_EMOJI="[FAIL]"
          fi

          {
            echo "## ${STATUS_EMOJI} Endpoint Validation Report"
            echo "---"
            echo "**Endpoint:** ${{ secrets.TEST_API_ENDPOINT || inputs.endpoint }}"
            echo "**Evaluation Function:** ${{ inputs.eval_function }}"
            echo ""
            echo "| Metric | Value |"
            echo "| :--- | :--- |"
            echo "| **Fetched from DB** | ${TOTAL} |"
            echo "| **Actually Tested** | ${TESTED} |"
            echo "| **Passed** | ${PASSES} |"
            echo "| **Failed** | **${ERRORS}** |"
            echo "| **Parsing Warnings** | ${PARSING_WARNINGS} |"
            echo "| **Feedback Warnings** | ${FEEDBACK_WARNINGS} |"
            echo "| **Pass Rate** | ${PASS_RATE}% |"
            echo "| **Pass Threshold** | ${THRESHOLD}% |"
            echo "| **Seed** | \`${SEED}\` |"
            echo "---"
            
            # Add Firestore link if available
            if [ -n "$FIRESTORE_LINK" ]; then
              echo ""
              echo "### Detailed Results"
              echo ""
              echo "**[View full results in Firestore](${FIRESTORE_LINK})**"
              echo ""
              echo "<details>"
              echo "<summary>Firestore Document Details</summary>"
              echo ""
              echo "- **Document ID:** \`${FIRESTORE_DOC_ID}\`"
              echo "- **Collection:** \`test-results\`"
              echo "- **Project:** ${{ secrets.GCP_PROJECT_ID }}"
              echo ""
              echo "</details>"
            fi
            
            echo "---"
          } >> "$GITHUB_STEP_SUMMARY"