diff --git a/.github/workflows/deploy-foundry.yml b/.github/workflows/deploy-foundry.yml index e6ef213..642d595 100644 --- a/.github/workflows/deploy-foundry.yml +++ b/.github/workflows/deploy-foundry.yml @@ -143,14 +143,14 @@ jobs: echo "version=$version" >>"$GITHUB_OUTPUT" - name: Smoke-test responses id: responses - timeout-minutes: 3 + timeout-minutes: 6 env: AGENT_VERSION: ${{ steps.status.outputs.version }} run: | ulimit -f 8192 azd ai agent invoke IssueLens "$AGENT_TEST_PROMPT" --environment "$AZD_ENV_NAME" \ --protocol responses --version "$AGENT_VERSION" --new-session --new-conversation \ - --timeout 120 --output raw --no-prompt >"$RUNNER_TEMP/issuelens-responses.txt" 2>"$RUNNER_TEMP/issuelens-smoke-errors.txt" || + --timeout 300 --output raw --no-prompt >"$RUNNER_TEMP/issuelens-responses.txt" 2>"$RUNNER_TEMP/issuelens-smoke-errors.txt" || { echo "::error::Responses invocation failed; inspect Foundry before retrying."; exit 1; } sed -n 's/^data: *//p' "$RUNNER_TEMP/issuelens-responses.txt" | grep -v '^\[DONE\]' | jq -e -s --arg expected "$EXPECTED_REPLY" ' @@ -165,7 +165,7 @@ jobs: { echo "::error::Responses did not complete with the expected smoke reply."; exit 1; } - name: Smoke-test invocations id: invocations - timeout-minutes: 3 + timeout-minutes: 6 env: AGENT_VERSION: ${{ steps.status.outputs.version }} run: | @@ -173,7 +173,7 @@ jobs: jq -n --arg input "$AGENT_TEST_PROMPT" '{input: $input}' >"$RUNNER_TEMP/issuelens-payload.json" azd ai agent invoke IssueLens --environment "$AZD_ENV_NAME" --protocol invocations \ --input-file "$RUNNER_TEMP/issuelens-payload.json" --version "$AGENT_VERSION" --new-session \ - --timeout 120 --output raw --no-prompt >"$RUNNER_TEMP/issuelens-invocations.txt" 2>"$RUNNER_TEMP/issuelens-smoke-errors.txt" || + --timeout 300 --output raw --no-prompt >"$RUNNER_TEMP/issuelens-invocations.txt" 2>"$RUNNER_TEMP/issuelens-smoke-errors.txt" || { echo "::error::Invocations request failed; inspect Foundry before retrying."; exit 1; } sed -n 's/^data: *//p' "$RUNNER_TEMP/issuelens-invocations.txt" | jq -e -s --arg expected "$EXPECTED_REPLY" ' diff --git a/docs/guide.md b/docs/guide.md index 5adf8f2..4d20242 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -1162,9 +1162,13 @@ or null. Its available-tools inventory is not tool execution; tool requests, tool events, and all other sub-agent events still fail the smoke check. The job is limited to 40 minutes, the native deployment wait to 20 minutes, and -each invocation to 120 seconds within a 3-minute step. The summary records the -commit, logical GitHub environment, version, readiness, and protocol outcomes, -not Azure identifiers or endpoints. Azure CLI account output is disabled; +each invocation to 300 seconds within a 6-minute step. This gives the SDK's +existing model retries time to complete after transient Foundry capacity or +throttling errors, with a minute of step headroom for CLI setup and response +validation. It does not add workflow-level retries or guarantee model capacity; +timeouts, invocation failures, and invalid replies still fail the job. +The summary records the commit, logical GitHub environment, version, readiness, +and protocol outcomes, not Azure identifiers or endpoints. Azure CLI account output is disabled; azd configuration/deployment output, readiness details, and raw agent responses stay in runner-local files, including on failure. The workflow never prints `azd env get-values` or uploads these files. Temporary files and `.azure` are diff --git a/tests/test_deploy_foundry_workflow.py b/tests/test_deploy_foundry_workflow.py index 2d068cf..f346ad0 100644 --- a/tests/test_deploy_foundry_workflow.py +++ b/tests/test_deploy_foundry_workflow.py @@ -34,6 +34,7 @@ def setUpClass(cls): cls.gate = cls.steps[1] cls.extensions = next(step for step in cls.steps if step["name"] == "Install and verify the Foundry extension") + cls.responses = next(step for step in cls.steps if step.get("id") == "responses") cls.invocations = next(step for step in cls.steps if step.get("id") == "invocations") def run_workflow_step(self, step, stub, env): @@ -107,6 +108,71 @@ def run_extension_setup(self, inventory, *, install_exit=0, list_exit=0): "LIST_EXIT": str(list_exit), }) + def responses_sse(self, reply="ISSUELENS_DEPLOYMENT_OK"): + event = { + "type": "response.completed", + "response": { + "status": "completed", "error": None, "incomplete_details": None, + "output": [{ + "type": "message", "role": "assistant", "status": "completed", + "content": [{"type": "output_text", "text": reply}], + }], + }, + } + return f"event: response.completed\ndata: {json.dumps(event)}\n\ndata: [DONE]\n\n" + + def run_responses_smoke(self, stream, *, invocation_exit=0): + stub = """ +azd() { + if [[ "$*" != "ai agent invoke IssueLens $AGENT_TEST_PROMPT --environment $AZD_ENV_NAME --protocol responses --version $AGENT_VERSION --new-session --new-conversation --timeout 300 --output raw --no-prompt" ]]; then + return 99 + fi + printf '%s' "$RESPONSES_SSE" + return "$RESPONSES_EXIT" +} +""" + if os.name == "nt": + # Git Bash cannot set RLIMIT_FSIZE; Linux CI exercises the real limit. + stub += '\nulimit() { [[ "$*" == "-f 8192" ]]; }\n' + with tempfile.TemporaryDirectory() as directory: + return self.run_workflow_step(self.responses, stub, self.workflow["env"] | { + "RUNNER_TEMP": pathlib.Path(directory).as_posix(), + "AGENT_VERSION": "40", + "RESPONSES_SSE": stream, + "RESPONSES_EXIT": str(invocation_exit), + }) + + def test_responses_accept_exact_completed_reply(self): + result = self.run_responses_smoke(self.responses_sse()) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(result.stdout, "") + + def test_responses_reject_invalid_streams_with_extended_timeout(self): + valid = self.responses_sse() + cases = { + "empty": "", + "CLI heading only": "Invoking agent IssueLens\n", + "wrong reply": self.responses_sse("wrong"), + "extra whitespace": self.responses_sse("ISSUELENS_DEPLOYMENT_OK\n"), + "cancelled": valid.replace('"status": "completed"', '"status": "cancelled"'), + "missing completion": "event: response.in_progress\ndata: {}\n\n", + "duplicate completion": valid + valid, + "malformed JSON": valid + "data: not JSON\n\n", + "error after reply": valid + 'data: {"type":"error","message":"synthetic-error"}\n\n', + } + for name, stream in cases.items(): + with self.subTest(case=name): + result = self.run_responses_smoke(stream) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("::error::Responses did not complete", result.stdout) + self.assertNotIn("synthetic-error", result.stdout + result.stderr) + + def test_responses_reject_nonzero_azd_exit_even_with_valid_output(self): + result = self.run_responses_smoke(self.responses_sse(), invocation_exit=9) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("::error::Responses invocation failed", result.stdout) + self.assertNotIn("::error::Responses did not complete", result.stdout) + def invocation_events(self): return [ { @@ -129,7 +195,7 @@ def invocations_sse(self, events): def run_invocations_smoke(self, stream, *, invocation_exit=0): stub = """ azd() { - if [[ "$*" != "ai agent invoke IssueLens --environment $AZD_ENV_NAME --protocol invocations --input-file $RUNNER_TEMP/issuelens-payload.json --version $AGENT_VERSION --new-session --timeout 120 --output raw --no-prompt" ]]; then + if [[ "$*" != "ai agent invoke IssueLens --environment $AZD_ENV_NAME --protocol invocations --input-file $RUNNER_TEMP/issuelens-payload.json --version $AGENT_VERSION --new-session --timeout 300 --output raw --no-prompt" ]]; then return 99 fi jq -e --arg input "$AGENT_TEST_PROMPT" '. == {input: $input}' "$RUNNER_TEMP/issuelens-payload.json" >/dev/null || return 98 @@ -529,14 +595,15 @@ def test_both_protocols_check_the_new_version_and_expected_reply(self): self.assertIn('.version == $version and .status == "active"', status["run"]) for protocol in ("responses", "invocations"): step = next(step for step in self.steps if step.get("id") == protocol) - self.assertEqual(step["timeout-minutes"], "3") + self.assertEqual(step["timeout-minutes"], "6") self.assertEqual(step["env"], {"AGENT_VERSION": "${{ steps.status.outputs.version }}"}) for text in ( f"--protocol {protocol}", '--version "$AGENT_VERSION"', "--new-session", - "--timeout 120", "--output raw", "ulimit -f 8192", + "--timeout 300", "--output raw", "ulimit -f 8192", 'jq -e -s --arg expected "$EXPECTED_REPLY"', "exit 1", ): self.assertIn(text, step["run"]) + self.assertEqual(step["run"].count("azd ai agent invoke"), 1) self.assertIn('--arg input "$AGENT_TEST_PROMPT" \'{input: $input}\'', self.commands) self.assertIn("--new-conversation", self.commands) self.assertIn('select(.type == "response.completed")', self.commands) @@ -583,6 +650,8 @@ def test_setup_guide_documents_the_official_guide_and_all_configuration(self): "1.34.2", "1.0.0-beta.2", "No automatic retry or rollback", "No live deployment", "installedVersion", "--no-dependencies", "Bash and jq", + "each invocation to 300 seconds within a 6-minute step", + "does not add workflow-level retries", ): self.assertIn(text, guide) for name, version in FOUNDRY_EXTENSIONS.items():