diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index e5539a94c..1057b3ac6 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -39,16 +39,30 @@ RUN apt-get update \ # Tool versions - centralized for easy updates. # This devcontainer currently supports amd64 tool downloads only. +# Finding the latest versions: +# KUBECTL_VERSION -> https://github.com/kubernetes/kubernetes/releases +# KUSTOMIZE_VERSION -> https://github.com/kubernetes-sigs/kustomize/releases +# KUBEBUILDER_VERSION -> https://github.com/kubernetes-sigs/kubebuilder +# GOLANGCI_LINT_VERSION -> https://github.com/golangci/golangci-lint/releases +# K3D_VERSION -> https://github.com/k3d-io/k3d +# FLUX_VERSION -> https://github.com/fluxcd/flux2 +# FLUX_OPERATOR_VERSION -> https://github.com/controlplaneio-fluxcd/flux-operator +# TASK_VERSION -> https://github.com/go-task/task/releases +# TILT_VERSION -> https://github.com/tilt-dev/tilt/releases +# ACTIONLINT_VERSION -> https://github.com/rhysd/actionlint/releases + ENV PATH="/go/bin:/usr/local/go/bin:${PATH}" \ KUBECTL_VERSION=v1.36.1 \ KUSTOMIZE_VERSION=5.8.1 \ - KUBEBUILDER_VERSION=4.14.0 \ + KUBEBUILDER_VERSION=4.14.1 \ GOLANGCI_LINT_VERSION=v2.12.2 \ HELM_VERSION=v4.2.0 \ K3D_VERSION=v5.8.3 \ - FLUX_VERSION=2.8.6 \ - FLUX_OPERATOR_VERSION=0.49.0 \ - TASK_VERSION=v3.50.0 + FLUX_VERSION=2.8.8 \ + FLUX_OPERATOR_VERSION=0.50.0 \ + TASK_VERSION=v3.51.1 \ + TILT_VERSION=v0.37.3 \ + ACTIONLINT_VERSION=1.7.12 # https://github.com/fluxcd/flux2/releases # https://fluxoperator.dev/ @@ -107,6 +121,18 @@ RUN curl -fsSL "https://github.com/go-task/task/releases/download/${TASK_VERSION | tar -xzO task > /usr/local/bin/task \ && chmod +x /usr/local/bin/task +# Install actionlint (static linter for GitHub Actions workflows) +RUN asset="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" \ + && base="https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}" \ + && tmpdir="$(mktemp -d)" \ + && curl -fsSL "${base}/${asset}" -o "${tmpdir}/${asset}" \ + && curl -fsSL "${base}/actionlint_${ACTIONLINT_VERSION}_checksums.txt" -o "${tmpdir}/checksums.txt" \ + && cd "${tmpdir}" \ + && grep " ${asset}$" checksums.txt | sha256sum -c - \ + && tar -xzf "${asset}" actionlint \ + && install -m 0755 actionlint /usr/local/bin/actionlint \ + && rm -rf "${tmpdir}" + # Set working directory WORKDIR /workspaces @@ -127,7 +153,8 @@ RUN chgrp -R godev /go && \ RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.19.0 \ && go install github.com/onsi/ginkgo/v2/ginkgo@v2.28.1 \ && go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20260318145839-6c9615a2a166 \ - && go install golang.org/x/tools/cmd/goimports@v0.43.0 + && go install golang.org/x/tools/cmd/goimports@v0.43.0 \ + && go install github.com/boyter/scc/v3@v3.7.0 # Initialize golangci-lint cache by running it once on an empty directory # This downloads linter dependencies without needing source code @@ -192,7 +219,6 @@ RUN printf '%s\n' \ >> /etc/bash.bashrc # Install Tilt CLI (local dev loop orchestration; dev stage only) -ENV TILT_VERSION=v0.37.1 RUN arch="$(dpkg --print-architecture)" && \ case "${arch}" in \ amd64) tilt_arch="x86_64" ;; \ diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json index 9d81b9292..e2a33b7be 100644 --- a/.devcontainer/devcontainer-lock.json +++ b/.devcontainer/devcontainer-lock.json @@ -1,14 +1,14 @@ { "features": { - "ghcr.io/devcontainers/features/common-utils:2.5.8": { - "version": "2.5.8", - "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:c42fdefe6d737a3a6f61cc52b23c7c9a565d08cc4d9c303669a7cf2ee5fd81fc", - "integrity": "sha256:c42fdefe6d737a3a6f61cc52b23c7c9a565d08cc4d9c303669a7cf2ee5fd81fc" + "ghcr.io/devcontainers/features/common-utils:2.5.9": { + "version": "2.5.9", + "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a", + "integrity": "sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a" }, - "ghcr.io/devcontainers/features/docker-outside-of-docker:1": { - "version": "1.9.1", - "resolved": "ghcr.io/devcontainers/features/docker-outside-of-docker@sha256:dc89605f01ff2f24252c61f7c8ba2a58ccdbc14f2ebf87a7952d9e2b89834850", - "integrity": "sha256:dc89605f01ff2f24252c61f7c8ba2a58ccdbc14f2ebf87a7952d9e2b89834850" + "ghcr.io/devcontainers/features/docker-outside-of-docker:1.10.0": { + "version": "1.10.0", + "resolved": "ghcr.io/devcontainers/features/docker-outside-of-docker@sha256:c2c2cf829505ead8e4892c88c31b6594ae94a2bbb209e16e1fac456c1a3a624e", + "integrity": "sha256:c2c2cf829505ead8e4892c88c31b6594ae94a2bbb209e16e1fac456c1a3a624e" }, "ghcr.io/devcontainers/features/git:1": { "version": "1.3.5", diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f70523b9b..bfb22bbd9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -8,8 +8,8 @@ "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/${localWorkspaceFolderBasename},type=bind", "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", "features": { - "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}, - "ghcr.io/devcontainers/features/common-utils:2.5.8": { + "ghcr.io/devcontainers/features/docker-outside-of-docker:1.10.0": {}, + "ghcr.io/devcontainers/features/common-utils:2.5.9": { "userUid": "automatic", "userGid": "automatic", "username": "vscode" @@ -25,7 +25,8 @@ 19090, 10350, 16379, - 19080 + 19080, + 19081 ], "portsAttributes": { "13000": { @@ -47,6 +48,10 @@ "19080": { "label": "Flux", "onAutoForward": "notify" + }, + "19081": { + "label": "Allure", + "onAutoForward": "notify" } }, "customizations": { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f2735e45..08b75515d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -273,9 +273,17 @@ jobs: && export INSTALL_MODE=plain-manifests-file && task test-e2e-quickstart-manifest needs_artifact: true + e2e_ginkgo_procs: "2" + k3d_agent_count: "0" - name: full script: "task test-e2e-full" needs_artifact: false + # Bumped 1 -> 4: the full suite was serialized to avoid the + # cross-target rule-change snapshot coupling, now fixed (per-target + # effective watch plan — docs/design/gittarget-isolation-on-rule-change.md). + # Validated locally at --procs=4; trialing the same in CI. + e2e_ginkgo_procs: "4" + k3d_agent_count: "0" env: PROJECT_IMAGE: ${{ needs.docker-build.outputs.image }} CI_CONTAINER: ${{ needs.build-ci-container.outputs.image }} @@ -296,6 +304,51 @@ jobs: run: | echo "${{ secrets.GITHUB_TOKEN }}" | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin + - name: Start disk usage sampler + run: | + # Sample free space on / every 10s for the whole e2e run so we can see + # the peak (not just the post-run value). /var/lib/docker — k3d node + # volumes, pulled images, build cache — lives on this filesystem. + ( while true; do + printf '%s avail=%s\n' "$(date -u +%H:%M:%S)" \ + "$(df --output=avail -BM / | tail -1 | tr -d ' ')" + sleep 10 + done ) > "df-samples-${{ matrix.name }}.log" 2>/dev/null & + echo "SAMPLER_PID=$!" >> "$GITHUB_ENV" + + - name: Bring up cluster + Flux services (explicit, streaming logs) + run: | + # Install-mode-/image-independent slice of prepare-e2e. Runs the + # slow, hang-prone cluster + Flux + services bring-up here so its + # logs stream live, instead of being buffered inside Ginkgo's + # SynchronizedBeforeSuite (where a stall just looks like a silent + # 15m hang ending in a goroutine dump). The in-suite prepare-e2e + # no-ops this part because the stamps already exist. + docker run --rm \ + --network host \ + -v "${GITHUB_WORKSPACE}:${{ env.CI_WORKDIR }}" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -w "${{ env.CI_WORKDIR }}" \ + -e IMAGE_DELIVERY_MODE=${{ env.IMAGE_DELIVERY_MODE }} \ + -e K3D_AGENT_COUNT=${{ matrix.k3d_agent_count }} \ + -e HOST_PROJECT_PATH=${{ github.workspace }} \ + -e KUBECONFIG=${{ env.CI_WORKDIR }}/.stamps/cluster/k3d-gitops-reverser-test-e2e/.kube/config \ + ${{ env.CI_CONTAINER }} \ + bash -c " + git config --global --add safe.directory ${{ env.CI_WORKDIR }} + # The cluster is created here but reused by the test container + # below. start-cluster.sh writes the k3d context into KUBECONFIG; + # point it at a file on the mounted workspace so the next + # container inherits the context (otherwise it is lost with this + # ephemeral container and kubectl reports 'context does not exist'). + # Living under .stamps/cluster//.kube keeps it tied to the + # cluster lifecycle: cleanup-installs.sh (per-namespace) leaves it + # alone, while clean-cluster (rm -rf .stamps/cluster/) clears + # it together with the cluster it points at. + mkdir -p .stamps/cluster/k3d-gitops-reverser-test-e2e/.kube + task _services-ready + " + - name: Run E2E tests in CI container run: | docker run --rm \ @@ -305,13 +358,65 @@ jobs: -w "${{ env.CI_WORKDIR }}" \ -e PROJECT_IMAGE=${{ env.PROJECT_IMAGE }} \ -e IMAGE_DELIVERY_MODE=${{ env.IMAGE_DELIVERY_MODE }} \ + -e E2E_GINKGO_PROCS=${{ matrix.e2e_ginkgo_procs }} \ + -e K3D_AGENT_COUNT=${{ matrix.k3d_agent_count }} \ -e HOST_PROJECT_PATH=${{ github.workspace }} \ + -e KUBECONFIG=${{ env.CI_WORKDIR }}/.stamps/cluster/k3d-gitops-reverser-test-e2e/.kube/config \ ${{ env.CI_CONTAINER }} \ bash -c " git config --global --add safe.directory ${{ env.CI_WORKDIR }} ${{ matrix.script }} " + - name: Report disk usage (peak + final) + if: always() + run: | + kill "${SAMPLER_PID}" 2>/dev/null || true + samples="df-samples-${{ matrix.name }}.log" + { + echo "### Disk usage (${{ matrix.name }})" + echo + echo "Lowest free space on \`/\` during the run:" + echo '```' + sort -t= -k2 -n "${samples}" 2>/dev/null | head -1 || echo "(no samples captured)" + echo '```' + echo "Final state:" + echo '```' + df -h / + echo + docker system df + echo '```' + echo "
Full df sample timeline" + echo + echo '```' + cat "${samples}" 2>/dev/null || echo "(no samples captured)" + echo '```' + echo "
" + } | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Make E2E artifacts readable by the runner + if: always() + # The E2E suite runs in a root container that mounts the workspace, so + # everything it writes under .stamps/ (e.g. e2e-repo-artifacts/) is + # root-owned with restrictive perms. The upload step below runs on the + # host as the non-root runner user and fails with EACCES when its glob + # tries to scandir those dirs. Relax perms so the runner can traverse + # and read them. a+rX adds read for files and traverse for dirs only. + run: sudo chmod -R a+rX .stamps || true + + - name: Upload E2E raw Ginkgo JSON reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: e2e-ginkgo-reports-${{ matrix.name }} + path: .stamps/**/ginkgo-report-*.json + # Reports live under the dot-dir .stamps/, which upload-artifact treats + # as hidden and excludes by default — without this the upload silently + # finds no files. See the "No files were found" gotcha in upload-artifact v4+. + include-hidden-files: true + if-no-files-found: error + retention-days: 60 + # Release job only runs on push to main after tests pass release-please: name: Release Please diff --git a/AGENTS.md b/AGENTS.md index b0322c4f3..773315704 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,10 @@ task test # Must pass all unit tests with >90% coverage task test-e2e # Must pass end-to-end tests ``` +If you change a GitHub Actions workflow, also run `task lint-actions`, which lints +`.github/workflows/ci.yml` with `actionlint`. Both `actionlint` and `golangci-lint` +ship in the devcontainer image. + ## PRE-IMPLEMENTATION BEHAVIOR 1. **Check Docker availability for e2e tests**: Before running `task test-e2e`, verify Docker is running with `docker info` or ask user to start Docker daemon if needed @@ -64,6 +68,12 @@ task test-e2e # Must pass end-to-end tests - Add race condition protection - Use temporary directories for testing +### CI Workflows (`.github/workflows/`) +- After editing any workflow, run `task lint-actions` to catch errors with + `actionlint` before pushing +- A workflow-only change still counts as a CI/config change, so it is **not** + covered by the markdown/docs-only validation exception + ## TESTING REQUIREMENTS - Write unit tests with >90% coverage diff --git a/Taskfile-build.yml b/Taskfile-build.yml index 1957b374e..f00c912ae 100644 --- a/Taskfile-build.yml +++ b/Taskfile-build.yml @@ -8,6 +8,7 @@ vars: CONTROLLER_GEN: '{{.CONTROLLER_GEN | default "controller-gen"}}' ENVTEST: '{{.ENVTEST | default "setup-envtest"}}' GOLANGCI_LINT: '{{.GOLANGCI_LINT | default "golangci-lint"}}' + SCC: '{{.SCC | default "scc"}}' INSTALL_NAME: '{{.INSTALL_NAME | default "gitops-reverser"}}' NAMESPACE: '{{.NAMESPACE | default "gitops-reverser"}}' DEFAULT_AUDIT_REDIS_ADDR: '{{.DEFAULT_AUDIT_REDIS_ADDR | default "valkey:6379"}}' @@ -121,13 +122,18 @@ tasks: cmds: - | export KUBEBUILDER_ASSETS="$({{.ENVTEST}} use {{.ENVTEST_K8S_VERSION}} --bin-dir "$(pwd)/bin" -p path)" - go test $(go list ./... | grep -v /e2e) -coverprofile cover.out + go test $(go list ./... | grep -Ev '/test/e2e$') -coverprofile cover.out lint: desc: Run golangci-lint cmds: - '{{.GOLANGCI_LINT}} run' + lint-actions: + desc: Lint GitHub Actions workflows with actionlint + cmds: + - actionlint .github/workflows/ci.yml + lint-fix: desc: Run golangci-lint and perform fixes cmds: @@ -138,6 +144,11 @@ tasks: cmds: - '{{.GOLANGCI_LINT}} config verify' + loc: + desc: Count lines of project code with scc (excludes vendored external-sources) + cmds: + - '{{.SCC}} --exclude-dir external-sources,vendor,bin,dist {{.CLI_ARGS}}' + build: desc: Build manager binary deps: diff --git a/Taskfile.yml b/Taskfile.yml index 4c73c3036..c8e992a2d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -5,6 +5,11 @@ version: '3' # See .env.example for the available keys. dotenv: ['.env'] +# Default tasks to run once per invocation so the e2e DAG can fan tasks +# out via `deps:` without racing — e.g. `_cluster-ready` reachable through +# both `_image-loaded` and `_flux-installed` must not start twice. +run: once + set: - errexit - nounset diff --git a/Tiltfile b/Tiltfile index dd1fc5594..3a876666f 100644 --- a/Tiltfile +++ b/Tiltfile @@ -91,7 +91,6 @@ test_targets = [ 'test-e2e-full', 'test-e2e-signing', 'test-e2e-manager', - 'test-e2e-audit-redis', 'test-image-refresh', ] for name in test_targets: diff --git a/docs/README.md b/docs/README.md index 4c8bae344..02b429544 100644 --- a/docs/README.md +++ b/docs/README.md @@ -27,8 +27,11 @@ These directories are useful when changing internals, but they are not the prima docs surface: - [`design/`](design/): architecture notes, investigations, and implementation plans +- [`finished/`](finished/): design docs and reports for work that has shipped or been resolved, + kept for historical context - [`ci/`](ci/): CI/devcontainer rationale and troubleshooting - [`future/`](future/): ideas that are intentionally deferred +- [`serious-bug/`](serious-bug/): worked bug reports against upstream projects - [`audit-setup/`](audit-setup/): cluster-specific audit delivery notes and examples These root-level files are also working notes rather than polished user docs: diff --git a/docs/architecture.md b/docs/architecture.md index cbb5e34d9..5a6c76f34 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -884,7 +884,7 @@ For deeper context on specific decisions: - [GitTarget lifecycle and repo architecture](design/gittarget-lifecycle-and-repo-architecture.md) — GitTarget state machine details - [Watch and audit rule matching improvement](design/watch-audit-rule-matching-improvement.md) — known issues with namespace matching and startup bootstrap - [Kubernetes API resource catalog](design/kubernetes-api-resource-catalog.md) — the served-resource discovery model behind `APIResourceCatalog` -- [WatchRule GVR resolution](design/watchrule-gvr-resolution-plan.md) — how bare rule resources resolve to concrete GVRs, and the snapshot trust model +- [WatchRule GVR resolution](finished/watchrule-gvr-resolution-plan.md) — how bare rule resources resolve to concrete GVRs, and the snapshot trust model - [Multi-cluster audit ingestion implications](design/multi-cluster-audit-ingestion-implications.md) — what multi-cluster means beyond ingestion - [SOPS/age key management](design/sops-repo-bootstrap-and-key-management-architecture.md) — encryption architecture - [Audit webhook TLS design](design/audit-webhook-tls-design.md) — webhook transport security diff --git a/docs/ci/e2e-allure-reporting.md b/docs/ci/e2e-allure-reporting.md new file mode 100644 index 000000000..532d54da2 --- /dev/null +++ b/docs/ci/e2e-allure-reporting.md @@ -0,0 +1,90 @@ +# E2E Allure Reporting + +The CI e2e job uploads raw Ginkgo JSON reports for each matrix entry. Developers +can download those artifacts and render a visual Allure HTML report locally when +the timeline view is useful. + +The e2e tasks still run through the repository Taskfile so local and CI execution +stay aligned. Each task writes Ginkgo JSON reports under +`.stamps/cluster///ginkgo-report-.json`. After the e2e +run, CI uploads those files, plus the timing summary, as the +`e2e-ginkgo-reports-` artifact. + +The raw Ginkgo JSON files are the source of truth for local tooling and timing +analysis. To render an Allure report from downloaded or locally produced Ginkgo +JSON files, use: + +```bash +go run ./test/e2e/tools/ginkgo-allure --output-dir allure-results ... +``` + +During an active e2e run, GitHub Actions logs remain the live view. After the run +finishes, the downloaded Ginkgo JSON can be converted locally into an Allure +report with a visual timeline, status breakdown, labels, durations, and captured +Ginkgo output. + +The converter is deliberately small and one-way: + +- Ginkgo `passed` maps to Allure `passed`. +- Ginkgo `failed` maps to Allure `failed`. +- Ginkgo `pending` and `skipped` map to Allure `skipped`. +- Ginkgo timeout, interrupt, abort, and panic states map to Allure `broken`. +- Captured `GinkgoWriter` and parallel stdout/stderr output are attached as + plain-text test artifacts. + +## Allure labels + +The converter attaches labels so the report's grouping and timeline views are +useful for a parallel suite: + +- `thread` — `ginkgo-process-`, the Ginkgo parallel process that ran the + spec. This is what gives the Allure timeline its per-process lanes, so you can + see how specs were distributed across processes and spot a single overloaded + lane. Only emitted for parallel runs (`ParallelProcess > 0`). +- `suite` — the Ginkgo suite description. +- `package` — the base name of the suite's source path. +- `parentSuite` / `subSuite` — the spec's container hierarchy (outermost + `Describe` as `parentSuite`, the remaining nested containers as `subSuite`). +- `tag` — one per Ginkgo `Label(...)`, plus a `Serial` tag for specs that ran + serially. + +## Local devcontainer use + +The devcontainer does not install Java or the Allure CLI. Local report rendering +uses Docker to run Allure in an on-demand Java runtime container, keeping the +devcontainer image smaller. + +Run any e2e task that writes a Ginkgo JSON report, for example: + +```bash +task test-e2e +``` + +Then generate a local Allure HTML report: + +```bash +task allure-e2e-report +``` + +The report is written to `.stamps/allure-report`. To serve it from inside the +devcontainer on the forwarded Allure port: + +```bash +task allure-e2e-open +``` + +This serves the report on port `19081`. By default, the local Allure task reads +the newest Ginkgo JSON report from the current e2e context and namespace: +`.stamps/cluster///ginkgo-report-*.json`. + +To render a different set of reports, set `ALLURE_GINKGO_REPORTS_DIR` to another +directory or set `ALLURE_GINKGO_REPORTS` to an explicit whitespace-separated +list of report files. To render every report in the selected directory, set +`ALLURE_INCLUDE_ALL_GINKGO_REPORTS=true`. + +The same tasks can reuse an older local Ginkgo JSON report in that directory, so +you can generate the report after a failed run without rerunning the suite. + +The first local render pulls the `eclipse-temurin:17-jre` image and downloads the +Allure CLI into `.stamps/allure-cli`. Later renders reuse both Docker's image +cache and the local Allure CLI cache. diff --git a/docs/ci/findings.md b/docs/ci/findings.md index 6ae8f7458..93d571330 100644 --- a/docs/ci/findings.md +++ b/docs/ci/findings.md @@ -1,10 +1,10 @@ ## CI/Devcontainer Findings (Current Baseline) -Last updated: 2026-02-13 +Last updated: 2026-06-02 Recent investigation: [E2E full-suite shared-state flake](../design/e2e-full-suite-shared-state-investigation.md) -This folder documents why the repository uses its current devcontainer and CI behavior, especially around Go caches, workspace paths, and Kind access from inside the container. +This folder documents why the repository uses its current devcontainer and CI behavior, especially around Go caches, workspace paths, and k3d cluster access from inside the container. ### 1) Workspace path model @@ -46,7 +46,7 @@ Why: - Stable module/build caching independent of repo bind mount - Fewer permission regressions than putting caches in the workspace tree -### 4) Kind + kubectl access model inside devcontainer +### 4) k3d + kubectl access model inside devcontainer The current working model is: @@ -54,27 +54,25 @@ The current working model is: - Devcontainer run args include: - `--group-add=docker` - `--add-host=host.docker.internal:host-gateway` -- Kind cluster config sets: - - `networking.apiServerAddress: "0.0.0.0"` -- `test/e2e/cluster/start-cluster.sh` rewrites kubeconfig server endpoints from - `127.0.0.1|localhost|0.0.0.0` to `host.docker.internal:` and sets - `tls-server-name=localhost` +- `test/e2e/cluster/start-cluster.sh` lets k3d pick the API server port, then + rewrites kubeconfig server endpoints from `127.0.0.1|localhost|0.0.0.0` to + `host.docker.internal:` and sets `tls-server-name=localhost` Why this is required: -- If Docker publishes Kind API server on host loopback only (`127.0.0.1`), it is not reachable via `host.docker.internal` from the container. -- Binding on `0.0.0.0` plus kubeconfig rewrite makes in-container `kubectl` stable without host networking. +- If Docker publishes the k3d API server on host loopback only (`127.0.0.1`), it is not reachable via `host.docker.internal` from the container. +- The kubeconfig rewrite to `host.docker.internal` makes in-container `kubectl` stable without host networking. ### 5) Practical verification checklist After devcontainer rebuild/reopen: ```bash -# 1) Kind setup -make setup-cluster +# 1) Cluster setup (creates the k3d cluster via the e2e Taskfile) +bash test/e2e/cluster/start-cluster.sh # 2) Confirm API publish bind (expected 0.0.0.0 or ::) -docker inspect gitops-reverser-test-e2e-control-plane --format '{{json .NetworkSettings.Ports}}' +docker inspect k3d-gitops-reverser-test-e2e-server-0 --format '{{json .NetworkSettings.Ports}}' # 3) Confirm kubeconfig server rewrite kubectl config view --minify | sed -n '/server:/p;/tls-server-name:/p' @@ -89,3 +87,4 @@ kubectl get nodes - `windows-devcontainer.md` - Windows-specific mount behavior and expected differences - `ci-root-user.md` - why CI containers run as root - `git-safe-dir-explained.md` - why `safe.directory` is required in containerized CI +- `e2e-allure-reporting.md` - how CI renders visual Allure reports from Ginkgo JSON diff --git a/docs/design/e2e-serial-registry.md b/docs/design/e2e-serial-registry.md new file mode 100644 index 000000000..85f6106ba --- /dev/null +++ b/docs/design/e2e-serial-registry.md @@ -0,0 +1,105 @@ +# E2E Serial Registry + +This is a **living document**. It is the authoritative list of e2e test +containers that must run `Serial` (i.e. never concurrently with any other spec) +under bounded Ginkgo parallelism (Phase 2.5 of +[e2e-speedup-plan.md](e2e-speedup-plan.md)). + +> De-serializing any row below is an isolation refactor, not a label flip. +> `crd_lifecycle`, the two audit-consumer specs, `bi_directional`, and +> `aggregated_apiserver` have all been de-serialized (see the "De-serialized" +> section) — only the two singleton-controller specs remain. + +## Rule + +A test container is `Serial` only when it mutates shared, **process-level or +cluster-wide** state that cannot be isolated by name. Prefer name isolation +(e.g. per-file CRD groups — see [test/e2e/icecream.go](../../test/e2e/icecream.go)) +over `Serial`; reserve `Serial` for conflicts name isolation cannot fix. + +**When you add a spec that performs a cluster-scoped write (rolls/scales the +controller, installs an `APIService`/CRD/cluster RBAC, or otherwise affects +shared apiserver state), update this registry in the same PR** — either by +isolating the state by name or by marking the container `Serial` and adding a +row below. + +## Serial containers + +| File | Container | Shared state it touches | Why name isolation can't fix it | +|---|---|---|---| +| [test/e2e/restart_snapshot_e2e_test.go](../../test/e2e/restart_snapshot_e2e_test.go) | `Restart Snapshot Safety` | Rollout-restarts the controller deployment. | The controller is a singleton; restarting it disrupts in-flight reconciles/commits for every other spec. | +| [test/e2e/image_refresh_test.go](../../test/e2e/image_refresh_test.go) | `image refresh dependency chain` | Changes the controller image / redeploys the controller. | Same singleton controller; an image swap perturbs all concurrent specs. | + +### De-serialized + +`Manager CRD Lifecycle` ([crd_lifecycle_e2e_test.go](../../test/e2e/crd_lifecycle_e2e_test.go)) +**was** `Serial` because installing/deleting its CRD changes cluster-wide +discovery and historically forced unrelated GitTargets to resnapshot (leaking +`reconcile: sync …` commits into other specs' exact `[CREATE]`/`[DELETE]` +assertions). It now runs parallel (`Ordered`, not `Serial`) because: + +- Finding #2's fix ([manager.go:1191](../../internal/watch/manager.go#L1191)) + only resnapshots a target when its *resolved* plan hash changes, so a catalog + refresh no longer perturbs targets that do not match the new GVR. +- Per-file CRD groups ([icecream.go:30](../../test/e2e/icecream.go#L30)) keep the + `IceCreamOrder` CRD name-isolated, and the only other wildcard-ish + `ClusterWatchRule` (`restart_snapshot`) is itself `Serial`, so no concurrent + spec has a rule matching the icecream group. + +The former `Audit Redis Queue` / `Audit Redis Consumer` containers were retired: +the producer-path and basic consumer-commit assertions duplicated coverage in +the WatchRule suite, so they were dropped. The one genuinely unique assertion — +that OIDC name/email claims in `user.extra` end up as the Git commit author — +moved to `Commit Author Attribution` +([commit_author_attribution_e2e_test.go](../../test/e2e/commit_author_attribution_e2e_test.go)), +which runs **parallel** (not `Serial`): its dedicated GitProvider uses a 0s +commit window so each event is its own commit, and it reads the author scoped to +each ConfigMap's unique path, so concurrent audit traffic cannot change the +commit it asserts on. + +The two `audit-consumer`-labelled containers (`Commit Window Batching` +([commit_window_batching_e2e_test.go](../../test/e2e/commit_window_batching_e2e_test.go)), +`Commit Request` ([commit_request_e2e_test.go](../../test/e2e/commit_request_e2e_test.go))) +were also de-serialized. Their real serial cause was **not** the shared audit +*pipeline* — namespace-scoped WatchRules already route each spec's events only to +its own GitTarget — but a shared *repo*: both borrowed one Gitea repo from a +`sync.Once` helper and asserted on whole-branch state (`Commit Request` reads +`HEAD`/`status.sha`; `Commit Window Batching` reads `git rev-list --count main`), +so the *other* audit spec's commits to the same `main` corrupted those reads. +Each now provisions its **own** repo via `SetupRepo`, so the only writer to its +`main` is its own GitTarget, fed exclusively by audit events from its own +namespace. The batching property is a per-GitTarget branch-worker behaviour (each +target runs its own commit window), so events for other GitTargets cannot enter +this spec's grouped commit. The retired `sync.Once` helper (`audit_helpers_test.go`) +was deleted. + +`Bi Directional` ([bi_directional_e2e_test.go](../../test/e2e/bi_directional_e2e_test.go)) +was de-serialized too. It already owned a dedicated repo and per-file CRD group; +its historical "+2 commits only under parallelism" came from cluster-wide GVR +catalog churn (another spec installing/deleting a CRD) dragging unrelated targets +into a resnapshot whose `reconcile: sync …` commits inflated its exact count. +Finding #2's fix ([manager.go:1191](../../internal/watch/manager.go#L1191)) — +a target only resnapshots when its *resolved* plan hash changes — removed that +mechanism, the same fix that unblocked `crd_lifecycle`. + +`Aggregated API server` ([aggregated_apiserver_e2e_test.go](../../test/e2e/aggregated_apiserver_e2e_test.go)) +was de-serialized after its `Serial` justification proved **stale**. The note +claimed it "installs/removes a cluster-scoped `APIService` while it is in flight", +but the wardle APIService (`v1alpha1.wardle.example.com`) is installed once at +cluster setup by the apiservice-audit-proxy HelmRelease +([setup/flux/releases/aggregated-api.yaml](../../test/e2e/setup/flux/releases/aggregated-api.yaml)) +and stays Available for the whole suite. The spec only *reads* it — it never +registers or removes it — so there is no in-flight discovery perturbation to +serialize against. Everything the spec mutates is name-isolated (own namespace, +own repo, namespace-scoped flunder WatchRule, namespaced Flunders), and its one +cluster-wide assertion (`gitopsreverser_audit_shallow_dropped_total == 0`) is a +global zero-invariant that is parallel-safe by design. + +## Triage list (watch during stability runs) + +`controller_basics_e2e_test.go` spec #5 ("should receive audit webhook events") +reads a **global Prometheus audit counter** with a baseline/delta pattern. It +remained parallel and passed, because the delta only needs the counter to +*increase*. If a future change makes that read exact, concurrent audit traffic +from other parallel specs could break it — harden the read (filter by +namespace/resource) or move the container into the Serial table above. diff --git a/docs/design/e2e-speedup-plan.md b/docs/design/e2e-speedup-plan.md new file mode 100644 index 000000000..ac2630f2a --- /dev/null +++ b/docs/design/e2e-speedup-plan.md @@ -0,0 +1,176 @@ +# E2E Speedup Plan + +Make `task test-e2e*` faster for the three audiences that pay the cost: local +devs, AI agents, and CI. Phases 0–3 below have **shipped**; this doc keeps the +baseline, the outcomes, and the rationale for what was deliberately *not* done. + +The authoritative list of which specs still run `Serial` (and why) is the living +[e2e-serial-registry.md](e2e-serial-registry.md). + +## Non-goals + +- Rewriting the e2e harness from scratch. +- Reducing the number of e2e specs (no spec was merged or deleted; the Phase 2 + split preserved spec count). + +## Baseline (2026-05-28, warm devcontainer, k3d reused) + +| | | +|---|---| +| Warm `task prepare-e2e` | **1.4 s** (everything stamp-cached) | +| Smoke suite (`task test-e2e`, 20 of 46 specs) | **418.7 s** (~7 min) | + +### Per-spec ranking (smoke filter, warm) + +``` + Duration % Spec + 128.4 s 30.8 Restart Snapshot Safety keeps the git mirror intact when the controller restarts + 40.9 s 9.8 Commit Window Batching collapses a burst of events into one grouped commit and one push + 34.1 s 8.2 Aggregated API server should install and serve flunders through the aggregation layer + 28.9 s 6.9 Manager should create Git commit when IceCreamOrder CRD is installed via ClusterWatchRule + 25.7 s 6.2 Commit Signing should produce per-event commits verifiable locally and by Gitea + 22.3 s 5.3 Commit Request finalizes a CommitRequest created with metadata.generateName + 21.9 s 5.2 Commit Request finalizes the open commit window on demand and reports the resulting SHA + 15.5 s 3.7 Manager should delete Git file when ConfigMap is deleted via WatchRule + 9.9 s 2.4 Manager should create Git commit when ConfigMap is added via WatchRule + 9.1 s 2.2 Manager should commit encrypted Secret manifests when WatchRule includes secrets + 4.4 s 1.1 Manager should backfill pre-existing ConfigMap when WatchRule is added afterwards + 4.1 s 1.0 Manager should receive audit webhook events from kube-apiserver + 2.1 s 0.5 Manager should handle a normal and healthy GitProvider + 1.9 s 0.5 Manager should run successfully + 1.7 s 0.4 Manager should validate GitProvider with real Gitea repository + 0.1 s 0.0 Manager should ensure the metrics endpoint is serving metrics + 0.1 s 0.0 Manager should expose the controller service +``` + +**Findings that shaped the plan:** the top 5 specs are ~62 % of smoke wallclock; +`Restart Snapshot Safety` alone is 31 %, of which ~75 s was a passive +negative-assertion wait; namespace teardown is a consistent 20–25 s per spec +(finalizer waits); per-spec setup is ~5–10 s. The structural blockers to +parallelism were the shared `Manager` Describe and cluster-scoped resources. + +## What shipped + +### Phase 0 — Visibility + +Added `-ginkgo.json-report` to the e2e `go test` invocations plus a small parser +(`test/e2e/tools/spec-timings/`) that prints the duration-sorted table above from +structured data. No custom reporter needed. + +### Phase 1 — `prepare-e2e` as a real `task` DAG + +Replaced the flat `cmds:` chains in `prepare-e2e` with `deps:` so independent +stamps (`_age-key`/`_sops-secret-yaml`, `_image-loaded`, `_install-cleanup`, +`manifests`) run concurrently. The critical path is the strictly-serial Flux +ramp-up (`_cluster-ready → _flux-installed → _flux-setup-ready → +_services-ready → install → _controller-deployed → _webhook-tls-ready → +_aggregated-api-ready`); everything else folds onto it. + +Two ordering constraints had to stay sequential: `portforward-ensure` runs +*after* `_webhook-tls-ready` (the k3d server-node restart inside it kills +concurrent port-forwards), and `_install-cleanup` is its own node so +`_sops-secret-yaml` can depend on it rather than racing a cleanup that wipes its +output. + +**Measured:** cold `task prepare-e2e` 230 s → 213 s (Flux ramp dominates a cached +image build; with a fresh Go build the parallel branches save more). Warm stays +~1 s. + +### Phase 2 — Mechanical split of the `Manager` Describe + +Split the 21-spec `Manager` Describe into four files, each its own top-level +Describe with per-file fixtures, replacing the package-global `managerRepo` with +per-Describe locals: + +- `controller_basics_e2e_test.go` (no Gitea repo) +- `gitprovider_validation_e2e_test.go` +- `watchrule_configmap_secret_e2e_test.go` +- `crd_lifecycle_e2e_test.go` (owns its own per-file CRD) + +Spec count and observable behavior preserved. This phase alone is *slower* +sequentially (more namespace teardowns); it's the prerequisite for Phase 2.5. + +### Phase 2.5 — Bounded Ginkgo parallelism + Serial registry + +Parallelism is driven by the `ginkgo` CLI (`--procs=$E2E_GINKGO_PROCS`, default +`2`), not `go test`. `SynchronizedBeforeSuite` runs `prepare-e2e` + CRD +pre-cleanup once on process #1. Per-file CRD-group isolation +([test/e2e/icecream.go](../../test/e2e/icecream.go)) gives `crd_lifecycle`, +`restart_snapshot`, and `bi_directional` their own API groups so they never +collide by name. The [e2e-serial-registry.md](e2e-serial-registry.md) captures +the specs that still need `Serial`. + +**Measured (local, procs=2):** smoke **352 s** vs 418.7 s sequential (~16 %). +**CI runs the `full` leg at `E2E_GINKGO_PROCS=1`** — on stock `ubuntu-latest` +runners the singleton controller is CPU-starved under two concurrent streams plus +its own signing/git work, so a different spec timed out each run. Local/default +stays `procs=2`. + +### Phase 3 — Drain-signal metrics for `Restart Snapshot Safety` + +Replaced the ~75 s blind negative-assertion wait with two metrics, both gated on +the *new* controller pod (`pod=""`) so a rollout's brief two-series overlap +can't produce a false pass: + +- `gitopsreverser_target_reconcile_completed_total{gittarget_namespace,gittarget_name,trigger}` + — a **counter** (not a latched gauge): on a fresh pod it starts at 0, so + `> 0` proves *that pod* completed its own post-restart snapshot reconcile. +- `gitopsreverser_branch_worker_queue_depth{provider_namespace,provider_name,branch}` + — a gauge that reaches 0 only once every accepted item is fully handled and a + successful push has cleared `pendingWrites` (depth counts in-flight items via + an `atomic.Int64`, not `len(eventQueue)`, to avoid a mid-commit false drain). + +Metrics live in [internal/telemetry/exporter.go](../../internal/telemetry/exporter.go). +Label keys are prefixed (`gittarget_namespace`, not bare `namespace`) because the +pod scrape with `honor_labels: false` would otherwise overwrite a bare +`namespace` and move it to `exported_namespace`. + +**Measured:** restart spec ~128 s → ~67 s. Both gates resolve on the first poll +after the rollout settles; the residual time is fixed setup (Prometheus client, +Gitea repo, CRD install, building the mirror, the rollout itself) plus the 15 s +stability window — not the drain wait, which is now ~0 s. + +## Considered and skipped + +Recorded so revisiting is cheap. + +1. **Short-circuit `BeforeSuite` to skip warm `prepare-e2e`.** Re-implementing + the stamp freshness check in Go duplicates logic `task` already owns and gives + two sources of truth. The ~1.4 s warm cost is small; Phase 1's DAG is the + better lever. *Revisit if* warm prepare grows back past a few seconds. + +2. **CI matrix sharding by Ginkgo label.** GitHub Actions throttles this repo at + 3+ concurrent runners, so shards queue instead of running in parallel — + defeating the wallclock benefit while burning more CPU minutes. *Revisit if* we + get real concurrent runner capacity, a different CI provider, or self-hosted + runners. + +3. **CI cluster reuse (self-hosted runner or Tekton).** Both require durable + operational ownership (a VM/cluster to secure, upgrade, monitor, pay for) plus + multi-tenant isolation between concurrent pipelines. Phases 1–3 attack the + cheaper per-run cost first. *Revisit if* CI wallclock is still dominated by + cluster boot after Phases 1–3 *and* the team will take on the ops burden. + +4. **Removing `Ordered` *within* the new Manager files.** Phase 2 already lets the + four files run on four workers; intra-file parallelism only helps a file with + many slow specs, which the post-split layout doesn't have, and adds real + ordering-correctness risk. *Revisit if* one file grows a slow tail of + independent specs. + +5. **Tekton / Shipwright "cloud-native CI".** Iteration speed is dominated by + cluster boot and test execution, not the build step; pushing images through a + registry every build makes per-iteration cost go *up*. The one legitimate + angle (warm infra on a persistent cluster) is item 3 with extra moving parts. + *Revisit if* we want pipeline-as-CRD for a separate reason (preview + environments, on-demand test infra). + +## Reproducing the baseline + +```bash +task prepare-e2e +mkdir -p /tmp/e2e-baseline +go test -timeout 15m ./test/e2e/ \ + -ginkgo.v -ginkgo.label-filter=smoke \ + -ginkgo.json-report=/tmp/e2e-baseline/smoke.json +go run ./test/e2e/tools/spec-timings /tmp/e2e-baseline/smoke.json +``` diff --git a/docs/design/e2e-test-design.md b/docs/design/e2e-test-design.md index a68238354..23850c2f5 100644 --- a/docs/design/e2e-test-design.md +++ b/docs/design/e2e-test-design.md @@ -108,7 +108,6 @@ These are the canonical Task entry points on the current worktree: | `task test-e2e-full` | Entire e2e package | | `task test-e2e-manager` | Manager-focused scenarios | | `task test-e2e-signing` | Commit-signing scenarios | -| `task test-e2e-audit-redis` | Audit webhook to Redis scenarios | | `task test-image-refresh` | Image rebuild / reload invalidation chain | | `task test-e2e-quickstart-helm` | Quickstart framework with Helm install | | `task test-e2e-quickstart-manifest` | Quickstart framework with manifest install | @@ -120,7 +119,7 @@ installing the system under test. The intended interpretation of these entry points is: -- `task test-e2e`, `task test-e2e-manager`, `task test-e2e-signing`, `task test-e2e-audit-redis`, and `task test-image-refresh` are standard controller behavior checks and should normally run on the default `config-dir` install +- `task test-e2e`, `task test-e2e-manager`, `task test-e2e-signing` and `task test-image-refresh` are standard controller behavior checks and should normally run on the default `config-dir` install - `task test-e2e-quickstart-helm` is the important validation for the Helm install path - `task test-e2e-quickstart-manifest` is the important validation for the single-file manifest install path - `task test-e2e-full` is a full spec run that includes the bi-directional scenario, but it is still not the primary way to validate all install modes @@ -200,7 +199,7 @@ These namespaces are typically named: - `-test-manager` - `-test-signing` -- `-test-audit-redis` +- `-test-audit-consumer` - and so on Those namespaces are created in `BeforeAll` and removed in `AfterAll` for the suite. @@ -234,8 +233,9 @@ The only cluster-level variable still consumed across files is: Repo state (names, checkout paths, Secret names, optional receiver webhook info) flows through a typed `RepoArtifacts` struct returned by `SetupRepo`, not through package-global env vars. -One important current detail is that `audit_redis_e2e_test.go` uses file-local lazy repo -initialization so either top-level `Describe` can run first without hidden ordering assumptions. +One important current detail is that the remaining audit-consumer specs share a lazy repo +helper in `audit_helpers_test.go`, so `Commit Window Batching` and `Commit Request` can +run first without hidden ordering assumptions. ## Repo Model: One Repo Per E2E Test File @@ -245,7 +245,8 @@ Each repo-using e2e file owns its own repo: |---|---|---| | `e2e_test.go` | `testNamespaceFor("manager")` | `e2e-manager-` | | `signing_e2e_test.go` | `testNamespaceFor("signing")` | `e2e-signing-` | -| `audit_redis_e2e_test.go` | `testNamespaceFor("audit-consumer")` | `e2e-audit-redis-` | +| `commit_window_batching_e2e_test.go` / `commit_request_e2e_test.go` | `testNamespaceFor("audit-consumer")` via shared helper; per-spec producer namespaces | `e2e-audit-consumer-` | +| `commit_author_attribution_e2e_test.go` | `testNamespaceFor("commit-author")` | `e2e-commit-author-` | | `quickstart_framework_e2e_test.go` | `testNamespaceFor("quickstart-framework")` | `e2e-quickstart-framework-` | | `bi_directional_e2e_test.go` | `testNamespaceFor("bi-directional")` | `e2e-bi-directional-` | | `demo_e2e_test.go` | fixed `vote` | fixed `demo` | @@ -260,9 +261,9 @@ Repo stamps live under: not under the install namespace. -The `audit_redis_e2e_test.go` file additionally uses a `sync.Once`-backed `ensureAuditRedisRepo()` -helper so either of its two top-level `Describe` blocks can initialize the shared file-local repo -first without cross-container ordering coupling. +The `audit_helpers_test.go` file additionally uses a `sync.Once`-backed +`ensureAuditConsumerRepo()` helper so either audit-consumer container can initialize the shared +repo first without cross-container ordering coupling. ## Why CI Could Pass Even If This Was Fragile @@ -362,7 +363,9 @@ The e2e package currently contains: |---|---|---| | `e2e_test.go` | `manager` | Main reconciliation and GitOps behavior | | `signing_e2e_test.go` | `signing` | Commit signing behavior | -| `audit_redis_e2e_test.go` | `audit-redis` | Audit webhook to Redis producer/consumer | +| `commit_window_batching_e2e_test.go` | `audit-consumer` | Commit-window batching through the audit pipeline | +| `commit_request_e2e_test.go` | `audit-consumer` | CommitRequest finalization through the audit pipeline | +| `commit_author_attribution_e2e_test.go` | `manager`, `smoke` | OIDC claim to Git author attribution | | `image_refresh_test.go` | `image-refresh` | Build/load/restart invalidation chain | | `quickstart_framework_e2e_test.go` | `quickstart-framework` | Installer-level quickstart flow | | `bi_directional_e2e_test.go` | `bi-directional` | Flux plus gitops-reverser shared ownership | @@ -384,7 +387,7 @@ Currently that includes: - secret encryption path - ConfigMap create/delete Git commits - cluster-scoped CRD install Git commit -- audit-to-Redis producer/consumer path +- audit pipeline batching, commit-request, and author-attribution paths - one signing verification path That keeps the default run closer to a product smoke test than to a full infra regression sweep. @@ -416,7 +419,6 @@ Use: ```bash task test-e2e-manager task test-e2e-signing -task test-e2e-audit-redis task test-image-refresh task test-e2e-full ``` diff --git a/docs/design/upgrade-finding.md b/docs/design/upgrade-finding.md new file mode 100644 index 000000000..62103c9fc --- /dev/null +++ b/docs/design/upgrade-finding.md @@ -0,0 +1,157 @@ +# gitops-reverser WatchList Bookmark Report + +Date: 2026-05-25 + +## Summary + +`apiservice-audit-proxy` 0.5.0 appears to have addressed the original +transport-level watch-stream problem, but the remaining production warning in +`gitops-reverser` points to a separate client-side watch-list behavior. + +`gitops-reverser` is built with `k8s.io/client-go v0.36.1`. In that version, +the `WatchListClient` feature defaults to enabled. Dynamic informers created by +`gitops-reverser` therefore use Kubernetes streaming-list semantics: + +```text +watch=true +sendInitialEvents=true +allowWatchBookmarks=true +resourceVersionMatch=NotOlderThan +``` + +client-go then waits for a special bookmark event annotated as the end of the +initial events stream. The production warning means that bookmark is not being +received. + +## Production Symptom + +Representative log line: + +```text +Warning: event bookmark expired err="pkg/mod/k8s.io/client-go@v0.36.1/tools/cache/reflector.go:343: hasn't received required bookmark event marking the end of initial events stream, received last event 59.998946102s ago" +``` + +This is not a generic watch timeout. It specifically means client-go is waiting +for the initial-events-end bookmark required by streaming-list mode. + +## Evidence + +`gitops-reverser` creates dynamic shared informer factories for watched +resources: + +- `external-resources/gitops-reverser/internal/watch/manager.go` +- `external-resources/gitops-reverser/internal/watch/manager_catalog.go` + +The relevant call sites are: + +```go +dynamicinformer.NewDynamicSharedInformerFactory(client, 0) +dynamicinformer.NewFilteredDynamicSharedInformerFactory(client, 0, ns, nil) +``` + +and: + +```go +dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 0) +``` + +In `client-go v0.36.1`, dynamic informers wrap their list/watcher with +watch-list support: + +```go +cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{...}) +``` + +The reflector enables watch-list mode from the client-go feature gate: + +```go +r.useWatchList = clientfeatures.FeatureGates().Enabled(clientfeatures.WatchListClient) +``` + +`WatchListClient` defaults to true as of the v1.35 client-go feature defaults, +which are present in `v0.36.1`. + +When enabled, the reflector starts the initial stream with: + +```go +metav1.ListOptions{ + ResourceVersion: lastKnownRV, + AllowWatchBookmarks: true, + SendInitialEvents: ptr.To(true), + ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan, + TimeoutSeconds: &timeoutSeconds, +} +``` + +It only considers the initial stream complete when it receives a bookmark with: + +```go +meta.GetAnnotations()[metav1.InitialEventsAnnotationKey] == "true" +``` + +The repeated production warning is emitted by client-go while waiting for that +bookmark. + +## Interpretation + +The proxy can pass watch events through, but it does not and should not +synthesize Kubernetes streaming-list bookmarks. The initial-events-end bookmark +must be produced by the Kubernetes API server or aggregated backend serving the +watched resource. + +So the remaining failure is most likely: + +1. `gitops-reverser` accidentally opted into streaming-list mode through the + client-go upgrade. +2. One or more watched API paths, especially aggregated APIs, do not produce + the required initial-events-end bookmark. +3. client-go keeps the reflector in its initial watch-list phase and logs the + missing-bookmark warning every 10 seconds. + +The design documents in `gitops-reverser` discuss future use of +`sendInitialEvents=true`, but the current implementation uses ordinary dynamic +informers. That makes this behavior an implicit dependency-default change, +rather than a fully implemented and tested watch-list design. + +## Immediate Workaround + +Disable the client-go `WatchListClient` feature in the `gitops-reverser` pod: + +```yaml +env: + - name: KUBE_FEATURE_WatchListClient + value: "false" +``` + +The Helm chart already exposes `.Values.env`, so this can be applied as a values +override. + +Example values fragment: + +```yaml +env: + - name: KUBE_FEATURE_WatchListClient + value: "false" +``` + +After rollout, the dynamic informers should return to the older LIST-then-WATCH +behavior and stop requiring the initial-events-end bookmark. + +## Recommended Follow-Up + +For `gitops-reverser`, make the choice explicit instead of relying on the +client-go default: + +- Disable `WatchListClient` by default in the manager process, or document and + set the environment override in the chart defaults. +- Add a regression test or e2e lane that watches resources through the + aggregated API path and confirms informer sync completes. +- Revisit streaming-list support later as an intentional feature, with explicit + tests for `sendInitialEvents=true`, `allowWatchBookmarks=true`, and the + `k8s.io/initial-events-end` bookmark. + +For `apiservice-audit-proxy`, the existing watch-stream work remains useful: +the proxy still must keep watch responses open, flush promptly, avoid audit +body spooling for watches, and avoid imposing write deadlines. This report is +about a different layer: client-go reflector semantics in the downstream +consumer. diff --git a/docs/design/watch-and-catalog-architecture.md b/docs/design/watch-and-catalog-architecture.md new file mode 100644 index 000000000..760fc7e6f --- /dev/null +++ b/docs/design/watch-and-catalog-architecture.md @@ -0,0 +1,459 @@ +# Watch & Catalog Architecture — Requirements, Current State, Target Design + +Status: **vision / proposed** — supersedes the framing in +[watchrule-wildcard-support-plan.md](../future/watchrule-wildcard-support-plan.md), +[watchrule-wildcard-and-resolution-semantics.md](watchrule-wildcard-and-resolution-semantics.md), +and [rule-set-snapshot-discovery-lag-fix.md](../finished/rule-set-snapshot-discovery-lag-fix.md); +those become implementation details of the model described here. + +This document is in three parts: + +1. **Requirements** — how it *should* work, including requirements not yet + written down anywhere. +2. **Current architecture** — what exists today and where it falls short. +3. **Target architecture** — one clean model and how we get there. A large + refactor is acceptable; correctness and simplicity win over diff size. + +--- + +# Part 1 — Requirements: how it should work + +## 1.0 One sentence + +The application keeps a **live, authoritative model of the cluster's API +surface**, and every `GitTarget` mirrors a **deterministic projection** of that +surface (selected by its rules) into git, reacting to surface changes +**incrementally** and **never writing an uncertain or partial view**. + +## 1.1 The catalog is the single source of truth + +There must be exactly one component that knows "what does this cluster currently +serve, and can I trust that knowledge right now." Everything else +(resolution, informers, snapshots, status) asks the catalog; nothing else talks +to discovery or guesses. + +The catalog must: + +- **Maintain last-known-good.** A `GroupVersion` that blips out during an + apiserver rollout or CRD upgrade must not vanish from the model the instant + discovery hiccups. The catalog keeps the prior entries and marks the GV + *degraded*, not *gone*. +- **Distinguish the three reasons a resource can be "absent":** + 1. *Never served* / not yet served (stable). + 2. *Degraded* — discovery temporarily can't confirm it (transient). + 3. *Withdrawn* — confirmed removed from the surface (e.g. CRD uninstalled). + These three drive completely different downstream actions, so the catalog must + label them, not collapse them to "empty". +- **Debounce withdrawal.** A GV is only promoted from *degraded* to *withdrawn* + after a confirmation window / repeated clean discovery that still omits it. + This is what "abstracts away shortly-unavailable CRDs" means concretely. +- **Expose confidence.** A consumer can always ask "is the surface authoritative + enough to make a destructive (delete) decision right now?" +- **Emit deltas.** Consumers react to *changes* (GVR added / withdrawn / degraded + / recovered), not by recomputing the world each tick. + +## 1.2 Wildcards behave like Kubernetes policy APIs (non-negotiable) + +Skipping wildcards is **not** an option. `WatchRule`/`ClusterWatchRule` selectors +must follow the admission-webhook / RBAC / audit-policy idiom that users already +know: + +- `apiGroups: ["*"]` matches all groups; `resources: ["*"]` matches all + resources; `apiVersions: ["*"]` matches all versions. +- A `*` rule is a *live* selection: when a new CRD appears that matches it, the + target starts watching it automatically; when one is withdrawn, it stops. +- The selector is a **projection of the catalog**, evaluated continuously — not a + one-time literal expansion frozen at apply time. + +Decisions to lock (see open questions): the meaning of *empty* vs `*` (proposal: +empty `apiGroups` = "resolve to the single served group, else surface an error"; +`["*"]` = "all served groups" — a strict mode and a wildcard mode, both useful), +and that *subresources* (`pods/log`, `pods/*`) are **out of scope** for the +list/watch+mirror model and must not be advertised. + +## 1.3 Incremental, relaxed reconcile + +An active `GitTarget` must respond to a newly added or removed CRD by changing +**only the part that changed**: + +- New GVR matches a target's plan → start one informer for that GVR, seed **only + that GVR's objects** into the mirror, done. No re-snapshot of the rest of the + target. +- GVR withdrawn → stop that one informer; apply the configured withdrawal policy + to only that GVR's files. + +The unit of reconciliation is the **(GitTarget, GVR) cell**, not the whole +GitTarget. Rationale (the user's, and worth stating as a requirement): because +live audit/event coverage is high-quality, a full-target re-list is rarely needed +— the only thing a new GVR genuinely needs is its *initial seed*; steady state is +maintained by events. A full re-snapshot should be a **fallback**, not the +normal path. + +## 1.4 The authoritative-write invariant (the core safety property) + +The mirror must **never** be written from a partial or uncertain view, because a +resource that is merely *unconfirmed* is indistinguishable from a *deleted* one, +and writing the difference would mirror phantom deletions into git. + +Concretely: + +- A delete is only ever written for a cell whose source-of-truth was + **authoritatively observed** (a successful list of that GVR, or a real DELETE + event for that object) **or** whose removal was **intent-driven** (the rules no + longer select it — see 1.6). +- If a GVR's list fails, its GV is degraded, or the controller lacks permission, + that cell is **left untouched and retried** — an *inability to observe* must + never be read as a deletion. +- The dividing line is **intent vs observability**: a cell leaving the desired + set because of a rule/scope/policy change is an intent-driven mutation and may + delete; a cell that is merely unobservable right now must hold. +- This invariant is per-cell, so one flaky GVR cannot corrupt an entire target. + +## 1.5 Convergence, determinism, idempotency + +- **Deterministic projection.** A target's desired mirror is a pure function of + (its rules, the authoritative catalog surface, cluster object state). Same + inputs → same files. This is what makes the plan-hash and tests trustworthy. +- **Convergence.** After any transient disruption (discovery gap, list failure, + permission blip), the system returns to the correct mirror automatically within + a bounded time, with no operator action and no spurious churn commits. +- **Idempotency.** Re-running a reconcile with unchanged inputs produces no + commit. + +## 1.6 A cell can disappear for several reasons; the cause decides the action + +This is the most important semantic to get right. "Cell removed" is **not** one +case — and collapsing them all into one policy is wrong. A cell can leave a +target's desired set because of an **intent** change or because of an +**observability** change, and the mirror must treat those oppositely: + +| Cause | Class | Mirror action | +|---|---|---| +| Rule deleted / narrowed / ops changed so the cell is no longer selected | intent | **Delete** the cell's managed files — required to keep the mirror a deterministic projection of the rules. | +| Namespace / scope change | intent | Old cell deselected → delete its managed files; new cell selected → seed it. | +| GitTarget rewiring (provider/branch/path change) | intent | Projection moves: delete from the old path, seed at the new one. | +| Resource policy now excludes the kind | intent/config | Likely **delete** (open question 6) — it is a config-driven deselection, not an outage. | +| Surface withdrawal (CRD uninstalled) | observability | **Keep** files (proposed) and record withdrawal in status/log. Absence is an operator surface change, not GitOps intent; deleting hundreds of files on a CRD upgrade/reinstall would be destructive and noisy. | +| Object deleted while kind still served | observability (authoritative) | Real deletion, driven by DELETE event / authoritative re-list → **delete** the file. | +| GV degraded / list failed / permission denied | observability (non-authoritative) | **Hold** the cell — never delete (see 1.4). | + +The principle: **only intent-driven removals and authoritatively-observed +deletions mutate the mirror; non-authoritative absences hold.** The reconciler +must therefore know *why* a cell left the desired set, not merely *that* it did. + +This requires a notion of **managed projection** (see 1.7): the set of files a +given cell owns, so "delete the cell's files" is well-defined and scoped. + +## 1.7 Requirements you may be missing + +Beyond the four beliefs above, a complete design needs: + +- **Deterministic file ownership (managed projection).** There must be a + deterministic mapping from a cell `{GitTarget, GVR, scope, namespaces}` to the + set of repo paths it owns, and a way to enumerate/diff *only* those paths. This + underpins two things at once: deleting exactly a deselected cell's files (1.6), + and **cell-scoped repo diffing** so a per-cell reconcile never has to read or + rewrite the whole GitTarget path (which would smuggle whole-target behavior + back in). Files not owned by any current cell are "unmanaged" and are never + touched unless an intent change brought them into scope. + +- **RBAC / least privilege.** "Watch everything" means the controller's + ServiceAccount must `list`/`watch` every selected GVR. A wildcard implies broad + RBAC. Required: either a documented broad ClusterRole, or per-GVR + permission-denied handling that degrades that *cell* gracefully (treat as + "cannot observe" → never delete, surface in status) instead of failing the + target. +- **Scale bounds & backpressure.** One informer per selected GVR across the whole + surface is real memory + apiserver load, and a CRD burst can cause a list + storm. Required: a soft cap with observability, and coalescing of surface + bursts (the existing single-slot refresh channel is a start). +- **Startup gating.** On cold start, do not snapshot (and certainly do not delete) + until the catalog has reached initial trust — otherwise an empty early surface + could wipe the mirror. +- **Observability / status conditions.** `WatchRule`/`ClusterWatchRule`/`GitTarget` + status must show: resolved GVR count (incl. wildcard expansion), degraded + surface, withdrawn types, permission failures, and unseeded/retrying cells. +- **Version selection.** A resource served at multiple versions must mirror + exactly one (preferred) to avoid duplicate files; define wildcard-version + behavior (proposal: still collapse to preferred unless a specific version is + named). +- **Resource policy under wildcard.** The default denylist + (pods/events/leases/…) still applies to `*`; define whether users can opt + excluded kinds back in. +- **Sensitive resources.** Wildcards will select Secrets and similar. This must + compose with the existing sensitive-resource classification work + ([sensitive-resource-classification-plan.md](sensitive-resource-classification-plan.md)). +- **Migration / backward compatibility.** Changing selector semantics changes + what *existing* rules watch. Required: no silent expansion of an existing user's + scope on upgrade — a migration story or explicit opt-in. +- **Causal ordering per cell.** A GVR's initial seed must be reconciled before its + live events are flushed, so the seed isn't overwritten or double-counted. Today + this is done per-target; it must hold per-cell. + +--- + +# Part 2 — Current architecture + +## 2.1 Components + +- **`APIResourceCatalog`** ([api_resource_catalog.go](../../internal/watch/api_resource_catalog.go)) + — in-memory index of served resources (`byGVR`, `byResource`, `byGroupRes`, + `byGroupVer`) with per-GV state (`trusted`/`degraded`) and a `generation` + counter. Refreshed from discovery; **already preserves entries for degraded + GVs** (`markFailedGroupVersions`) and removes confirmed-gone GVs only when + discovery is complete (`removeUndiscoveredGroupVersions`). +- **`RuleGVRResolver`** ([rule_gvr_resolver.go](../../internal/watch/rule_gvr_resolver.go)) + — maps one rule selector to concrete GVRs via the catalog, returning + `ResolveMiss` values classified by reason. **Refuses wildcards** in + `preflightMisses`. +- **`Manager`** ([manager.go](../../internal/watch/manager.go)) — owns the + reconcile loop, the active informer set (`activeInformers` keyed by + `GVR → namespace → cancel`), GVR resolution for both the watch plan and the + snapshot, and snapshot emission. +- **Event surface triggers** ([manager_catalog.go:278-303](../../internal/watch/manager_catalog.go#L278-L303)) + — informers on CRDs and APIServices that coalesce changes into + `catalogRefreshCh`. +- **`EventRouter` / per-GitTarget streams / reconcilers** — translate informer + events and control events (`RequestRepoState`, `RequestClusterState`) into git + commits, with a RECONCILING→LIVE buffering handshake per target. + +## 2.2 Control flow today + +``` +discovery / CRD trigger / 30s tick + │ + ▼ +ReconcileForRuleChange (manager.go:740) + ├─ RefreshAPIResourceCatalog → catalog updated + ├─ computeRequestedGVRs → resolve ALL rules → desired GVR set + ├─ compareGVRs → added / removed (per GVR+namespace) + ├─ snapshotTargetsNeedingDelivery → per-target plan-hash diff + ├─ stopInformer(removed) + ├─ beginReconciliationForTargets → buffer live events + ├─ startInformersForGVRs(added) → seed via informer cache + ├─ emitSnapshotForRuleChange → RequestRepoState + RequestClusterState + │ └─ whole-target cluster list, ABORTS on any blocking miss/list error + └─ completeReconciliationForTargets → flush buffered events +``` + +## 2.3 What already works in our favor + +- **Dynamic surface reaction exists.** The CRD/APIService triggers + reconcile + loop already start/stop informers as the surface changes. Reacting to a new + GVR is the steady state, not a new capability. +- **Per-(GVR, namespace) informer diffing exists** (`compareGVRs`, + `findAddedGVRs`/`findRemovedGVRs`). +- **Degraded-aware catalog exists** — last-known-good is already preserved. +- **Per-target snapshot isolation exists** (plan-hash selection). + +## 2.4 Where it falls short (the gaps Part 3 must close) + +1. **Two resolution call sites, two truths.** The watch-plan path + (`currentRuleSetSnapshots`, manager.go:1248/1264) and the snapshot path + (manager.go:559-594) both resolve rules independently — duplicated logic that + has already drifted (the bug that birthed + [watchrule-gvr-resolution-plan.md](../finished/watchrule-gvr-resolution-plan.md)). +2. **Snapshots are whole-target, not per-cell.** Adding one GVR re-lists and + re-diffs the *entire* target (`emitSnapshotForRuleChange` → + `RequestClusterState` over all GVRs). Violates requirement 1.3. +3. **One failure aborts the whole target.** A single GVR list failure or blocking + miss aborts the entire target snapshot (manager.go:596-628). Safe but coarse; + at wildcard scale it means a target may rarely snapshot. Violates the + *per-cell* form of requirement 1.4. +4. **Wildcards refused** at planning (requirement 1.2 unmet) — and the CRD field + docs advertise the opposite, so a `*` rule applies cleanly and silently + watches nothing. +5. **Degraded handling is bolted on, not central.** The discovery-lag fix + reintroduces "is this authoritative?" logic at the snapshot-selection layer + because the catalog doesn't expose confidence/deltas as first-class outputs. + Many `ResolveMiss` reasons + special cases = the exception sprawl the user + wants gone. +6. **Withdrawal semantics are emergent**, not decided (requirement 1.6). + +--- + +# Part 3 — Target architecture + +The strategy: make the **catalog** a proper live model that emits deltas and +confidence, make the **watch plan** a single deterministic projection consumed in +one place, and make **reconciliation per-cell and incremental**. This removes the +duplicate resolution, the whole-target snapshots, and most of the miss/degraded +special-casing in one move. + +## 3.1 Layer 1 — `ClusterSurface` (evolve the catalog) + +Promote `APIResourceCatalog` into the authoritative, observable +`ClusterSurface`: + +- **State per GVR:** `served | degraded | withdrawn`, plus the existing + allowed/listable/scope/preferred metadata. (`served`/`degraded` already exist; + add explicit `withdrawn` with a debounce before a degraded GV is declared + withdrawn.) +- **Per-cell confidence, not a global flag.** Expose + `AuthoritativeFor(GVR) Confidence` (and a startup-trust gate), **not** a global + `Authoritative() bool`. A single boolean would either block healthy targets + because one unrelated APIService is degraded, or get bypassed and become unsafe. + Confidence travels with each planned cell so destructive decisions are gated + per cell (1.4). Degradation is observed at GV granularity (discovery fails per + GroupVersion) and projected down to the GVRs in that GV — see open question 7 + on GV-vs-GVR granularity. +- **Current surface + generation is the source of truth; deltas are only + triggers.** Expose a queryable current surface snapshot (with its `generation`) + as the authority, and a `Subscribe() <-chan SurfaceDelta` + (`{GVR, Change: Added|Withdrawn|Degraded|Recovered}`) purely as an + *optimization* that tells consumers *when* and *roughly what* to re-evaluate. + Consumers must establish correctness by **re-diffing desired cells against + active cells** against the current surface, never by replaying deltas as if they + were complete — deltas can be missed across restart, backpressure, or + subscription churn. Deltas make reconvergence cheap; the re-diff makes it + correct. +- **Absorbs transience centrally.** No consumer ever sees a flapping CRD as + add/remove churn; the surface only reports `Withdrawn` after the debounce. This + is requirement 1.1, in one place, deleting the need for scattered + `CatalogUnavailable`/`DiscoveryDegraded` handling elsewhere. + +## 3.2 Layer 2 — `WatchPlan` (one resolver, projection semantics) + +A single `WatchPlan` projects rules onto the surface, used by **both** informers +and snapshots (kills gap 2.4.1): + +- `Plan(rules, surface) → map[GitTarget]TargetPlan`, where a `TargetPlan` is a + set of **cells**: `{GVR, scope, namespaces, ops}`. +- **Wildcards are projection, not expansion-at-apply** (requirement 1.2): `*` + selects all matching `served` cells from the surface *now*; the plan is + recomputed on every surface delta, so membership tracks the cluster live. +- **Empty vs `*`**: empty = strict unique-or-error; `*` = all. One small, explicit + rule — not a pile of miss reasons. +- **Resolution errors still exist and still matter.** Strict-empty ambiguity, + not-served, and disallowed are real errors the resolver must report. The + critical rule: an unresolved or errored selector must mark its cells + **unresolved/held**, never collapse them into an *empty desired set* — otherwise + a failed resolve looks like "the user wants nothing here" and the reconciler + deletes the projection (1.4). **Resolution failure ≠ empty desired state.** + What this design removes is the *duplicated, drifting* blocking-miss handling + spread across two resolution sites and the snapshot layer — not error handling + itself, which consolidates into one place with one meaning. + +## 3.3 Layer 3 — per-cell reconciler (incremental) + +Replace whole-target snapshotting with a **cell reconciler** driven by two delta +sources — rule changes and surface deltas — joined against current plans: + +``` +SurfaceDelta{GVR, …} ─┐ (triggers only) +RuleChange{target} ─┼─► RE-DIFF desired cells (WatchPlan over current +periodic tick ─┘ surface+generation) vs active cells + managed files + │ + ▼ + per-cell actions: + cell appeared → start informer + SEED ONLY THIS GVR (cell-scoped) → additions + cell removed → classify cause (1.6): + intent (rule/scope/policy) → stop informer + DELETE the + cell's managed files + surface withdrawal → stop informer + KEEP files, + record in status + non-authoritative (degraded/ + denied/list-failed) → HOLD: do not stop as removed, + do not delete, retry + cell unchanged → nothing +``` + +Two things this diagram makes explicit, both from review: + +- **Correctness is the re-diff, not the delta.** Desired cells are recomputed + from the `WatchPlan` over the *current* surface snapshot (and its generation) + and diffed against active cells and the managed-file set. Deltas and the + periodic tick only *schedule* a re-diff; missing a delta can never cause + divergence because the next re-diff is authoritative (§3.1). +- **Repo-side scoping must match.** Seeding/diffing/deleting a cell operates only + on that cell's managed files (1.7). `RequestRepoState` must be answerable at + cell granularity — list/diff just `{target, GVR, scope, namespaces}` — or the + per-cell path silently degrades back into whole-target snapshotting. + +Properties this gives us for free: + +- **Incremental** (requirement 1.3): a new CRD seeds exactly one GVR's objects on + the targets that select it; everything else is untouched. +- **Per-cell safety** (requirement 1.4): seeding lists only that GVR; on failure + the cell stays `unseeded/retrying` and writes nothing — no other cell, no + deletions. The "partial view = phantom deletes" failure mode becomes + structurally impossible because deletes are scoped to a cell that just listed + successfully. +- **Convergence** (requirement 1.5): a failed/degraded cell is simply retried on + the next relevant delta or tick; no target-wide churn. +- **Full re-snapshot becomes the fallback**, triggered only by explicit + resync/repair, not routine surface changes. + +Per-cell causal ordering reuses the existing RECONCILING→LIVE buffering, scoped to +the cell (requirement 1.7). + +## 3.4 How the existing docs fold in + +- **Discovery-lag fix** → disappears as a special case: the surface never reports + a degraded GVR as withdrawn, so a transient gap can't drop a cell or erase a + baseline. The plan-hash machinery is replaced by cell-level delta diffing. +- **Wildcard semantics doc** → becomes the spec for `WatchPlan` projection. +- **Wildcard support plan** → its phases map onto building Layers 1–3; the + "snapshot robustness" risk (its Phase 2) is resolved structurally by per-cell + seeding rather than needing a bespoke list-failure policy. +- **Initial-seed-via-watch-list** → the per-cell seed is exactly where the + cache-backed `WatchListClient` seed belongs. + +## 3.5 Suggested build order + +Each step is independently shippable and leaves the system correct. + +1. **Unify resolution** behind one `WatchPlan` used by both informer and snapshot + paths (removes drift; no behavior change). Pure refactor, fully unit-testable. +2. **Add surface state + deltas** to the catalog (`served/degraded/withdrawn`, + `Authoritative()`, `Subscribe()`), with withdrawal debounce. Internal; no + consumer change yet. +3. **Per-cell seeding** for *added* cells (incremental seed), keeping whole-target + snapshot as the fallback path. Closes requirement 1.3 for additions. +4. **Per-cell withdrawal policy** (requirement 1.6) for removed cells. +5. **Enable wildcards** in `WatchPlan` projection (requirement 1.2) — now safe + because Layers 1–3 handle dynamic membership, per-cell safety, and scale. +6. **Fix the field docs + admission validation** to match (and add status + conditions, RBAC guidance, scale caps). + +## 3.6 Cleanliness check (the user's steer) + +This design *removes* code as much as it adds: one resolver instead of two; one +authority for "is the surface trustworthy" instead of miss-reason handling spread +across resolver, snapshot, and selection layers; structural per-cell safety +instead of the whole-target abort + discovery-lag patch. The number of distinct +"cases" a maintainer must hold in their head drops from "many `ResolveMiss` +reasons × two resolution sites × whole-target abort rules × discovery-lag +bookkeeping" to "a surface with three states (queried as the source of truth, +with deltas as cheap triggers), projected by one resolver into cells that each +reconcile independently and classify their own removal cause." + +--- + +## Open decisions (need a human call before building) + +1. **Empty vs `*`**: adopt the strict-`[]` / wildcard-`*` split, or make `[]` + also mean "all" to match RBAC exactly? (Proposal: strict `[]`, wildcard `*`.) +2. **Surface-withdrawal default**: keep files on CRD uninstall (proposed) or + delete them? Per-GitTarget override? (This is the *observability* class in 1.6.) +3. **RBAC model for wildcard**: ship a broad ClusterRole, or require users to + grant it and degrade cells gracefully on permission-denied? +4. **Migration**: how do existing rules behave on upgrade — frozen semantics, or + opt-in to the new projection? +5. **Scale cap**: hard limit on informer count per controller, or + observe-and-warn only? +6. **Resource-policy deselection** (ref 1.6): when the default denylist newly + excludes a kind that was being mirrored, delete its managed files (treat as a + config-driven intent change) or keep them (treat as observability)? +7. **`withdrawn` granularity** (ref 3.1): is withdrawal a GV-level state, a + GVR-level state, or both? Degradation is naturally GV-level (discovery fails + per GroupVersion) while deltas are GVR-level — the model must state how the two + compose rather than switching between them. +8. **Preferred-version change**: when a resource's preferred version changes, is + that `Withdrawn(old)+Added(new)` (risks a delete+reseed and file churn), an + `Updated` delta, or an explicit **migration** action (move/rewrite files, + never delete)? Affects whether a routine apiserver upgrade churns the mirror. +9. **Rule-deselection disposition** (ref 1.6): when a rule stops selecting a cell, + delete its managed files immediately (proposed — preserves the deterministic + mirror) or leave them as *unmanaged historical* files? If the latter, define + how unmanaged files are marked and ever reclaimed. diff --git a/docs/design/watch-audit-rule-matching-improvement.md b/docs/design/watch-audit-rule-matching-improvement.md index 28d96f176..bcb2ef3a8 100644 --- a/docs/design/watch-audit-rule-matching-improvement.md +++ b/docs/design/watch-audit-rule-matching-improvement.md @@ -481,7 +481,8 @@ E2E tests (existing, to be strengthened per the E2E section below): - [test/e2e/e2e_test.go](/workspaces/gitops-reverser/test/e2e/e2e_test.go:717) - [test/e2e/tilt_playground_e2e_test.go](/workspaces/gitops-reverser/test/e2e/tilt_playground_e2e_test.go:55) -- [test/e2e/audit_redis_e2e_test.go](/workspaces/gitops-reverser/test/e2e/audit_redis_e2e_test.go:54) +- [test/e2e/commit_window_batching_e2e_test.go](/workspaces/gitops-reverser/test/e2e/commit_window_batching_e2e_test.go:54) +- [test/e2e/commit_request_e2e_test.go](/workspaces/gitops-reverser/test/e2e/commit_request_e2e_test.go:54) ## E2E Evidence From A Real Repo History diff --git a/docs/design/watchrule-wildcard-and-resolution-semantics.md b/docs/design/watchrule-wildcard-and-resolution-semantics.md new file mode 100644 index 000000000..6b4fe8a22 --- /dev/null +++ b/docs/design/watchrule-wildcard-and-resolution-semantics.md @@ -0,0 +1,131 @@ +# WatchRule Wildcard & Resolution Semantics + +How GitOps Reverser turns a `WatchRule`/`ClusterWatchRule` resource selector into +a concrete set of watchable GVRs, what it deliberately refuses, and why there is +no "watch everything" rule. + +Source of truth: `RuleGVRResolver` in +[rule_gvr_resolver.go](../../internal/watch/rule_gvr_resolver.go), backed by the +[APIResourceCatalog](../../internal/watch/api_resource_catalog.go). See also +[watchrule-gvr-resolution-plan.md](../finished/watchrule-gvr-resolution-plan.md). + +> **Note:** this documents *current* behavior. The CRD field docs promise +> admission-webhook-style wildcard semantics that this resolver does not yet +> implement — see +> [watchrule-wildcard-support-plan.md](../future/watchrule-wildcard-support-plan.md) for +> that finding and the plan to resolve it. + +## Two directions, treated differently + +Resource selectors are used in two distinct places, and they handle wildcards +**oppositely**: + +| Direction | Code | Question it answers | `*` behavior | +|---|---|---|---| +| **Planning / resolution** | `RuleGVRResolver.Resolve(...)` ([rule_gvr_resolver.go:67](../../internal/watch/rule_gvr_resolver.go#L67)) | "Which concrete GVRs should I set up informers for and snapshot?" | **Refused** (`Wildcard*` miss). | +| **Matching** | `matchesAPIGroups`/`matchesResources` ([manager.go:480-513](../../internal/watch/manager.go#L480-L513)) | "Does this concrete incoming event's GVR match a rule?" | **Honored** (`"*"` matches anything). | + +The planning direction is the only one that creates informers and drives +snapshots (`gvrFromCompiledRule → Resolve`, +[gvr.go:79-92](../../internal/watch/gvr.go#L79-L92)). The matching direction only +runs against a GVR an informer **already delivered** +([collectWatchRuleNamespaces](../../internal/watch/manager.go#L416-L424)). + +**Consequence:** a `*` rule with no concrete companion rule sets up zero +informers and therefore watches nothing. The matching-side `*` only ever matches +events some *other* concrete rule already brought in. A lone `resources: ["*"]` +rule looks like "watch everything" but is effectively a no-op (it logs a +`WildcardResource` miss at `V(1)`). + +## The preflight gate + +`Resolve` calls `resolveResource` per resource name, which first runs +`preflightMisses` ([rule_gvr_resolver.go:108-131](../../internal/watch/rule_gvr_resolver.go#L108-L131)). +This gate short-circuits **before the catalog is ever queried**: + +| Condition | Result | Meaning | +|---|---|---| +| `resource == ""` | dropped, no miss | Empty resource entry is ignored. | +| `resource == "*"` | `WildcardResource` | `*` in `resources` — expansion unsupported. | +| `hasWildcard(groups)` | `WildcardGroup` | `*` in `apiGroups` — expansion unsupported. | +| `strings.Contains(resource, "/")` | `NotServed` | Subresource (e.g. `pods/status`) planning unsupported. | +| catalog nil or not `Ready()` | `CatalogUnavailable` | Discovery hasn't populated a catalog yet. | +| otherwise | proceed | Look the resource up in the catalog. | + +`WildcardResource` vs `WildcardGroup` is purely *which axis* held the `*`. +Resource is checked first, so `apiGroups:["*"], resources:["*"]` reports +`WildcardResource`. + +## What the resolver *does* expand from the catalog + +Two narrower, **bounded** expansions are supported — neither is an open wildcard: + +- **Empty `apiGroups`** (not `*`): `resourceCandidates` looks the resource name + up across *all* groups via + [`entriesForResource`](../../internal/watch/api_resource_catalog.go#L286-L290), + but [`ambiguityMiss`](../../internal/watch/rule_gvr_resolver.go#L141-L152) + refuses with `Ambiguous` if more than one group serves that name. So it is + "find this one kind without naming its group," requiring a unique answer — not + a fan-out. +- **Empty or `*` `apiVersions`**: + [`choosePreferredVersions`](../../internal/watch/rule_gvr_resolver.go#L212-L235) + collapses to the single **preferred** version per group-resource. + +Note the asymmetry: a version wildcard *is* accepted (it resolves to exactly one +version), but group/resource wildcards are refused. The dividing line is +**boundedness**: does the selector resolve to a fixed, knowable set? + +## There is no "watch everything" rule + +By design, no single rule can watch the whole cluster: + +- `resources: ["*"]` → `WildcardResource`, refused. +- `apiGroups: ["*"]` → `WildcardGroup`, refused. +- Empty `apiGroups` → resolves *one* kind across groups, and only if unambiguous. +- Empty `resources` → the `Resolve` loop iterates `resources`, so it yields zero + GVRs and watches nothing. + +The only way to watch broadly is to **enumerate each concrete resource kind +explicitly**. That list is static: a newly installed CRD is not picked up until a +rule names it (or a wildcard pattern would match it — which is exactly what is +refused). Dynamic "all current and future kinds" is intentionally not +expressible. + +## Why this is intentional + +1. **Bounded vs unbounded.** A version wildcard collapses to one preferred + version. A group/resource wildcard expands to an open-ended set whose + membership depends on what is installed right now. The resolver only expands + selectors that resolve to a fixed set. +2. **Plan stability.** A wildcard plan's membership would change every time *any* + CRD is installed or removed, churning the per-GitTarget effective-watch-plan + hash on unrelated cluster activity — the same spurious re-snapshot problem + described in + [rule-set-snapshot-discovery-lag-fix.md](../finished/rule-set-snapshot-discovery-lag-fix.md), + but permanent and amplified. Refusing wildcards keeps watch plans + deterministic. +3. **Snapshot safety.** Snapshotting `*` would commit the entire cluster (minus + policy-excluded kinds) to git on every rule-change snapshot — heavy, + surprising, and far more exposed to the partial-view aborts that + `RequestClusterState` enforces ([manager.go:596-601](../../internal/watch/manager.go#L596-L601)). +4. **A terse alternative already exists.** Empty `apiGroups` lets a rule name a + resource without its group as long as the name is unambiguous, so users are + not pushed toward wildcards just to stay concise. + +## Relationship to the snapshot miss classification + +`blockingSnapshotMisses` ([manager.go:657-669](../../internal/watch/manager.go#L657-L669)) +groups `WildcardGroup`/`WildcardResource` with `CatalogUnavailable`/ +`DiscoveryDegraded` as "blocking" — all four prevent emitting a snapshot. But the +two families differ in nature: + +- **Transient-blocking** (`CatalogUnavailable`, `DiscoveryDegraded`): the answer + changes once discovery recovers. Genuinely "ask again later." +- **Stable-blocking** (`WildcardGroup`, `WildcardResource`): a fixed property of + the rule text. Retrying never changes it; it is effectively a configuration + error to surface to the user. + +This distinction matters for the state model in +[rule-set-snapshot-discovery-lag-fix.md](../finished/rule-set-snapshot-discovery-lag-fix.md): +only the transient family should drive "retry next cycle"; the wildcard family +will not resolve on its own. diff --git a/docs/design/aggregated-api-requestheader-ca-miswire-report.md b/docs/finished/aggregated-api-requestheader-ca-miswire-report.md similarity index 100% rename from docs/design/aggregated-api-requestheader-ca-miswire-report.md rename to docs/finished/aggregated-api-requestheader-ca-miswire-report.md diff --git a/docs/design/audit-consumer-username-attribution-gap.md b/docs/finished/audit-consumer-username-attribution-gap.md similarity index 100% rename from docs/design/audit-consumer-username-attribution-gap.md rename to docs/finished/audit-consumer-username-attribution-gap.md diff --git a/docs/design/commit-window-refactor.md b/docs/finished/commit-window-refactor.md similarity index 100% rename from docs/design/commit-window-refactor.md rename to docs/finished/commit-window-refactor.md diff --git a/docs/future/design-audit-ingestion-hardening.md b/docs/finished/design-audit-ingestion-hardening.md similarity index 100% rename from docs/future/design-audit-ingestion-hardening.md rename to docs/finished/design-audit-ingestion-hardening.md diff --git a/docs/future/design-commit-context-api.md b/docs/finished/design-commit-context-api.md similarity index 99% rename from docs/future/design-commit-context-api.md rename to docs/finished/design-commit-context-api.md index edfff086a..183416e00 100644 --- a/docs/future/design-commit-context-api.md +++ b/docs/finished/design-commit-context-api.md @@ -8,7 +8,7 @@ > for the deep-dive on the aggregated-API audit gap; everything else here is reframed there. > > Original status: design — concrete pass at Option 3 from -> [idea-end-user-commit-messages.md](idea-end-user-commit-messages.md). +> [idea-end-user-commit-messages.md](../future/idea-end-user-commit-messages.md). > Date: 2026-05-07 ## Goals @@ -138,7 +138,7 @@ These are listed to validate that the resource shape generalises. None of them a initial implementation. - `commitNow` — finalize the matching open commit window immediately instead of waiting for the - silence timer. Open questions are tracked in [idea-end-user-commit-messages.md](idea-end-user-commit-messages.md) + silence timer. Open questions are tracked in [idea-end-user-commit-messages.md](../future/idea-end-user-commit-messages.md) under "Commit now." - `proposeAsPullRequest` — instruct the branch worker to push the in-progress commit window to a feature branch and open a pull request via `GitProvider`, instead of pushing to the target @@ -763,9 +763,9 @@ namespace `team-a`. ## References - Parent design exploration: - [idea-end-user-commit-messages.md](idea-end-user-commit-messages.md) + [idea-end-user-commit-messages.md](../future/idea-end-user-commit-messages.md) - Audit-carried transport addendum (alternatives this design supersedes for v1alpha1): - [addendum-end-user-commit-messages-audit-transports.md](addendum-end-user-commit-messages-audit-transports.md) + [addendum-end-user-commit-messages-audit-transports.md](../future/addendum-end-user-commit-messages-audit-transports.md) - Existing audit pipeline reference: `docs/design/audit-consumer-next-steps.md` - The aggregated-API audit gap and the related proxy: [external-sources/apiservice-audit-proxy/README.md](../../external-sources/apiservice-audit-proxy/README.md) diff --git a/docs/future/design-commit-request-api.md b/docs/finished/design-commit-request-api.md similarity index 96% rename from docs/future/design-commit-request-api.md rename to docs/finished/design-commit-request-api.md index 9a1128ecd..6ea9f957d 100644 --- a/docs/future/design-commit-request-api.md +++ b/docs/finished/design-commit-request-api.md @@ -1,7 +1,7 @@ # Design: `CommitRequest` CRD > Status: **implemented**. Namespaced CRD in API group `configbutler.ai/v1alpha1`. -> Deferred work is tracked in [design-commit-request-phase-2.md](design-commit-request-phase-2.md). +> Deferred work is tracked in [design-commit-request-phase-2.md](../future/design-commit-request-phase-2.md). > Date: 2026-05-18 ## What it is @@ -168,7 +168,7 @@ every event already commits immediately, so the signal simply finds no open wind ## Alternatives considered -The parent exploration ([idea-end-user-commit-messages.md](idea-end-user-commit-messages.md)) and +The parent exploration ([idea-end-user-commit-messages.md](../future/idea-end-user-commit-messages.md)) and the superseded aggregated-API design ([design-commit-context-api.md](design-commit-context-api.md)) weighed several transports. They are summarized here so the choice is not relitigated. @@ -209,15 +209,15 @@ clean success/SHA signal. Garbage collection, multi-target "save everything I edited," retry for transient finalize failures, and outcome metrics are intentionally out of this first cut. They are tracked — with the object-lifecycle question front and center — in -[design-commit-request-phase-2.md](design-commit-request-phase-2.md). +[design-commit-request-phase-2.md](../future/design-commit-request-phase-2.md). ## References -- Deferred / phase-2 work: [design-commit-request-phase-2.md](design-commit-request-phase-2.md) +- Deferred / phase-2 work: [design-commit-request-phase-2.md](../future/design-commit-request-phase-2.md) - Superseded aggregated-API design, kept for the audit-gap deep-dive: [design-commit-context-api.md](design-commit-context-api.md) - Parent exploration — transport options and the audit-stream-as-source-of-truth principle: - [idea-end-user-commit-messages.md](idea-end-user-commit-messages.md) + [idea-end-user-commit-messages.md](../future/idea-end-user-commit-messages.md) - Audit ingestion pipeline this design rides on: [design-audit-ingestion-hardening.md](design-audit-ingestion-hardening.md) - Implementation: `api/v1alpha1/commitrequest_types.go`, diff --git a/docs/future/design-rule-change-snapshot-trigger.md b/docs/finished/design-rule-change-snapshot-trigger.md similarity index 98% rename from docs/future/design-rule-change-snapshot-trigger.md rename to docs/finished/design-rule-change-snapshot-trigger.md index 669a77342..fc05fed3a 100644 --- a/docs/future/design-rule-change-snapshot-trigger.md +++ b/docs/finished/design-rule-change-snapshot-trigger.md @@ -213,7 +213,7 @@ The `EventRouter.SnapshotDeliveryDrops()` counter stays at 0 in steady state und ## References -- Trigger bug summary: [idea-cross-kind-dependency-watches.md](idea-cross-kind-dependency-watches.md) (related coordination concern around dependency-watches). +- Trigger bug summary: [idea-cross-kind-dependency-watches.md](../future/idea-cross-kind-dependency-watches.md) (related coordination concern around dependency-watches). - The old early-return: `ReconcileForRuleChange` in [manager.go](../../internal/watch/manager.go). - The snapshot consumer drop instrumentation: `RouteRepoStateEvent` and `RouteClusterStateEvent` in [event_router.go](../../internal/watch/event_router.go). - The GitTarget restart gate: `evaluateSnapshotGate` in [gittarget_controller.go](../../internal/controller/gittarget_controller.go). diff --git a/docs/design/e2e-cleanup-scoping.md b/docs/finished/e2e-cleanup-scoping.md similarity index 96% rename from docs/design/e2e-cleanup-scoping.md rename to docs/finished/e2e-cleanup-scoping.md index abb5514da..3f1d8b1f3 100644 --- a/docs/design/e2e-cleanup-scoping.md +++ b/docs/finished/e2e-cleanup-scoping.md @@ -15,7 +15,7 @@ The e2e suite originally had one global flag, `e2ePreserveResources`, that decid 2. The registered Gomega fail handler and the SIGINT/SIGTERM handler — both fed the same global preservation bit, so a normal assertion failure was effectively treated like a suite-wide abort. 3. The playground bootstrap, which flipped it deliberately to keep the playground live for the developer after the suite finishes. -Once the flag was on, `skipCleanupBecauseResourcesArePreserved` returned `true` for every cleanup call until the process exited. That meant a single failure poisoned the rest of the run: every spec that ran afterward leaked its WatchRules, GitTargets, ConfigMaps, and namespaces. Those leftovers then interfered with subsequent specs through the controller's own informer (most visibly, the signing batch spec — see [e2e-watchrule-cross-spec-interference.md](e2e-watchrule-cross-spec-interference.md)). +Once the flag was on, `skipCleanupBecauseResourcesArePreserved` returned `true` for every cleanup call until the process exited. That meant a single failure poisoned the rest of the run: every spec that ran afterward leaked its WatchRules, GitTargets, ConfigMaps, and namespaces. Those leftovers then interfered with subsequent specs through the controller's own informer (most visibly, the signing batch spec — see [e2e-watchrule-cross-spec-interference.md](../design/e2e-watchrule-cross-spec-interference.md)). ## Vision and design choice @@ -128,7 +128,7 @@ CI runs always pick up event dumps and controller logs from `dumpFailureDiagnost ## What this does not solve -This change does not fix the underlying ghost-WatchRule bug in the controller (described in [e2e-watchrule-cross-spec-interference.md](e2e-watchrule-cross-spec-interference.md) §"Open root causes" #1). The manager create/delete specs would still occasionally see ghost commits from prior specs that ran inside the same describe block, because those specs share the same `managerRepo` and namespace. Those need either: +This change does not fix the underlying ghost-WatchRule bug in the controller (described in [e2e-watchrule-cross-spec-interference.md](../design/e2e-watchrule-cross-spec-interference.md) §"Open root causes" #1). The manager create/delete specs would still occasionally see ghost commits from prior specs that ran inside the same describe block, because those specs share the same `managerRepo` and namespace. Those need either: - the controller-side fix (drain event-router state on WatchRule deletion), or - a unique-namespace-per-spec restructuring of the Manager describe block. diff --git a/docs/design/e2e-runtime-state-vs-stamps.md b/docs/finished/e2e-runtime-state-vs-stamps.md similarity index 100% rename from docs/design/e2e-runtime-state-vs-stamps.md rename to docs/finished/e2e-runtime-state-vs-stamps.md diff --git a/docs/design/e2e-test-review.md b/docs/finished/e2e-test-review.md similarity index 99% rename from docs/design/e2e-test-review.md rename to docs/finished/e2e-test-review.md index 7dea1ab06..33045eb4f 100644 --- a/docs/design/e2e-test-review.md +++ b/docs/finished/e2e-test-review.md @@ -1,6 +1,6 @@ # E2E Test Review: Gomega/Ginkgo Implementation -Related investigation: [E2E Full-Suite Shared-State Investigation](./e2e-full-suite-shared-state-investigation.md) +Related investigation: [E2E Full-Suite Shared-State Investigation](../design/e2e-full-suite-shared-state-investigation.md) ## What's Good diff --git a/docs/git-author.md b/docs/finished/git-author.md similarity index 100% rename from docs/git-author.md rename to docs/finished/git-author.md diff --git a/docs/finished/gittarget-isolation-on-rule-change.md b/docs/finished/gittarget-isolation-on-rule-change.md new file mode 100644 index 000000000..e905ffe3f --- /dev/null +++ b/docs/finished/gittarget-isolation-on-rule-change.md @@ -0,0 +1,422 @@ +# GitTarget Isolation on Rule Changes + +> **Status: ✅ Implemented and shipped.** Snapshot selection is now driven purely +> by a per-target *effective watch plan* hash (resolved GVR + scope + unioned +> operations + destination); the global `force` flag is gone. Validated by unit +> tests, a non-Serial e2e regression, and a `--procs=4` smoke run (21/21). See +> the Rollout Plan below for the per-step landing notes. The sections before it +> are kept as the original design rationale. + +## Problem + +`GitTarget` should be a user-facing isolation boundary: rules for one target +should not change the commit behavior of another target. + +The parallel e2e smoke run exposed a cross-target coupling: + +- `Manager WatchRule ConfigMap and Secret should create Git commit when + ConfigMap is added via WatchRule` created a ConfigMap and expected an event + commit such as `[CREATE] v1/configmaps/...`. +- Another spec changed WatchRules/GVRs at roughly the same time. +- The manager target entered rule-change snapshot mode even though its own + user-visible rule set had not changed. +- The ConfigMap was written correctly, but the latest commit was + `reconcile: sync 2 resources`, not `[CREATE] ...`. + +That is more than a brittle test assertion. It means unrelated GitTargets can +affect each other's audit/commit semantics. + +## Why It Matters + +GitTarget isolation keeps the system predictable: + +- Auditability: an event commit preserves operation, resource path, and author + intent. A snapshot commit is less specific. +- Multi-tenancy: one team or namespace changing rules should not perturb another + target's repo history. +- Testability: parallel e2e should reveal real isolation problems, not require + broad `Serial` markings to hide them. +- Operational clarity: rule-change behavior is easier to reason about when it is + scoped to the target whose effective watch plan changed. + +## Current Behavior + +Per-target snapshot tracking **already exists**. `currentRuleSetSnapshots` +(`internal/watch/manager.go`) computes a per-GitTarget hash, and +`snapshotTargetsNeedingDelivery` already compares each target's current hash +against its last-delivered hash and skips targets whose hash is unchanged. So +the machinery for target-scoped snapshots is in place. + +The coupling comes from a single global override. `ReconcileForRuleChange` +computes global GVR additions/removals and passes them as a `force` flag: + +```go +targets := m.snapshotTargetsNeedingDelivery(len(added) > 0 || len(removed) > 0) +``` + +`added`/`removed` are *global* deltas. When `force=true`, +`snapshotTargetsNeedingDelivery` **bypasses the per-target hash comparison** and +selects every current GitTarget — even targets whose effective rule set did not +change. That one wire is the cross-target coupling. + +There is a second, subtler reason the global `force` exists, and it constrains +the fix. The per-target hash is computed over the **rule text** +(`rule.ResourceRules`, source, provider, branch, path) — *not* over the +**resolved GVR set**. So if target A has a wildcard rule and a newly installed +CRD makes that wildcard match a new served resource, A's *rule text* is +unchanged, A's hash is unchanged, and without `force` A would never snapshot — +letting the new informer's initial `ADDED` events leak as individual `[CREATE]` +commits. The global `force` was masking this by snapshotting everyone. + +Shared informers are fine. The coupling is purely that global informer churn is +wired into per-target snapshot selection as a `force` override, and that the +per-target hash is not yet sensitive to the resolved watch set. + +## Desired Behavior + +Only GitTargets whose effective watched resource set changed should receive a +rule-change snapshot. + +Examples: + +- If target A watches ConfigMaps in namespace A, and target B adds a signing + WatchRule, target A should keep processing live events normally. +- If target A's own WatchRule changes, target A should snapshot. +- If target A has a wildcard rule and a newly installed CRD makes that wildcard + match a new served resource, target A should snapshot. +- If a global informer has to start because target B now needs a GVR, only + targets that depend on that new watch should be protected from its initial + ADDED events. + +## Component Overview + +```mermaid +flowchart TD + Controllers["WatchRule / ClusterWatchRule controllers
(reconcile on rule CRD create/update/delete)"] + Controllers -->|ReconcileForRuleChange| Manager + + subgraph Manager["watch.Manager (manager.go)"] + direction TB + RuleStore["RuleStore
effective rules per GitTarget"] + Catalog["RefreshAPIResourceCatalog
what the cluster serves"] + Resolver["rule_gvr_resolver +
computeRequestedGVRs()"] + RuleStore --> Resolver + Catalog --> Resolver + + Resolver --> Global["GLOBAL informer lifecycle
compareGVRs(added, removed)
start/stopInformer per GVR
(SHARED across targets)"] + Resolver --> Scoped["TARGET-SCOPED snapshot selection
snapshotTargetsNeedingDelivery(force)
currentRuleSetSnapshots()
per-target hash"] + + Global -.->|"len(added)+len(removed) > 0
=> force=true ⟵ BUG"| Scoped + Global --> Active["activeInformers map"] + Scoped --> Targets["targets[] needing snapshot"] + end + + Active -->|"ADDED / UPDATED / DELETED events"| Router + Targets -->|"Begin/CompleteReconciliationForStream
+ RequestRepoState / RequestClusterState"| Router + + subgraph Router["EventRouter (event_router.go)"] + States["per-GitTarget stream state machine
LIVE_PROCESSING ⇄ RECONCILING (buffers events)"] + end + + Router -->|ProcessControlEvent| Reconciler + + subgraph Reconciler["ReconcilerManager → FolderReconciler"] + Commit["per GitTarget → branch worker → Git commit
[CREATE] event commit  vs  'reconcile: sync N resources' snapshot"] + end +``` + +Key takeaway: informers are legitimately **global/shared** (left branch); +snapshot selection is already **per-target** (right branch). The bug is purely +that the global left branch (`added`/`removed`) is wired into the right branch +as a `force` override. The fix is to cut that wire and make the per-target hash +sensitive to the *resolved* watch set so the right branch can stand on its own. + +## How the Data Structures Work Together + +The behavior above is the product of four data structures that are computed +fresh (or compared) on every `ReconcileForRuleChange`. Understanding where each +one comes from makes it clear why the coupling exists and what the fix touches. + +**1. The rule set — `RuleStore` (`internal/rulestore`).** +The controllers for `WatchRule` and `ClusterWatchRule` write the user's rules +into a shared `RuleStore`. It is the single source of *intent*: "target T wants +to watch these resources, write them to this provider/branch/path." Everything +downstream is derived from `RuleStore.SnapshotWatchRules()` and +`SnapshotClusterWatchRules()`. + +**2. What the cluster actually serves — `APIResourceCatalog`.** +`RefreshAPIResourceCatalog` queries discovery and records which GVRs are served, +listable, and watchable. Rules can name wildcards or resources that do not exist +yet, so intent (1) must be intersected with reality (2) before anything can be +watched. + +**3. The desired GVR set — `computeRequestedGVRs()`.** +This is the *global* join of (1) and (2): rules resolved against the catalog, +producing the flat set of GVRs that informers must cover across all targets. +`compareGVRs` diffs this against `activeInformers` to produce `added`/`removed`. +This drives informer lifecycle and is deliberately **target-agnostic** — +informers are shared, so it does not matter *which* target needs a GVR, only +that *some* target does. + +``` +RuleStore (intent) ─┐ + ├─► computeRequestedGVRs() ─► added/removed ─► start/stopInformer +APIResourceCatalog ─┘ (global, shared) +``` + +**4. The per-target snapshot ledger — `currentRuleSetSnapshots()` + +`lastDeliveredRuleSetHash` / `pendingRuleSetHash`.** +This is the target-scoped half. `currentRuleSetSnapshots()` walks the same +`RuleStore`, but instead of flattening everything into one GVR set it groups +entries **by GitTarget** (`gitDest.Key()`) and hashes each group. Today it hashes +the raw rule text; after the fix it hashes the target's *effective watch plan* +(resolved GVR + scope + operations + destination — see Section 2): + +``` +RuleStore ─► currentRuleSetSnapshots() ─► []ruleSetSnapshotTarget{gitDest, hash} + │ + snapshotTargetsNeedingDelivery(...) compares each target's + hash against lastDeliveredRuleSetHash[key]: + hash unchanged → skip (target already in sync) + hash changed → select, record in pendingRuleSetHash[key] +``` + +Two maps keyed by `gitDest.Key()` hold the ledger state across reconciles: + +- `lastDeliveredRuleSetHash[key]` — the hash whose snapshot was last successfully + emitted for that target. Updated by `markRuleSetSnapshotDelivered` only after + the snapshot actually reaches a reconciler. +- `pendingRuleSetHash[key]` — targets selected this pass but not yet confirmed + delivered. (The unit tests read this map to observe *which* targets were + selected, without wiring a full `EventRouter`.) + +**Where the two halves meet — and where it goes wrong.** +`ReconcileForRuleChange` is the only place that uses both halves, and it crosses +the wires here: + +```go +targets := m.snapshotTargetsNeedingDelivery(len(added) > 0 || len(removed) > 0) +``` + +The argument is the *global* GVR delta from structure (3); the function it feeds +is the *per-target* ledger from structure (4). When `force=true`, the +per-target hash comparison is skipped entirely and **every** target is selected. +So a global informer change — even one caused by an unrelated target — overrides +the otherwise-correct per-target isolation. + +The second flaw is upstream of the ledger: structure (4) hashes only **rule +text**, never the **resolved GVR** from structure (3). So a wildcard rule whose +*resolved* set grows (a new CRD appears) does not change the target's hash. The +`force` flag was the crude compensation for that blind spot. The fix is to feed +the resolved GVR into structure (4)'s hash so the ledger sees real watch-surface +changes, after which structure (3)'s global delta no longer needs to override +it. + +## Proposed Approach + +### 1. Test the Invariant First — ✅ done + +The invariant is pinned in `internal/watch/rule_change_snapshot_test.go`. The +first test was written **red against the old `force` behavior** and is now green +under the fix: + +- `TestReconcileForRuleChange_UnrelatedTargetNotSnapshotted_OnGlobalGVRChurn` — + two GitTargets A and B both watch ConfigMaps, both in steady state. Only B's + effective plan changes (it starts watching secrets) while unrelated global + informer churn happens concurrently (a stale GVR is dropped). It asserts B is + selected *and* A is not. This failed against the global `force` flag and passes + now that selection is purely per-target. +- `TestSnapshotTargetsNeedingDelivery_PerTargetHashIsolatesTargets` — exercises + the selection function directly and confirms only the changed target is + returned. +- `TestReconcileForRuleChange_RedundantDuplicateRule_DoesNotSnapshot` — the flip + side: a duplicate rule resolving to the same surface does not snapshot (what + dropping source identity buys). +- `TestReconcileForRuleChange_TargetGainsAlreadyWatchedGVR_Snapshots` — issue + #146 under the new model: a target gaining coverage of an already-watched GVR + still snapshots, with no informer churn. +- `TestCurrentRuleSetSnapshots_NamespacedWatchRulePlanByNamespace` — the + namespace dimension of the plan hash for namespaced WatchRules. + +How A's selection is observed without a full `EventRouter`: +`snapshotTargetsNeedingDelivery` records every selected target in +`pendingRuleSetHash` before emission, and with `EventRouter == nil` the emit +path is a guarded no-op that never clears it — so `pendingRuleSetHash` is a +faithful record of who was selected (see structure 4 above). + +### 2. Make the Per-Target Hash Reflect the Resolved Watch Set + +This is the core change. `currentRuleSetSnapshots` already produces a per-target +hash, but it hashes the **rule text** verbatim — the raw `apiGroups` / +`apiVersions` / `resources` patterns, scope, operations, destination, *and* the +source rule's namespace/name. The fix is to hash the target's **effective watch +plan** — what it actually watches after rule resolution and API discovery — not +what the rule literally says: + +```go +type targetWatchPlan struct { + GitDest types.ResourceReference + Entries []targetWatchPlanEntry + Hash uint64 +} +``` + +Not all of the rule text is equal. Split each field by whether GVR resolution +already captures it: + +| Rule field | In the plan hash? | Why | +|---|---|---| +| `apiGroups` / `apiVersions` / `resources` (incl. wildcards) | **Replace** with the resolved GVR set | These are *inputs* to resolution; the resolved GVR is the *output*, intersected with the catalog. The resolved set is strictly more accurate — it catches wildcard-meets-new-CRD, which the raw patterns miss. Hashing the raw patterns is exactly the blind spot that forced `force` to exist. | +| `scope` (namespaced/cluster) | **Replace** with resolved namespace scope | An input to namespace resolution; the resolved scope subsumes it. | +| `operations` | **Keep** | Not part of GVR resolution. Changes *which events* become commits (the existing narrowing test depends on an operations-only change still emitting a snapshot). | +| destination: `provider` / `branch` / `path` | **Keep** | Not part of GVR resolution. Changes *where* writes land. | +| source rule identity (namespace/name) | **Drop** | Pure identity — it does not change the effective watch surface. A redundant duplicate rule resolving to the same plan should not trigger a no-op snapshot. | + +So a `targetWatchPlanEntry` is built from the *resolved* watch surface plus the +non-resolvable fields that still affect writes: + +- **resolved GVR** (the missing ingredient today) +- resolved namespace / cluster scope +- operations +- destination details that affect writes: provider, branch, and path + +When more than one rule for the same target resolves to the same GVR, the plan +entry's `operations` is the **union** across those rules — operations add up, +there is no first-wins precedence. This matches how matching already works: +`RuleStore.GetMatchingRules` evaluates per-operation and returns every rule whose +operation set covers it, so a resource covered by rule A=`[CREATE]` and rule +B=`[UPDATE]` is effectively watched for both. The plan builder must preserve that +union rather than letting one rule's operations shadow another's. This invariant +is pinned by `TestGetMatchingRules_OverlappingRulesUnionOperations` in +`internal/rulestore/store_test.go`. + +Then: + +- The union of all target plans determines which informers must run (this is + effectively what `computeRequestedGVRs` already does — see Open Questions on + whether to wrap or replace it). +- The per-target plan hash determines which GitTargets need snapshots. +- Global GVR churn no longer forces unrelated targets into snapshot mode. + +Why the resolved GVR matters: once it is in the hash, the wildcard-meets-new-CRD +case (target A's effective set grows even though its rule text is unchanged) +naturally changes A's hash and selects A — which is exactly the case that breaks +if you only remove the `force` flag without enriching the hash. This is what +lets the global `force` go away safely. + +This should simplify the mental model: one resolved plan drives both informer +lifecycle and snapshot decisions. + +### 3. Keep Informers Global, But Snapshots Target-Scoped + +Informer management can remain global and shared: + +- start informers needed by the union of all target plans +- stop informers no longer needed by any target + +Snapshot management should be target-scoped: + +- call `BeginReconciliationForStream` +- emit `RequestRepoState` +- emit `RequestClusterState` +- call `CompleteReconciliationForStream` + +only for targets whose effective plan hash changed, or targets directly affected +by a newly available resolved resource. + +### 4. Preserve Protection Against Initial ADDED Events + +The current snapshot path exists for a good reason. When a new informer starts, +its cache emits initial ADDED events. Those should not become a burst of +individual `[CREATE]` commits. + +The refined rule should be: + +- if a newly started informer is relevant to target A, buffer/snapshot target A +- if that informer is irrelevant to target B, do not move target B into + `RECONCILING` + +That preserves safety without globally disturbing every target. + +Note that if Section 2 is done — the per-target hash includes resolved GVRs — +this protection mostly falls out for free and may not need a separate +informer→target index. A new informer only starts because some target's resolved +GVR set grew; that target's hash therefore changes and it is already selected for +a snapshot. The hash diff and the "which targets depend on this new informer" +question have the same answer. Prefer the resolved-plan-hash mechanism over +maintaining a second informer→target mapping; treat the two as alternatives, not +complementary layers. + +### 5. Add an E2E Regression + +After unit coverage proves the target selection logic, add or adapt an e2e +regression: + +- run two non-Serial targets concurrently +- trigger a rule change for target B +- create a ConfigMap for target A +- assert target A still receives the expected event commit, not a snapshot commit + +This should be smaller and more intentional than relying on incidental +cross-spec timing in the full smoke suite. + +## Non-Goals + +- Do not make every manager spec `Serial` as the primary fix. That hides the + product coupling. +- Do not remove shared informers. Sharing informers is efficient and not the + core problem. +- Do not weaken event-commit assertions unless the product decision is that + snapshot commits are acceptable for unrelated rule changes. + +## Rollout Plan + +1. ✅ Done — `rule_change_snapshot_test.go` has a two-target isolation test plus + a control test; alongside them + `TestReconcileForRuleChange_TargetGainsAlreadyWatchedGVR_Snapshots` (issue + #146 under the new model) and + `TestCurrentRuleSetSnapshots_NamespacedWatchRulePlanByNamespace` (namespace + dimension) pin the surrounding contract. +2. ✅ Done — `currentRuleSetSnapshots` now hashes the effective watch plan via a + `targetWatchPlan`: it **replaces** the raw resource-matching patterns and + scope with the resolved GVR + namespace scope, **keeps** operations + (unioned across rules per GVR) and the destination (provider/branch/path), + and **drops** source rule identity. A target with no resolvable rules has an + empty plan and is omitted. +3. ✅ Done — `snapshotTargetsNeedingDelivery` no longer takes a `force` argument; + selection relies solely on per-target hash diffs. +4. ✅ Done — the new-informer ADDED-burst protection falls out of the enriched + hash (a new informer only starts when some target's resolved GVR set grew, so + that target's hash changes and it is selected). No separate informer→target + index was needed. +5. ✅ Done — `test/e2e/gittarget_isolation_e2e_test.go` ("Manager GitTarget + Isolation", non-Serial, smoke) runs two GitTargets in one repo at separate + paths, repeatedly churns target B's rule set while creating ConfigMaps for + target A, and asserts target A's path-scoped commit stays a `[CREATE]` event + commit (never `reconcile: sync`). +6. ✅ `task fmt`, `task vet`, `task lint`, `task test`, `task generate`, and + `task manifests` all pass with no drift. `task test-e2e` (smoke) green at the + default 2 procs (20/20) and at `--procs=4` (21/21, isolation spec included and + passing under real parallel contention — the scenario that originally flaked). + +## Open Questions + +- ~~Should path/provider/branch changes be part of the same watch-plan hash?~~ + They already are (`currentRuleSetSnapshots` hashes provider/branch/path today), + and they should stay: changing the write destination genuinely requires a + fresh snapshot to populate the new branch/path. Keep them in the hash. +- For removed rules, should the affected target always snapshot to remove stale + files, or should deletion cleanup be handled by a more explicit target-state + transition? +- Should the resolved-GVR plan builder replace `computeRequestedGVRs` / + `currentRuleSetSnapshots`, or wrap them? Recommendation: wrap initially. + `computeRequestedGVRs` drives the tested informer lifecycle; derive the + per-target resolved entries alongside it and feed them into the hash rather + than rewriting the resolver, to keep the change low-risk. + +## Recommendation + +Work on this before further e2e speed work. Parallel e2e uncovered a user-facing +isolation issue: unrelated GitTargets can change each other's commit semantics. +The cleanest end state is a per-target effective watch plan that drives both +global informer lifecycle and target-scoped snapshot decisions. diff --git a/docs/future/idea-audit-enrichment-side-channel.md b/docs/finished/idea-audit-enrichment-side-channel.md similarity index 100% rename from docs/future/idea-audit-enrichment-side-channel.md rename to docs/finished/idea-audit-enrichment-side-channel.md diff --git a/docs/design/partial-object-audit-event-handling.md b/docs/finished/partial-object-audit-event-handling.md similarity index 100% rename from docs/design/partial-object-audit-event-handling.md rename to docs/finished/partial-object-audit-event-handling.md diff --git a/docs/design/path-scoped-bootstrap-template-design.md b/docs/finished/path-scoped-bootstrap-template-design.md similarity index 100% rename from docs/design/path-scoped-bootstrap-template-design.md rename to docs/finished/path-scoped-bootstrap-template-design.md diff --git a/docs/design/remove-gittarget-admission-webhook.md b/docs/finished/remove-gittarget-admission-webhook.md similarity index 100% rename from docs/design/remove-gittarget-admission-webhook.md rename to docs/finished/remove-gittarget-admission-webhook.md diff --git a/docs/finished/rule-set-snapshot-discovery-lag-fix.md b/docs/finished/rule-set-snapshot-discovery-lag-fix.md new file mode 100644 index 000000000..95923ef7a --- /dev/null +++ b/docs/finished/rule-set-snapshot-discovery-lag-fix.md @@ -0,0 +1,237 @@ +# Rule-Set Snapshot Discovery-Lag Fix + +> Status: **shipped**. Drives the per-GitTarget effective-watch-plan hash in +> `internal/watch/manager.go`. + +## Problem + +This fix introduced the per-GitTarget effective-watch-plan hash: + +- `currentRuleSetSnapshots()` resolves rules through the API resource catalog. +- It ignores `ResolveMiss` values from `resolver.Resolve(...)` + (`gvrs, _ := resolver.Resolve(...)`). +- If a target's rules temporarily resolve to zero GVRs, the target is omitted + entirely ([manager.go:1280-1282](../../internal/watch/manager.go#L1280-L1282)). +- `snapshotTargetsNeedingDelivery()` treats an omitted target as "this target no + longer has rules" and deletes `lastDeliveredRuleSetHash` + ([manager.go:1189-1194](../../internal/watch/manager.go#L1189-L1194)). +- When discovery recovers, the same target appears again with the same real + watch plan but no remembered baseline, so it gets a full rule-change snapshot + even though the user's rules did not change. + +That makes a transient discovery gap look like a real watch-plan change. It can +hide event commits behind later `reconcile: sync ...` commits and is the reason +`crd_lifecycle` still needs `Serial` protection. + +## Two facts that shape the fix + +Before designing anything, two existing properties narrow what the fix actually +has to do: + +1. **Partial snapshots are already refused downstream.** + `RequestClusterState` aborts whenever `blockingSnapshotMisses(...)` is + non-empty — "refusing to snapshot a partial cluster view" + ([manager.go:596-601](../../internal/watch/manager.go#L596-L601)). So a + half-resolved plan can never *emit* a snapshot. The selection layer does not + need to be perfect about blocked plans for **correctness** — only to avoid + noisy select-then-abort retry loops. + +2. **"No rules" vs "rules that didn't resolve" needs no miss classification.** + It is already encoded in the RuleStore. `currentRuleSetSnapshots()` only + builds a plan for a target that appears in `SnapshotWatchRules()` / + `SnapshotClusterWatchRules()`. A target with zero rules never gets a plan and + falls out of `currentKeys` naturally. The bug exists solely because the + `if len(p.entries) == 0 { continue }` guard drops a target that *still has + rules* but momentarily resolved to nothing. + +Together these mean the headline bug has a small, safe fix, and the richer +classification is a set of optional refinements layered on top. + +## Expected states — and when they happen + +A target's resolved plan can be in one of four states. The two that produce zero +GVRs (states 3 and 4) look identical to the old code, which is the root cause. + +| State | Trigger | Stable? | Action | +|---|---|---|---| +| 1. No rules | The last WatchRule/ClusterWatchRule for the GitTarget was deleted, or its `gitTargetRef` was repointed. | — | Prune remembered state. | +| 2. Resolved plan | One or more rules resolve to served, allowed GVRs. The normal steady state. | yes | Compare hash to last settled; emit a snapshot only if it changed. | +| 3. Stable empty plan | Rules exist but resolve to zero watchable GVRs for a reason that will not change on its own: `NotServed`, `Ambiguous`, `Disallowed`. | yes | No snapshot (nothing to watch). Optionally record a no-watch settled hash (Refinement B). | +| 4. Blocked / uncertain | Rules exist and *would* resolve, but discovery can't answer authoritatively right now: `CatalogUnavailable`, `DiscoveryDegraded`, `WildcardGroup`, `WildcardResource`. | no | Do not snapshot, do not mutate remembered state. Retry next cycle. | + +Concrete triggers: + +- **State 1 — No rules.** `kubectl delete watchrule`, or editing a rule to point + at a different GitTarget. The target no longer exists in the RuleStore + snapshot, so no plan is built for it. +- **State 2 — Resolved plan.** Steady state, every reconcile. A real change is + e.g. a rule that newly adds `secrets` alongside `configmaps`, or a wildcard + that begins matching a freshly installed CRD. +- **State 3 — Stable empty.** The user wrote a rule for a CRD that was never + installed, typo'd a resource name, omitted `apiGroups` in a way that matches + multiple groups, or named a resource the operator is configured to refuse. + Re-running discovery returns the same answer, so this is settled. +- **State 4 — Blocked.** **This is the bug's trigger.** A CRD is mid + install/delete (the `crd_lifecycle` spec), the controller just started and + discovery is still warming up, or the aggregation layer briefly 503s. Unlike + state 3, the answer **changes on retry**. + +The crucial pair is **3 vs 4**: both yield zero GVRs, but 3 is a settled fact and +4 is a temporary "ask again later." The original code collapsed both into "target +removed," which is what turned a routine CRD install (state 4) into a spurious +full re-snapshot of every unrelated target. + +## Fix (ship this) + +The headline bug is fixed by a near-trivial change. No new types, no miss +classification, no rename. + +1. In `currentRuleSetSnapshots()`, **stop dropping empty-entry plans.** Keep + every target that has at least one rule, even if it currently resolves to zero + GVRs. (Delete the `if len(p.entries) == 0 { continue }` guard.) +2. In `snapshotTargetsNeedingDelivery()`, **if a plan has no entries, `continue` + — skip selection and leave `lastDeliveredRuleSetHash` untouched.** There is + nothing to snapshot, and the remembered baseline must survive the gap. +3. The existing prune loop now fires only for targets with genuinely zero rules. + That is already correct — no change needed. + +Walk the transient-gap case through it: `valid -> (gap, empty) -> valid`. During +the gap the target is kept but skipped, `lastDeliveredRuleSetHash` stays at the +valid hash; on recovery the hash matches, so no snapshot is emitted. Bug gone. + +This alone fixes finding #2 and unblocks de-serializing `crd_lifecycle`. + +### What the Fix gives up + +Exactly **one** behavioral difference versus a fully classified design: the +`valid -> stable-empty -> valid` *edit* sequence (a user edits a rule to +something non-watchable, then edits it back). The Fix keeps the baseline at the +valid hash the whole time, so on return it sees "unchanged" and skips the +recovery snapshot — even though no live events were watched during the empty +window. Refinement B below addresses exactly this case if you want it. + +## Refinement A (optional): skip blocked plans instead of aborting them + +The Fix is safe for the **mixed** case (one resolved GVR plus one blocking miss → +non-empty entries) because the downstream abort at +[manager.go:596](../../internal/watch/manager.go#L596) refuses the partial +snapshot. But it does so by *selecting* the target every cycle and aborting it, +which logs an error each reconcile until discovery recovers. + +To avoid that noise, capture misses in the plan and add a `blocked` flag: + +```go +// fields added to the existing targetWatchPlan, not a parallel type +blocked bool +blockingMisses []ResolveMiss +``` + +Set `blocked` when any rule for the target produces a miss in +`blockingSnapshotMisses(...)` (`CatalogUnavailable`, `DiscoveryDegraded`, +`WildcardGroup`, `WildcardResource`). Then in `snapshotTargetsNeedingDelivery()`: + +```go +if plan.blocked { + // Rules exist, but the current resolved plan is not authoritative. + // Keep lastDeliveredRuleSetHash and pendingRuleSetHash as-is; + // discovery/catalog recovery will retry. + continue +} +``` + +This is a quality-of-noise optimization, not a correctness fix — the Fix is +already safe without it. + +## Refinement B (optional): settle a stable-empty hash + +This is the *only* mechanism that fixes the `valid -> stable-empty -> valid` edit +case, and the only reason to compute a hash for an empty plan. + +Include the **stable** misses (`NotServed`, `Ambiguous`, `Disallowed`) in the +hash so a target can settle into a known "no-watch" plan and later transition +back to a resolved plan: + +``` +valid hash -> stable empty hash -> valid hash +``` + +The settled empty hash differs from the valid hash, so the final transition is +detected and the target is re-snapshotted. In `snapshotTargetsNeedingDelivery()`: + +```go +if !plan.hasEntries { // hasEntries == len(plan.entries) > 0 + // Stable empty/no-watch plan. No cluster snapshot to emit, but the + // rule-set state did advance. Mark it settled so a later empty -> resolved + // transition emits a snapshot. + lastDeliveredRuleSetHash[key] = plan.hash + delete(pendingRuleSetHash, key) + continue +} +``` + +Do **not** include blocking misses in the hash — a blocked plan is not an +authoritative plan, and a blocked target must hit the Refinement A path +(`continue` without mutating state) before this branch. + +When is this worth it? Only when a *served* resource churns during a window in +which the user deliberately edited the rule to be non-watchable and back. Rare, +and the worst case without it is a missed recovery snapshot that self-heals on +the next real rule change. For `NotServed` resources there are by definition no +objects to miss, so it is often moot. + +## Implementation notes + +- Extend the existing `targetWatchPlan` with the `blocked` / `blockingMisses` + fields rather than introducing a parallel `ruleSetSnapshotPlan` type. Express + `hasEntries` as `len(entries) > 0`, not a stored field. +- Reuse `blockingSnapshotMisses(...)` for the classification. If the name reads + oddly in a planning context, extract a thin alias such as + `blockingRuleSetPlanMisses(...)`. +- If Refinement B lands, `lastDeliveredRuleSetHash` would be more accurately + named `lastSettledRuleSetHash`, because stable empty plans are settled without + a delivered snapshot. Rename only if the patch stays readable; otherwise add a + comment explaining the broadened meaning. +- Log blocked plans at `V(1)` with `FormatResolveMisses(...)`. Avoid + default-verbosity logs on every periodic reconcile; catalog transition logs + already cover the noisy discovery edges. +- Do not mark a blocked plan as pending. Pending is for a snapshot that should be + emitted once a reconciler exists, not for a plan that is not yet authoritative. + +## Test plan + +Tests map onto the tiers, so you can land the Fix and its three tests and drop +`Serial` without ever building Refinement A or B. Add them near +[rule_change_snapshot_test.go](../../internal/watch/rule_change_snapshot_test.go): + +**Fix:** + +- `TestSnapshotTargets_DiscoveryUnavailableKeepsDeliveredHash`: seed a delivered + configmaps plan, swap the manager catalog to unavailable, call + `snapshotTargetsNeedingDelivery()`, and assert the target is not selected and + `lastDeliveredRuleSetHash` is not pruned. +- `TestSnapshotTargets_DiscoveryRecoveryDoesNotResnapshotUnchangedPlan`: after + the unavailable-catalog pass above, restore the common catalog and assert the + unchanged target is still not selected. +- `TestSnapshotTargets_RuleRemovalPrunesDeliveredHash`: remove the rule from the + RuleStore and assert the remembered hash is pruned. This guards against + confusing "blocked" (state 4) with "deleted" (state 1). + +**Refinement A:** + +- `TestSnapshotTargets_MixedResolvedAndBlockingMissDoesNotSnapshotPartialPlan`: + give one target a resolved GVR plus a discovery-degraded lookup. Assert the + target is not selected and no hash is advanced. +- `TestSnapshotTargets_BlockedTargetDoesNotAffectOtherTargets`: target A is + blocked by discovery, target B changes from configmaps to configmaps+secrets. + Assert B is selected and A's remembered state is preserved. + +**Refinement B:** + +- `TestSnapshotTargets_StableEmptyPlanSettlesAndRecoverySnapshots`: seed a + delivered configmaps plan, change the rule to a definitely not-served resource, + assert no snapshot is selected but the settled hash changes, then change back + to configmaps and assert the target is selected. + +The e2e follow-up is to remove `Serial` from `Manager CRD Lifecycle` only after +these unit tests pass and at least one parallel CI confidence window shows that +CRD install/delete no longer drags unrelated GitTargets into snapshots. diff --git a/docs/design/shallow-audit-event-misclassification.md b/docs/finished/shallow-audit-event-misclassification.md similarity index 100% rename from docs/design/shallow-audit-event-misclassification.md rename to docs/finished/shallow-audit-event-misclassification.md diff --git a/docs/design/watch-gvr-resolution-cleanup.md b/docs/finished/watch-gvr-resolution-cleanup.md similarity index 100% rename from docs/design/watch-gvr-resolution-cleanup.md rename to docs/finished/watch-gvr-resolution-cleanup.md diff --git a/docs/design/watchrule-gvr-resolution-plan.md b/docs/finished/watchrule-gvr-resolution-plan.md similarity index 99% rename from docs/design/watchrule-gvr-resolution-plan.md rename to docs/finished/watchrule-gvr-resolution-plan.md index 2c31b7f57..eba04e8a7 100644 --- a/docs/design/watchrule-gvr-resolution-plan.md +++ b/docs/finished/watchrule-gvr-resolution-plan.md @@ -147,7 +147,7 @@ its own discovery lookup or applying its own core `/v1` defaults. The detailed discovery model, refresh triggers, trust rules, and streaming-list note live in -[`kubernetes-api-resource-catalog.md`](./kubernetes-api-resource-catalog.md). +[`kubernetes-api-resource-catalog.md`](../design/kubernetes-api-resource-catalog.md). ### Catalog responsibilities diff --git a/docs/future/design-commit-request-phase-2.md b/docs/future/design-commit-request-phase-2.md index e277b039c..d15918271 100644 --- a/docs/future/design-commit-request-phase-2.md +++ b/docs/future/design-commit-request-phase-2.md @@ -1,6 +1,6 @@ # Phase 2: `CommitRequest` — deferred work -> Status: proposed. Follow-up to [design-commit-request-api.md](design-commit-request-api.md). +> Status: proposed. Follow-up to [design-commit-request-api.md](../finished/design-commit-request-api.md). > Date: 2026-05-18 The first cut of `CommitRequest` is intentionally minimal. This doc tracks what was deliberately diff --git a/docs/future/design-snapshot-engine-evolution.md b/docs/future/design-snapshot-engine-evolution.md index 776e8ec18..7c16c0d4d 100644 --- a/docs/future/design-snapshot-engine-evolution.md +++ b/docs/future/design-snapshot-engine-evolution.md @@ -1,7 +1,7 @@ # Design: snapshot engine evolution > Status: proposed — continuation of -> [design-rule-change-snapshot-trigger.md](design-rule-change-snapshot-trigger.md). +> [design-rule-change-snapshot-trigger.md](../finished/design-rule-change-snapshot-trigger.md). > Date: 2026-05-21 > Author context: written after a critical review of the rule-change snapshot > trigger implementation ([ed3a1db](#) "fix: better reconcile"). @@ -402,7 +402,7 @@ per-target. This removes a whole-cluster re-list amplification on CRD install. ### 1.5 Doc hygiene -[design-rule-change-snapshot-trigger.md](design-rule-change-snapshot-trigger.md) +[design-rule-change-snapshot-trigger.md](../finished/design-rule-change-snapshot-trigger.md) still cites `manager.go:660` for the old early-return; it is now line 696. Fix the stale reference since the doc is marked "implemented". @@ -675,7 +675,7 @@ flowchart LR ## References - Predecessor: - [design-rule-change-snapshot-trigger.md](design-rule-change-snapshot-trigger.md). + [design-rule-change-snapshot-trigger.md](../finished/design-rule-change-snapshot-trigger.md). - API discovery background (discoverability and verbs): [docs/design/kubernetes-api-discovery.md](../design/kubernetes-api-discovery.md). - API server capacity tuning: diff --git a/docs/future/idea-end-user-commit-messages.md b/docs/future/idea-end-user-commit-messages.md index d849d825b..39bf74df0 100644 --- a/docs/future/idea-end-user-commit-messages.md +++ b/docs/future/idea-end-user-commit-messages.md @@ -422,7 +422,7 @@ this comparison underweighted on the first pass. The aggregated API is also futu future temporary actions (`commitNow`, "propose as PR") fit the same shape with no new transports, and the close-off pattern is a clean fit for parallel-write frontends. -The detailed design lives in [design-commit-context-api.md](design-commit-context-api.md). +The detailed design lives in [design-commit-context-api.md](../finished/design-commit-context-api.md). The audit-carried transports remain valuable as a fallback for clusters that already have the infrastructure in place (impersonation in regular use, or OIDC `AuthenticationConfiguration` diff --git a/docs/future/idea-unify-pending-write-kinds.md b/docs/future/idea-unify-pending-write-kinds.md index 058fb7b60..f7c875232 100644 --- a/docs/future/idea-unify-pending-write-kinds.md +++ b/docs/future/idea-unify-pending-write-kinds.md @@ -19,7 +19,7 @@ unset. The temptation is to make that the dispatch and delete the `Kind` enum. ## Use case walkthrough -The four use cases in [commit-window-refactor.md](../design/commit-window-refactor.md#use-cases): +The four use cases in [commit-window-refactor.md](../finished/commit-window-refactor.md#use-cases): 1. **Burst collapse.** Per-event-only behavior (atomic is one commit by definition). Unification neither helps nor breaks this. diff --git a/docs/future/initial-seed-via-watch-list.md b/docs/future/initial-seed-via-watch-list.md new file mode 100644 index 000000000..7cf831e3e --- /dev/null +++ b/docs/future/initial-seed-via-watch-list.md @@ -0,0 +1,241 @@ +# Initial Seed via client-go WatchListClient + +> Status: proposed — not yet implemented +> Scope: `internal/watch` snapshot path, dynamic informer construction, the +> client-go `WatchListClient` feature gate +> Related: [`docs/design/upgrade-finding.md`](../design/upgrade-finding.md), +> [`docs/serious-bug/cozystack-bugreport.md`](../serious-bug/cozystack-bugreport.md) + +## Goal + +Make initial cluster-state seeding flow through a single, cache-backed source of +truth — the dynamic informer cache that `internal/watch` already maintains — +and stop issuing a separate hand-rolled `LIST` per GVR per namespace whenever a +`GitTarget` needs a snapshot. + +Use the client-go `WatchListClient` feature (streaming-list, on by default +since client-go v1.35 / v0.36.x) as the *transport* that hydrates that cache, +rather than something to fight against. The bookmark-warning we observed in +production ([`docs/design/upgrade-finding.md`](../design/upgrade-finding.md)) was a hint that +we had silently opted into streaming-list without redesigning the seed path +around it; the right response is to redesign, not to disable the gate. + +## Assumption + +This plan assumes that every aggregated apiserver we watch implements the +WatchList contract correctly — i.e. honors `SendInitialEvents=true` and emits +the `k8s.io/initial-events-end` bookmark. Standard kube-apiserver has done +this since 1.27. Aggregated APIs are the risk surface; see the +[Aggregated-API caveat](#aggregated-api-caveat) section below. + +If the assumption proves false in a given cluster, the per-GVR fallback in +Phase B keeps the system functional for misbehaving resources without giving up +the benefits on the rest. + +## Today's state — duplicated seeding + +Two seed paths run side-by-side: + +1. **Implicit informer seed.** + [`internal/watch/manager.go:977-981`](../../internal/watch/manager.go#L977-L981) + creates a `dynamicinformer.NewDynamicSharedInformerFactory` per GVR / + namespace. client-go performs its own `LIST` (or, with `WatchListClient=true`, + a streaming-list) and hydrates an in-memory `cache.Store`. Only the + per-event callbacks are wired through `EventRouter`; the populated store is + otherwise unused. + +2. **Explicit by-hand LIST.** + [`GetClusterStateForGitDest`](../../internal/watch/manager.go#L461) → + [`listResourcesForGVR`](../../internal/watch/manager.go#L630-L672) calls + `dc.Resource(gvr).List(...)` again, per matching namespace, against the + apiserver. This fires every time a `GitTarget` is snapshotted — rule + changes, periodic re-snapshot, controller restart — and re-LISTs state that + the informer cache already holds. + +For any GVR that already has a running informer, this is two full +materializations of the same cluster state per snapshot: one cached and +unused, one re-LISTed and thrown away. + +## The decision + +- **Embrace `WatchListClient`.** Keep the client-go feature gate on. Do not + ship the `--disable-watch-list-client` flag that the earlier upgrade-finding + document sketched as a workaround. +- **Snapshots read from the informer cache, not the apiserver.** The + hand-rolled `LIST` path goes away. The dynamic informer becomes the single + source of truth for cluster state, hydrated by streaming-list semantics on + the wire. +- **Treat aggregated-API conformance as a known risk and verify it + explicitly.** See the caveat section; the work item is to audit every + aggregated API we watch before turning this loose in production. + +## Phase A — snapshot from the informer cache + +Pure refactor inside `internal/watch`. No client-go feature-gate change yet. + +- Replace + [`listResourcesForGVR`](../../internal/watch/manager.go#L630-L672) with a + `snapshotFromCache(gvr, namespaces)` helper that calls + `informer.GetStore().List()` (or a `cache.NewGenericLister`-backed `Lister` + so namespace filtering stays cheap). +- Snapshot-time work becomes O(in-memory walk), not O(`LIST` × + namespaces). +- If a snapshot is requested for a GVR with no active informer yet (rule just + added, reconcile hasn't run): start the informer on-demand, + `WaitForCacheSync`, then read the cache. The "initial events" stream *is* + the snapshot, which is exactly the shape streaming-list optimizes for in + Phase B. +- `dynamicClientFromConfig`'s seed-only role at + [`manager.go:309-326`](../../internal/watch/manager.go#L309-L326) goes away. + The informer factory has its own client; the dynamic client only remains + for *informer construction*. +- The "refusing to snapshot a partial cluster view" guard at + [`manager.go:535-540`](../../internal/watch/manager.go#L535-L540) is + preserved — it now triggers on cache-sync timeout or per-GVR informer + failure instead of `LIST` failure. + +Sketch of the seam: + +```go +// Before: re-LISTs the apiserver, per namespace, on every snapshot +list, err := dc.Resource(gvr).Namespace(ns).List(ctx, metav1.ListOptions{}) + +// After: reads the already-hydrated informer cache +objs, err := m.cacheForGVR(gvr).ByNamespace(ns).List(labels.Everything()) +``` + +This phase is independently shippable and is the largest correctness / +performance win. It does not depend on the feature-gate decision; it just +makes Phase B safe to land afterwards. + +## Phase B — turn `WatchListClient` on intentionally + +Once the snapshot path is cache-backed, streaming-list becomes net-positive +in three ways: + +- Cheaper initial sync: one chunked watch instead of a single mega-LIST + response that the apiserver materializes whole in memory. +- Apiserver-side relief, especially for large resources (Secrets, + ConfigMaps). +- The cache is consistent with watch events the moment `HasSynced` flips — + no race window where we re-LIST and miss in-flight deletes. + +Work items: + +- Keep the feature gate on for the standard API surface. Do not introduce a + binary-level kill switch. +- For aggregated APIs that we discover (at startup or runtime) to be + non-conformant: build the informer with a custom `ListWatch` that bypasses + `ToListWatcherWithWatchListSemantics`. That means dropping below + `dynamicinformer.NewDynamicSharedInformerFactory` for those GVRs and + constructing `cache.NewSharedIndexInformer` directly with a plain + `ListWatch`. +- Detection is **per-GVR**, not per-`APIService`. Cozystack proved why: + one aggregated server can host both conformant and non-conformant resources + in different API groups (see the bug report). +- Detection strategy: start the informer with watch-list semantics; if + `HasSynced` doesn't flip within a deadline (e.g. 60 s, with the + missing-bookmark warning as a strong signal), tear it down, fall back to a + plain `ListWatch` for that GVR, and remember the fallback in a process-local + cache so subsequent restarts use the safe path immediately. +- Add an e2e lane that watches at least one aggregated-API resource and + asserts the informer syncs and the snapshot is complete. + +## Phase C — delete the legacy seed code + +After A and B land: + +- `listResourcesForGVR`, the per-namespace LIST fan-out in + `getNamespacesForGVR`, the seed-only dynamic client, and the comments + about "by-hand" seeding all go. +- `GetClusterStateForGitDest` shrinks to roughly: resolve GVRs → ensure + informers running and synced → iterate caches. +- The reconciler-startup story becomes "rules → GVRs → start informers → + snapshot is implicit," which is what the existing design docs were already + pointing toward but never reached. + +## Aggregated-API caveat + +This plan **only works if the aggregated apiservers we watch implement the +WatchList contract correctly**. Specifically, each aggregated `Watch` +implementation must: + +1. Read `options.SendInitialEvents` from the internal `ListOptions`. +2. Deliver all matching existing objects as `ADDED` events. +3. Emit a final `watch.Bookmark` event whose object carries + `metadata.annotations["k8s.io/initial-events-end"] = "true"` and a + `ResourceVersion` equal to the last RV seen during the initial set. +4. Continue with live events thereafter. + +Standard kube-apiserver and resources backed by `genericregistry.Store` get +this for free. Hand-written `rest.Storage` implementations in aggregated +servers — exactly the kind that show up in projects like Cozystack — must +implement it themselves. The Cozystack bug report +([`docs/serious-bug/cozystack-bugreport.md`](../serious-bug/cozystack-bugreport.md)) is a worked +example: one API group conformant, one not, in the same binary. + +Before relying on Phase B in any production cluster, we need to audit every +aggregated API surface present in that cluster. The audit list per cluster is: + +- Enumerate `APIService` objects (`kubectl get apiservices`). +- For each non-local `APIService` we actually watch, confirm one of: + - the backing implementation is `genericregistry.Store`-based (built on + `k8s.io/apiserver`'s default registry — emits the bookmark by + construction), **or** + - the project has explicit `SendInitialEvents` handling in its `Watch` + method (grep for `SendInitialEvents` and + `k8s.io/initial-events-end` in their source), **or** + - the GVR is on our explicit fallback list and gets the plain-ListWatch + informer path from Phase B. + +This audit is not a one-time activity for the project; it's an operational +check the first time we deploy into a cluster with unfamiliar aggregated APIs. +The Phase B runtime fallback exists so a missed audit degrades gracefully +instead of hanging the reflector. + +## Open questions + +- **How aggressively should the Phase B detection probe?** A 60 s deadline is + generous for healthy apiservers but stalls every controller restart by that + amount when an aggregated API is broken. Options: a shorter deadline with a + faster fallback; or a configurable per-GVR override in the `WatchRule` / + `ClusterWatchRule` spec; or a one-time at-startup probe that records + conformance per `APIService`. +- **Where does the fallback memory live?** Process-local is enough for a + single-pod controller, but if we move to leader-elected HA the fallback + decision should arguably be a `Condition` on the `WatchRule` so a new pod + doesn't re-pay the detection cost on every failover. +- **Cache memory cost.** Today the snapshot LIST path holds objects only for + the duration of the snapshot. Cache-backed snapshots hold them for the life + of the informer. For resources with extreme cardinality (e.g. cluster-wide + Pod watches we'd never want), the `WatchRule` matching is already the gate; + this just makes it explicit that adding a rule has a steady-state memory + cost. + +## Out of scope + +- Multi-cluster snapshotting. The dynamic informer is per-cluster; multi- + cluster seed is a separate design. +- Server-side filtering beyond label selectors. `WatchListClient` does not + change the filter contract. +- The audit-ingestion pipeline. Snapshot path and audit path are independent; + this plan does not touch + [`docs/design/audit-metrics-overhaul-plan.md`](../design/audit-metrics-overhaul-plan.md). +- A binary-level "disable WatchListClient" flag. Explicitly rejected by the + decision above. Operators who need to disable the gate in an emergency can + still set `KUBE_FEATURE_WatchListClient=false` via the chart's `.Values.env` + ([`charts/gitops-reverser/templates/deployment.yaml:107-125`](../../charts/gitops-reverser/templates/deployment.yaml#L107-L125)); + we just won't ship that as a first-class flag. + +## Suggested order of work + +1. **Phase A** as one PR — refactor `GetClusterStateForGitDest` onto informer + caches. Self-contained. Largest single win. +2. **Phase B detection + fallback** as a follow-up PR — per-GVR conformance + probe and plain-`ListWatch` fallback. Add an e2e lane that exercises a + known-good aggregated API. +3. **Phase C cleanup** PR — delete the legacy seed code now that nothing + calls it. +4. **Aggregated-API audit checklist** in `docs/` (or an extension of + [`docs/aggregated-api-guide.md`](../aggregated-api-guide.md)) so operators + know what to verify before turning Phase B loose in their cluster. diff --git a/docs/future/watchrule-wildcard-support-plan.md b/docs/future/watchrule-wildcard-support-plan.md new file mode 100644 index 000000000..94d67c6e2 --- /dev/null +++ b/docs/future/watchrule-wildcard-support-plan.md @@ -0,0 +1,192 @@ +# WatchRule Wildcard Support — Finding & Plan + +Status: **proposed** (not started) + +> **Superseded framing:** the finding below stands, but the recommended way to +> resolve it is the unified model in +> [watch-and-catalog-architecture.md](../design/watch-and-catalog-architecture.md). Treat +> this document as the focused finding + a tactical (Phase-0-first) fallback if +> the larger refactor is deferred. + +## Finding: documented behavior contradicts implemented behavior + +The `WatchRule`/`ClusterWatchRule` CRD field docs promise Kubernetes +admission-webhook / RBAC wildcard semantics, but the GVR resolver refuses them. + +The field comments in [watchrule_types.go:71-118](../../api/v1alpha1/watchrule_types.go#L71-L118) +(shipped as the CRD OpenAPI schema → `kubectl explain`) state: + +- "All fields except Resources are optional and default to matching all when not + specified." +- `apiGroups`: "Wildcards supported: `*` matches all groups", "`["*"]` or `[]` + matches all groups". +- `apiVersions`: "If empty, matches all versions", "`*` matches all versions". +- `resources`: "Wildcard semantics follow Kubernetes admission webhook patterns: + `*` matches all resources", "`pods/*` matches all pod subresources", + "`pods/log` matches specific subresource". + +The resolver's preflight gate +([rule_gvr_resolver.go:108-131](../../internal/watch/rule_gvr_resolver.go#L108-L131)) +does the opposite: + +| User writes | Field doc promises | Resolver actually does | +|---|---|---| +| `resources: ["*"]` | all resources | `WildcardResource` miss — **refused** | +| `apiGroups: ["*"]` | all groups | `WildcardGroup` miss — **refused** | +| `apiVersions: []` / `["*"]` | all versions | single **preferred** version only | +| `apiGroups: []` (ambiguous resource) | all groups | `Ambiguous` miss — **refused** | +| `resources: ["pods/log"]`, `["pods/*"]` | subresource match | `NotServed` — **refused** | + +There is **no admission validation** rejecting these (only `Operations` has an +enum), so a user can `kubectl apply` a `resources: ["*"]` rule, get no error, and +**silently watch nothing** — only a `V(1)` resolve-miss log. See +[watchrule-wildcard-and-resolution-semantics.md](../design/watchrule-wildcard-and-resolution-semantics.md) +for the full current semantics. + +### Why it matters + +The field docs were written in the Kubernetes admission-webhook / RBAC / +audit-policy idiom (the comment says so verbatim). **That is what users will +expect** — `*` and "empty = all" are universal in those APIs. The current +behavior is a silent, surprising divergence from a well-known contract. + +### Why it diverges today + +RBAC and audit policy evaluate selectors **lazily, per request**: a concrete GVR +already exists on the incoming request and the rule is just a predicate. GitOps +Reverser must **eagerly materialize** the watch set — an informer per GVR plus +authoritative `list` snapshots — so it needs concrete GVRs *before* any event +exists. The resolver was added to fix a concrete expansion bug +([watchrule-gvr-resolution-plan.md](../finished/watchrule-gvr-resolution-plan.md)) and, in +making expansion safe, narrowed the contract without updating the docs. Notably +the *matching* layer ([manager.go:480-513](../../internal/watch/manager.go#L480-L513)) +*does* honor `*` lazily like RBAC — but it only ever runs against GVRs an +informer already delivers, so it cannot rescue a wildcard rule that planned +nothing. + +## What already exists (good news for difficulty) + +The hard part of dynamic wildcard support — reacting to a changing API surface — +is **already built and running**: + +- API-surface trigger informers watch CRDs and APIServices and coalesce + add/update/delete into `catalogRefreshCh` + ([manager_catalog.go:278-303](../../internal/watch/manager_catalog.go#L278-L303)). +- The manager loop re-runs `ReconcileForRuleChange` on that signal and every 30s + ([manager.go:218-245](../../internal/watch/manager.go#L218-L245)). +- `ReconcileForRuleChange` re-resolves desired GVRs, diffs added/removed against + active informers, and starts/stops informers live + ([manager.go:751-794](../../internal/watch/manager.go#L751-L794)). +- Per-GitTarget snapshot isolation already scopes which targets re-snapshot on a + plan change + ([gittarget-isolation-on-rule-change.md](../finished/gittarget-isolation-on-rule-change.md), + [rule-set-snapshot-discovery-lag-fix.md](../finished/rule-set-snapshot-discovery-lag-fix.md)). + +So "we'd have to redo reconciles when a GVR is added" is **already the steady +state** for explicit rules. Wildcard support mostly means letting the resolver +emit a larger, catalog-derived set; the reconcile/informer/snapshot plumbing +downstream is unchanged. The churn cost is real but **scoped to the targets that +actually use a wildcard** — explicit-rule targets keep full isolation. + +The default watch policy is a small **denylist** +([resource_policy.go:30-47](../../internal/watch/resource_policy.go#L30-L47)): +pods, events, endpoints, endpointslices, leases, controllerrevisions, jobs, +cronjobs, flowcontrol. So "watch everything" really means "every served, allowed, +listable, watchable resource minus ~12 noisy kinds" — still large (every CRD +kind in the cluster), which drives the scale concerns below. + +## Difficulty assessment + +Ordered low → high effort/risk. + +### A. Resolver wildcard expansion — **moderate, low risk** + +Replace the preflight refusals with catalog enumeration: + +- `apiGroups: ["*"]` → enumerate all groups serving the named resource(s). +- `resources: ["*"]` → enumerate all resources in the named group(s) (or all + groups if combined with group `*`). +- Both `*` → every entry in the catalog. + +The catalog already holds the data (`byGVR`, `byGroupVer`, `byResource`); add +enumeration accessors (e.g. `allEntries()`, `entriesForGroup(group)`). Reuse the +existing `Allowed` / `Supports("list","watch")` / scope filters in +`gvrsForCandidates` so policy and listability still apply. Bounded, mechanical. + +### B. Empty-vs-wildcard semantics — **small, but a real decision** + +Today empty `apiGroups` demands a *unique* group (`Ambiguous` otherwise). A +wildcard must mean "**all** matches", not "exactly one". Decision: keep `[]` = +"resolve to the single served group, else error" (current, safe) and make `["*"]` += "all served groups". This preserves a useful strict mode while honoring the +documented wildcard. Same split for versions: `[]` = preferred (current), +`["*"]` = all served versions (new). + +### C. Subresources — **small; mostly a docs fix, not a feature** + +`pods/log` / `pods/*` cannot be `list`/`watch`-ed as a snapshot surface; the +list/watch+mirror model can't represent them. Recommend **dropping the +subresource promise from the field docs** rather than implementing it, and +keeping the `NotServed` classification. (Subresources still flow as audit events +through a different path; this is only about WatchRule planning.) + +### D. Snapshot robustness at wildcard scale — **moderate→large, the real risk** + +A wildcard target lists *every* allowed GVR. Two existing behaviors get stressed: + +- **Partial-view abort.** `RequestClusterState` aborts the whole target snapshot + if any single GVR `list` fails + ([manager.go:596-628](../../internal/watch/manager.go#L596-L628)) — correct + today because a missing list looks like deletions. Across hundreds of GVRs the + probability that *one* fails is much higher, so a wildcard target could rarely + produce a complete snapshot. Likely needs a per-GVR resilience strategy that + still distinguishes "failed to list" from "genuinely empty" without mirroring + spurious deletions. This is the hardest correctness question. +- **Informer scale / memory.** One informer per GVR across the whole surface is a + real resource cost; worth a soft cap or opt-in guardrail. + +### E. Plan-hash churn — **small mechanically, a policy choice** + +A wildcard target's effective-watch-plan hash changes on every CRD install/delete +→ it re-snapshots each time. Per-target isolation already contains this to +wildcard targets. Options: accept it (simplest; the user opted into "everything"), +or debounce/coalesce CRD bursts. The existing `catalogRefreshCh` already coalesces +to depth 1, so bursts are partly absorbed. **Probably acceptable as-is for now.** + +### F. Status & validation surface — **small** + +- `ResolveWatchRuleResources` / `ResolveClusterWatchRuleResources` + ([manager_catalog.go:230-265](../../internal/watch/manager_catalog.go#L230-L265)) + should report "wildcard expanded to N GVRs" in status so users see the effect. +- Correct the CRD field docs (requires `task generate` + `task manifests`). +- Optionally add a `DiscoveryDegraded`-aware guard: a wildcard expanded while a + relevant group's discovery is degraded is an incomplete plan and should block + (consistent with current snapshot-safety rules). + +## Proposed phased plan + +1. **Phase 0 — stop the silent lie (smallest, ship independently).** Either make + the docs match reality (state that `*`/subresources are rejected, `[]` + resolves uniquely/preferred) *or* add admission validation that rejects `*` + with a clear message. This removes the "applies cleanly, watches nothing" + trap regardless of whether full support lands. +2. **Phase 1 — resolver expansion (A + B + C).** Implement `*` enumeration with + the all-vs-unique split, drop the subresource promise, surface expansion in + status (F, partial). Gated behind the existing reconcile machinery; no + snapshot changes yet. Wildcard targets now watch live events correctly. +3. **Phase 2 — snapshot robustness (D).** Decide and implement per-GVR list + resilience so a wildcard target can take a complete, safe cluster snapshot. + This is the gating item for declaring wildcard support "done". +4. **Phase 3 — guardrails (D scale, E debounce).** Informer-count cap/observability + and optional CRD-burst debounce if churn proves costly in practice. + +## Open decisions + +- Phase 0: fix docs, or add rejecting validation, or both? (Both is cleanest.) +- D: what is the correct "one GVR failed to list" behavior for a wildcard target + — skip that GVR for this cycle (risk: looks like deletions) vs abort the whole + target (risk: wildcard targets rarely snapshot)? This needs its own mini-design. +- Should "watch everything" remain policy-filtered by the default denylist, or + should wildcard rules be able to opt into the excluded noisy kinds? +- Is the plan-hash churn for wildcard targets acceptable for v1, or is debounce a + prerequisite? diff --git a/docs/interpreting-metrics.md b/docs/interpreting-metrics.md index 009ab03da..820767f16 100644 --- a/docs/interpreting-metrics.md +++ b/docs/interpreting-metrics.md @@ -191,7 +191,7 @@ topk(15, sum by (resource, subresource, verb) ( Any row with a non-empty `subresource` is received-then-dropped at ingress (subresources are not mirrorable top-level objects). A large `pods`/`exec` row is the canonical example — see -[shallow-audit-event-misclassification.md](design/shallow-audit-event-misclassification.md). To +[shallow-audit-event-misclassification.md](finished/shallow-audit-event-misclassification.md). To look only at the dropped subresource traffic: ```promql diff --git a/docs/serious-bug/cozystack-bugreport.md b/docs/serious-bug/cozystack-bugreport.md new file mode 100644 index 000000000..8d05c6b27 --- /dev/null +++ b/docs/serious-bug/cozystack-bugreport.md @@ -0,0 +1,41 @@ +Title: core.cozystack.io REST storage does not emit k8s.io/initial-events-end bookmark for SendInitialEvents=true watches + +Summary + +The aggregated apiserver's resources in group core.cozystack.io (tenantsecrets, tenantmodules, tenantnamespaces) do not honor the SendInitialEvents field of ListOptions. Watch clients that opt into the WatchList / streaming-list protocol (kube-apiserver ≥ 1.27, client-go WatchListClient feature gate, on by default since client-go v1.35) never receive the required k8s.io/initial-events-end bookmark, so reflectors stay stuck in their initial-stream phase and log hasn't received required bookmark event marking the end of initial events stream every ~10s. The apps.cozystack.io/Application storage already implements this contract correctly — the three core resources do not. + +Reproduction + +Any client running client-go ≥ v0.30 with the default WatchListClient=true informing over tenantsecrets, tenantmodules, or tenantnamespaces: + + +Warning: event bookmark expired err="reflector.go:343: hasn't received required bookmark event marking the end of initial events stream, received last event 59.998946102s ago" +The warning fires on every reflector restart and continuously while the cache is unsynced from the WatchList perspective. + +Evidence (commit 1810263) + +Reference implementation that does it correctly: +pkg/registry/apps/application/rest.go:684-807 — reads options.SendInitialEvents, tracks lastResourceVersion, emits a watch.Bookmark with metadata.annotations["k8s.io/initial-events-end"] = "true" once initial ADDED events have been delivered, and even handles the "underlying watcher closes during initial snapshot" edge case. + +Missing in: + +pkg/registry/core/tenantsecret/rest.go:454 — Watch() ignores opts.SendInitialEvents and only forwards bookmark events from the backing corev1.Secret watcher; those carry no k8s.io/initial-events-end annotation. +pkg/registry/core/tenantmodule/rest.go:311 — same pattern, forwards bookmarks from the backing helmv2.HelmRelease watcher. +pkg/registry/core/tenantnamespace/rest.go:158 — same pattern, forwards bookmarks from the backing corev1.Namespace watcher. +A grep for SendInitialEvents or initial-events-end across pkg/registry/core/ returns zero hits. + +Proposed fix + +Lift the sendInitialEventsEndBookmark helper and its caller sites from application/rest.go into the three core resources, adapting the bookmark object to the resource's own TypeMeta. The mechanical shape per file: + +Near the top of Watch(): sendInitialEvents := opts.SendInitialEvents != nil && *opts.SendInitialEvents. +Track lastResourceVersion on every translated event from the backing watcher. +After delivering the initial Added events, emit a watch.Bookmark whose object is an empty TenantSecret / TenantModule / TenantNamespace carrying the right TypeMeta, ResourceVersion=lastResourceVersion, and annotations["k8s.io/initial-events-end"]="true". +Handle "underlying watcher closes before initial snapshot is complete" — emit the bookmark on the way out, same as application/rest.go:816-817. +Acceptance criteria + +A test alongside rest_watch_test.go for each of the three resources asserting: +When SendInitialEvents=true, all existing objects are emitted as ADDED. +A Bookmark event follows with annotations["k8s.io/initial-events-end"] == "true" and a non-empty ResourceVersion. +Subsequent live events are delivered after the bookmark. +An informer using client-go's WatchListClient feature reaches HasSynced=true against these resources without the missing-bookmark warning. diff --git a/hack/e2e/wait-flux-services.sh b/hack/e2e/wait-flux-services.sh index 276e5e33a..0ff0873dd 100755 --- a/hack/e2e/wait-flux-services.sh +++ b/hack/e2e/wait-flux-services.sh @@ -6,10 +6,54 @@ set -euo pipefail # # Inputs (env): # - CTX (required): kube context -# - FLUX_SERVICES_WAIT_TIMEOUT (optional): per-resource wait timeout; default 600s +# - FLUX_SERVICES_WAIT_TIMEOUT (optional): per-resource wait timeout; default 120s : "${CTX:?CTX is required}" -FLUX_SERVICES_WAIT_TIMEOUT="${FLUX_SERVICES_WAIT_TIMEOUT:-600s}" +FLUX_SERVICES_WAIT_TIMEOUT="${FLUX_SERVICES_WAIT_TIMEOUT:-120s}" + +# On a wait timeout we otherwise get a single opaque "timed out" line and have to +# guess. This dumps the resource's conditions, the source it pulls from +# (OCIRepository/HelmRepository), the workloads/events in its target namespace and +# recent Flux controller logs so the *cause* is visible in the CI log. Best-effort: +# every command is guarded so diagnostics never mask the original failure. +dump_diagnostics() { + local kind="$1" namespace="$2" name="$3" + + echo "::group::diagnostics ${kind}/${name} (ns=${namespace})" >&2 + echo "----- ${kind}/${name} conditions -----" >&2 + kubectl --context "${CTX}" -n "${namespace}" get "${kind}/${name}" -o wide >&2 2>&1 || true + kubectl --context "${CTX}" -n "${namespace}" get "${kind}/${name}" \ + -o jsonpath='{range .status.conditions[*]}{.type}={.status} reason={.reason} msg={.message}{"\n"}{end}' >&2 2>&1 || true + + if [[ "${kind}" == helmreleases.* ]]; then + # Resolve the chart source (OCIRepository/HelmRepository) and its readiness. + local src_kind src_name src_ns target_ns + src_kind="$(kubectl --context "${CTX}" -n "${namespace}" get "${kind}/${name}" -o jsonpath='{.spec.chartRef.kind}' 2>/dev/null || true)" + src_name="$(kubectl --context "${CTX}" -n "${namespace}" get "${kind}/${name}" -o jsonpath='{.spec.chartRef.name}' 2>/dev/null || true)" + src_ns="$(kubectl --context "${CTX}" -n "${namespace}" get "${kind}/${name}" -o jsonpath='{.spec.chartRef.namespace}' 2>/dev/null || true)" + src_ns="${src_ns:-${namespace}}" + if [[ -n "${src_kind}" && -n "${src_name}" ]]; then + echo "----- source ${src_kind}/${src_name} (ns=${src_ns}) -----" >&2 + kubectl --context "${CTX}" -n "${src_ns}" get "${src_kind}" "${src_name}" -o wide >&2 2>&1 || true + kubectl --context "${CTX}" -n "${src_ns}" get "${src_kind}" "${src_name}" \ + -o jsonpath='{range .status.conditions[*]}{.type}={.status} reason={.reason} msg={.message}{"\n"}{end}' >&2 2>&1 || true + fi + + target_ns="$(kubectl --context "${CTX}" -n "${namespace}" get "${kind}/${name}" -o jsonpath='{.spec.targetNamespace}' 2>/dev/null || true)" + target_ns="${target_ns:-${namespace}}" + echo "----- pods in target namespace ${target_ns} -----" >&2 + kubectl --context "${CTX}" -n "${target_ns}" get pods -o wide >&2 2>&1 || true + echo "----- recent events in ${target_ns} -----" >&2 + { kubectl --context "${CTX}" -n "${target_ns}" get events --sort-by=.lastTimestamp 2>&1 | tail -n 30; } >&2 || true + fi + + echo "----- recent flux controller logs (source/helm/kustomize) -----" >&2 + for ctrl in source-controller helm-controller kustomize-controller; do + echo "--- ${ctrl} ---" >&2 + kubectl --context "${CTX}" -n flux-system logs "deploy/${ctrl}" --tail=40 >&2 2>&1 || true + done + echo "::endgroup::" >&2 +} flux_ready_count=0 echo "⏳ Waiting for Flux-managed installations to become ready..." @@ -36,10 +80,14 @@ do while IFS=' ' read -r namespace name; do [[ -n "${namespace}" ]] || continue - kubectl --context "${CTX}" -n "${namespace}" \ + if ! kubectl --context "${CTX}" -n "${namespace}" \ wait "${kind}/${name}" \ --for=condition=Ready \ - --timeout="${FLUX_SERVICES_WAIT_TIMEOUT}" + --timeout="${FLUX_SERVICES_WAIT_TIMEOUT}"; then + echo "ERROR: ${kind}/${name} (ns=${namespace}) did not become Ready within ${FLUX_SERVICES_WAIT_TIMEOUT}" >&2 + dump_diagnostics "${kind}" "${namespace}" "${name}" + exit 1 + fi done <<<"${resources}" done diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index 5dda18c21..6e3ada1b2 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -28,12 +28,15 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-logr/logr" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -124,6 +127,23 @@ type BranchWorker struct { // firsts surfaces the first successful commit and push at default verbosity. firsts branchWorkerLogFirsts + + // hasUnpushedWork mirrors whether the event loop is currently holding a live + // open window or any committed-but-not-yet-pushed pending writes. The event + // loop is the only writer and, via syncQueueDepthMetric, the only reader; the + // atomic guards the store/load and lets the depth gauge account for retained + // work the channel length alone cannot see. + hasUnpushedWork atomic.Bool + + // inflightItems counts work items accepted onto eventQueue but not yet fully + // handled. It is incremented before the channel send (so it can never lag the + // loop's receive) and decremented only after handleQueueItem returns. Using + // len(eventQueue) instead would read 0 in the window between the loop + // receiving an item and finishing it — a scrape there could satisfy a drain + // gate while a commit/push is still in flight. The depth gauge sums this with + // hasUnpushedWork, so it reads 0 only once every accepted item has been + // handled and nothing is retained. + inflightItems atomic.Int64 } // branchWorkerLogFirsts logs the first successful commit and push of a worker's @@ -246,10 +266,17 @@ func (w *BranchWorker) EnqueueFinalize(signal *FinalizeSignal) { if signal == nil { return } + // Increment before the send so inflightItems can never lag the loop's + // receive; roll back if the queue is full and the item is dropped. + w.inflightItems.Add(1) select { case w.eventQueue <- WorkItem{Finalize: signal}: w.Log.V(1).Info("Finalize signal enqueued") + // Depth is published only from the loop goroutine (syncQueueDepthMetric); + // the loop republishes on every received item, so the gauge converges + // without an enqueue-side write that could latch a stale value. default: + w.inflightItems.Add(-1) w.Log.Error(nil, "Event queue full, finalize signal dropped") signal.reply(FinalizeResult{Branch: w.Branch, Err: ErrFinalizeQueueFull}) } @@ -260,13 +287,20 @@ func (w *BranchWorker) enqueueRequest(request *WriteRequest) { return } item := WorkItem{Request: request} + // Increment before the send so inflightItems can never lag the loop's + // receive; roll back if the queue is full and the item is dropped. + w.inflightItems.Add(1) select { case w.eventQueue <- item: w.Log.V(1).Info("Write request enqueued", "events", len(request.Events), "mode", request.CommitMode, "gitTarget", request.GitTargetName) + // Depth is published only from the loop goroutine (syncQueueDepthMetric); + // the loop republishes on every received item, so the gauge converges + // without an enqueue-side write that could latch a stale value. default: + w.inflightItems.Add(-1) w.Log.Error(nil, "Event queue full, request dropped", "events", len(request.Events), "mode", request.CommitMode, @@ -274,6 +308,35 @@ func (w *BranchWorker) enqueueRequest(request *WriteRequest) { } } +// recordQueueDepth publishes the current pending-work depth for this worker: +// the number of accepted-but-not-yet-handled items (queued or actively being +// processed) plus one when the event loop is holding retained unpushed work (a +// live open window or pending writes). It reads 0 only when the worker has +// fully drained — every accepted item handled and nothing retained — so a +// drain gate cannot be satisfied while a commit/push is still in flight. Called +// only from the loop goroutine (via syncQueueDepthMetric) so the OTel gauge — +// last-writer-wins — can never latch a stale depth from an enqueue goroutine +// that raced the loop's drain. No-op until the gauge is registered (e.g. in +// unit tests that never init the exporter). +func (w *BranchWorker) recordQueueDepth() { + if telemetry.BranchWorkerQueueDepth == nil { + return + } + depth := w.inflightItems.Load() + if w.hasUnpushedWork.Load() { + depth++ + } + ctx := w.ctx + if ctx == nil { + ctx = context.Background() + } + telemetry.BranchWorkerQueueDepth.Record(ctx, depth, metric.WithAttributes( + attribute.String("provider_namespace", w.GitProviderNamespace), + attribute.String("provider_name", w.GitProviderRef), + attribute.String("branch", w.Branch), + )) +} + // ListResourcesInPath returns resource identifiers found in a Git folder. // This is a synchronous service method called by EventRouter. func (w *BranchWorker) ListResourcesInPath(path string) ([]itypes.ResourceIdentifier, error) { @@ -521,14 +584,20 @@ func newBranchWorkerEventLoop(w *BranchWorker, commitWindow time.Duration) *bran func (l *branchWorkerEventLoop) run() { defer l.stopTimers() + l.syncQueueDepthMetric() for { commitC, pushC := l.timerChannels() select { case <-l.w.ctx.Done(): l.handleShutdown() + l.syncQueueDepthMetric() return case item := <-l.w.eventQueue: l.handleQueueItem(item) + // Decrement only after the item is fully handled; the post-handling + // open-window/pending-writes state is captured by syncQueueDepthMetric + // below, so there is no window where depth drops to 0 prematurely. + l.w.inflightItems.Add(-1) case <-commitC: l.commitTimer = nil l.finalizeOpenWindow() @@ -537,9 +606,20 @@ func (l *branchWorkerEventLoop) run() { l.pushTimer = nil l.pushPending() } + l.syncQueueDepthMetric() } } +// syncQueueDepthMetric refreshes the worker's unpushed-work flag from the loop's +// authoritative state and republishes the depth gauge. Called once per loop +// iteration (the loop goroutine is the only writer of hasUnpushedWork), so the +// gauge converges to 0 once every accepted item has been handled (inflightItems +// == 0) and nothing is retained (no open window, no pending writes). +func (l *branchWorkerEventLoop) syncQueueDepthMetric() { + l.w.hasUnpushedWork.Store(l.openWindow != nil || len(l.pendingWrites) > 0) + l.w.recordQueueDepth() +} + func (l *branchWorkerEventLoop) timerChannels() (<-chan time.Time, <-chan time.Time) { var commitC, pushC <-chan time.Time if l.commitTimer != nil { @@ -635,6 +715,31 @@ func (l *branchWorkerEventLoop) handleShutdown() { // the worker exits, even if a push was just sent. l.pushPending() } + l.drainUnhandledQueueItems() +} + +// drainUnhandledQueueItems clears items still buffered on eventQueue that the +// exiting loop will never handle. Each was counted into inflightItems at enqueue +// and is decremented only by the loop after handling, so without this drain the +// final syncQueueDepthMetric would publish a non-zero depth for the exiting +// worker that never clears — the loop has stopped, so nothing republishes a +// corrected value. A buffered finalize signal is answered with +// ErrWorkerShuttingDown so its caller unblocks instead of waiting out its +// timeout. The depth gauge then settles to 0 once the open window is finalized +// and pending writes are pushed (any genuinely-unpushed work keeps it non-zero, +// which is correct). +func (l *branchWorkerEventLoop) drainUnhandledQueueItems() { + for { + select { + case item := <-l.w.eventQueue: + if item.Finalize != nil { + item.Finalize.reply(FinalizeResult{Branch: l.w.Branch, Err: ErrWorkerShuttingDown}) + } + l.w.inflightItems.Add(-1) + default: + return + } + } } func (l *branchWorkerEventLoop) resetCommitTimer() { diff --git a/internal/git/branch_worker_metrics_test.go b/internal/git/branch_worker_metrics_test.go new file mode 100644 index 000000000..c30a99657 --- /dev/null +++ b/internal/git/branch_worker_metrics_test.go @@ -0,0 +1,186 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "context" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +const branchWorkerQueueDepthMetric = "gitopsreverser_branch_worker_queue_depth" + +func newMetricsTestWorker() *BranchWorker { + return &BranchWorker{ + GitProviderRef: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Log: logr.Discard(), + ctx: context.Background(), + contentWriter: newContentWriter(itypes.SensitiveResourcePolicy{}), + eventQueue: make(chan WorkItem, branchWorkerQueueSize), + branchBufferMaxBytes: DefaultBranchBufferMaxBytes, + } +} + +func queueDepthLabels() map[string]string { + return map[string]string{ + "provider_namespace": "test-ns", + "provider_name": "test-provider", + "branch": "main", + } +} + +// recordQueueDepth must report 0 for a freshly drained worker: empty queue and +// no retained unpushed work. +func TestRecordQueueDepth_DrainedReportsZero(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + w := newMetricsTestWorker() + w.recordQueueDepth() + + depth, ok := telemetry.CollectInt64Sum(reader, branchWorkerQueueDepthMetric, queueDepthLabels()) + require.True(t, ok, "expected a branch_worker_queue_depth sample") + assert.Equal(t, int64(0), depth) +} + +// recordQueueDepth must count both accepted-but-unhandled items (in flight) and +// the retained unpushed-work flag, so the gauge reflects work the channel length +// alone cannot see — including an item already dequeued and being processed. +func TestRecordQueueDepth_CountsInflightAndRetained(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + w := newMetricsTestWorker() + // Two items accepted but not yet fully handled, plus retained unpushed work. + w.inflightItems.Store(2) + w.hasUnpushedWork.Store(true) + + w.recordQueueDepth() + + depth, ok := telemetry.CollectInt64Sum(reader, branchWorkerQueueDepthMetric, queueDepthLabels()) + require.True(t, ok, "expected a branch_worker_queue_depth sample") + assert.Equal(t, int64(3), depth) +} + +// recordQueueDepth must still report > 0 for an item that has been dequeued from +// the channel but is still being handled — the exact window where len(eventQueue) +// would read 0 and a drain gate could be falsely satisfied mid-commit. +func TestRecordQueueDepth_InflightItemDequeuedButUnhandled(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + w := newMetricsTestWorker() + // Simulate the loop having received the item (channel empty) while it is + // still being processed: inflight accounts for it, retained flag not yet set. + w.inflightItems.Store(1) + w.hasUnpushedWork.Store(false) + + w.recordQueueDepth() + + depth, ok := telemetry.CollectInt64Sum(reader, branchWorkerQueueDepthMetric, queueDepthLabels()) + require.True(t, ok, "expected a branch_worker_queue_depth sample") + assert.Equal(t, int64(1), depth, "an in-flight item must keep depth > 0 even with an empty channel") +} + +// enqueueRequest must account for the accepted item in inflightItems but must +// NOT publish the depth gauge itself: the gauge is last-writer-wins, so an +// enqueue goroutine that raced the loop's drain could latch a stale value and +// hang the restart drain gate. Publication is the loop's job; once the loop +// observes the inflight item it reports the non-zero depth. +func TestEnqueueRequest_AccountsInflightWithoutPublishing(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + w := newMetricsTestWorker() + w.enqueueRequest(&WriteRequest{ + Events: []Event{{}}, + CommitMode: CommitModePerEvent, + }) + + // The item is accounted for, but enqueue did not touch the gauge. + assert.Equal(t, int64(1), w.inflightItems.Load()) + _, ok := telemetry.CollectInt64Sum(reader, branchWorkerQueueDepthMetric, queueDepthLabels()) + assert.False(t, ok, "enqueue must not publish the depth gauge; only the loop does") + + // The loop's publication (modelled here by a direct record) then reports the + // inflight item as non-zero depth. + w.recordQueueDepth() + depth, ok := telemetry.CollectInt64Sum(reader, branchWorkerQueueDepthMetric, queueDepthLabels()) + require.True(t, ok, "expected a branch_worker_queue_depth sample after the loop publishes") + assert.Equal(t, int64(1), depth) +} + +// On shutdown the loop must drain items still buffered on eventQueue so the depth +// gauge settles to 0. Each buffered item was counted into inflightItems at +// enqueue; left undrained, the exiting worker's final publish would latch a +// non-zero depth that never clears. +func TestHandleShutdown_DrainsBufferedItemsToZeroDepth(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + w := newMetricsTestWorker() + // Two write requests accepted onto the queue but never handled by the loop. + w.enqueueRequest(&WriteRequest{Events: []Event{{}}, CommitMode: CommitModePerEvent}) + w.enqueueRequest(&WriteRequest{Events: []Event{{}}, CommitMode: CommitModePerEvent}) + require.Equal(t, int64(2), w.inflightItems.Load()) + + loop := newBranchWorkerEventLoop(w, time.Second) + loop.handleShutdown() + // run() publishes once more after handleShutdown; model that here. + loop.syncQueueDepthMetric() + + assert.Equal(t, int64(0), w.inflightItems.Load(), "buffered items must be drained from the inflight count") + depth, ok := telemetry.CollectInt64Sum(reader, branchWorkerQueueDepthMetric, queueDepthLabels()) + require.True(t, ok, "expected a branch_worker_queue_depth sample after shutdown") + assert.Equal(t, int64(0), depth, "a drained, exiting worker must publish depth 0") +} + +// A finalize signal still buffered at shutdown must be answered with +// ErrWorkerShuttingDown so its caller unblocks instead of waiting out its +// timeout, and its inflight slot must be released. +func TestHandleShutdown_RepliesToBufferedFinalize(t *testing.T) { + _, err := telemetry.InitTestExporter() + require.NoError(t, err) + + w := newMetricsTestWorker() + result := make(chan FinalizeResult, 1) + w.EnqueueFinalize(&FinalizeSignal{Result: result}) + require.Equal(t, int64(1), w.inflightItems.Load()) + + loop := newBranchWorkerEventLoop(w, time.Second) + loop.handleShutdown() + + assert.Equal(t, int64(0), w.inflightItems.Load()) + select { + case res := <-result: + require.ErrorIs(t, res.Err, ErrWorkerShuttingDown) + assert.Equal(t, "main", res.Branch) + default: + t.Fatal("expected a finalize reply on shutdown, got none") + } +} diff --git a/internal/git/finalize_signal.go b/internal/git/finalize_signal.go index 481b881a5..c04f30795 100644 --- a/internal/git/finalize_signal.go +++ b/internal/git/finalize_signal.go @@ -24,6 +24,11 @@ import "errors" // because the worker's event queue is saturated. var ErrFinalizeQueueFull = errors.New("branch worker event queue full; finalize signal dropped") +// ErrWorkerShuttingDown is reported for a finalize signal that was still buffered +// on the event queue when the worker began shutting down, so its caller unblocks +// immediately instead of waiting out its timeout. +var ErrWorkerShuttingDown = errors.New("branch worker shutting down; finalize signal not processed") + // FinalizeOutcome is the terminal result of processing a FinalizeSignal. type FinalizeOutcome string diff --git a/internal/rulestore/store_test.go b/internal/rulestore/store_test.go index ff8aa25df..0212c75c8 100644 --- a/internal/rulestore/store_test.go +++ b/internal/rulestore/store_test.go @@ -386,6 +386,109 @@ func TestGetMatchingRules(t *testing.T) { } } +// TestGetMatchingRules_OverlappingRulesUnionOperations pins the semantics for +// "two WatchRules match the same resource": operations *add up* (union), there +// is no first-wins precedence and no single rule that "owns" the resource. +// +// Matching is evaluated per-operation: GetMatchingRules returns every rule whose +// operation set includes the operation being tested (an empty set means all). +// So for pods covered by rule A=[CREATE] and rule B=[UPDATE]: +// - a CREATE matches A only, +// - an UPDATE matches B only, +// - a DELETE matches neither (the union is exactly {CREATE, UPDATE}, not "all"), +// - and when two rules both cover CREATE, *both* are returned (additive, not +// deduped to one). +// +// This matters for the per-target effective watch plan +// (docs/design/gittarget-isolation-on-rule-change.md): a target's watched +// operation set for a GVR is the union across its rules, and each returned +// CompiledRule carries its own GitTarget, so two complementary rules pointing at +// the same target make that target watch the union. +func TestGetMatchingRules_OverlappingRulesUnionOperations(t *testing.T) { + store := NewStore() + + podsRule := func(name string, ops ...configv1alpha1.OperationType) configv1alpha1.WatchRule { + r := configv1alpha1.WatchRule{ + Spec: configv1alpha1.WatchRuleSpec{ + Rules: []configv1alpha1.ResourceRule{{ + Operations: ops, + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }}, + }, + } + r.Name = name + r.Namespace = "default" + return r + } + + // Two rules covering the same resource with complementary operations, plus a + // second CREATE rule to prove matches are additive rather than first-wins. + store.AddOrUpdateWatchRule( + podsRule("pod-create", configv1alpha1.OperationCreate), + "dest-a", "default", "repo", "gitops-system", "main", "a") + store.AddOrUpdateWatchRule( + podsRule("pod-update", configv1alpha1.OperationUpdate), + "dest-a", "default", "repo", "gitops-system", "main", "a") + store.AddOrUpdateWatchRule( + podsRule("pod-create-2", configv1alpha1.OperationCreate), + "dest-b", "default", "repo", "gitops-system", "main", "b") + + tests := []struct { + name string + operation configv1alpha1.OperationType + expectedNames []string + }{ + { + name: "CREATE returns both CREATE rules (additive, not first-wins)", + operation: configv1alpha1.OperationCreate, + expectedNames: []string{"pod-create", "pod-create-2"}, + }, + { + name: "UPDATE returns the UPDATE rule (operations union across rules)", + operation: configv1alpha1.OperationUpdate, + expectedNames: []string{"pod-update"}, + }, + { + name: "DELETE matches nothing (union is exactly CREATE+UPDATE, not all)", + operation: configv1alpha1.OperationDelete, + expectedNames: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matches := store.GetMatchingRules(nil, "pods", tt.operation, "", "v1", false) + + gotNames := make(map[string]struct{}, len(matches)) + for _, m := range matches { + gotNames[m.Source.Name] = struct{}{} + } + + if len(matches) != len(tt.expectedNames) { + t.Errorf("expected %d matches %v, got %d %v", + len(tt.expectedNames), tt.expectedNames, len(matches), keys(gotNames)) + } + for _, want := range tt.expectedNames { + if _, ok := gotNames[want]; !ok { + t.Errorf("expected rule %q to match %s, got %v", + want, tt.operation, keys(gotNames)) + } + } + }) + } +} + +// keys returns the keys of a set, for readable test failure messages. +func keys(m map[string]struct{}) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + func TestRuleStore_Readiness(t *testing.T) { store := NewStore() if store.IsReady() { diff --git a/internal/telemetry/exporter.go b/internal/telemetry/exporter.go index 487ca3d8c..2f5288e8d 100644 --- a/internal/telemetry/exporter.go +++ b/internal/telemetry/exporter.go @@ -62,6 +62,36 @@ var ( // RepoBranchQueueDepth is a gauge for per-repo-branch queue depth. RepoBranchQueueDepth metric.Int64UpDownCounter + // TargetReconcileCompletedTotal counts completed rule-set snapshot reconcile + // passes per GitTarget: each increment marks one pass where the snapshot + // decision was made and its write request submitted to the branch worker + // queue. Labelled by {gittarget_namespace, gittarget_name, trigger} where + // trigger is `rule_change` (the GVR/rule reconcile path) or `startup_replay` + // (a snapshot replayed once a FolderReconciler is created). A counter, not a + // latched gauge, on purpose: a counter resets to 0 on a fresh pod, so a + // per-pod `{pod=""} > 0` check after a rollout proves the new pod did + // its own reconcile — robust to the old pod's stale series that a Prometheus + // pod scrape may still be holding during the rollout, which a latched gauge + // (or a cross-pod sum-over-baseline) cannot distinguish. + // The label keys avoid the reserved `namespace`/`name`: a pod scrape with + // honor_labels=false would overwrite a metric's `namespace` attribute with the + // scraped pod's own namespace, making a per-GitTarget `namespace` selector + // silently match nothing. Load-bearing for the restart-snapshot e2e spec and + // useful long-term for spotting excessive reconciles via increase(...[5m]); + // treat the name/labels as a public observability contract. + TargetReconcileCompletedTotal metric.Int64Counter + // BranchWorkerQueueDepth gauges pending work for a single branch worker: + // accepted-but-not-yet-handled items (queued or actively being processed) + // plus any committed-but-not-yet-pushed work the worker is still holding. It + // reads 0 only when the worker has fully drained (every accepted item handled + // and nothing retained for replay), so it never reports drained while a + // commit/push is still in flight. Labelled by {provider_namespace, + // provider_name, branch}; the namespace/name keys are prefixed to avoid the + // reserved Prometheus pod-scrape target labels (see + // TargetReconcileCompletedTotal). Load-bearing for the restart-snapshot e2e + // spec's drain wait; treat the name/labels as a public observability contract. + BranchWorkerQueueDepth metric.Int64Gauge + // WatchDuplicatesSkippedTotal counts watch events skipped due to duplicate sanitized content. WatchDuplicatesSkippedTotal metric.Int64Counter // AuditEventsReceivedTotal counts audit events received from Kubernetes API server. @@ -236,6 +266,7 @@ func registerInstruments() error { {"gitopsreverser_secret_encryption_failures_total", &SecretEncryptionFailuresTotal}, {"gitopsreverser_secret_encryption_cache_hits_total", &SecretEncryptionCacheHitsTotal}, {"gitopsreverser_secret_encryption_marker_skips_total", &SecretEncryptionMarkerSkipsTotal}, + {"gitopsreverser_target_reconcile_completed_total", &TargetReconcileCompletedTotal}, } for _, s := range counters { v, err := otelMeter.Int64Counter(s.name) @@ -284,6 +315,7 @@ func registerInstruments() error { {"gitopsreverser_audit_queue_oldest_entry_age_seconds", &AuditQueueOldestEntryAgeSeconds}, {"gitopsreverser_audit_queue_oldest_pending_age_seconds", &AuditQueueOldestPendingAgeSeconds}, {"gitopsreverser_audit_debug_stream_length", &AuditDebugStreamLength}, + {"gitopsreverser_branch_worker_queue_depth", &BranchWorkerQueueDepth}, } for _, s := range gauges { v, err := otelMeter.Int64Gauge(s.name) diff --git a/internal/watch/catalog_test_helpers_test.go b/internal/watch/catalog_test_helpers_test.go index d84c2c933..e2e719fed 100644 --- a/internal/watch/catalog_test_helpers_test.go +++ b/internal/watch/catalog_test_helpers_test.go @@ -120,35 +120,3 @@ func commonTestDiscoveryClient() func() (apiResourceDiscovery, error) { return newCommonTestDiscovery(), nil } } - -// snapshotDeliveryTestDiscovery returns a trusted-but-minimal discovery that -// serves only an unrelated resource. The snapshot-delivery tests register -// configmaps rules purely to exercise rule-set-change snapshot emission; with -// configmaps absent from trusted discovery, rule resolution requests no informer -// GVRs, so those tests never plan real informers. This replaces the old -// discoveryFilter stub that returned no GVRs. -func snapshotDeliveryTestDiscovery() staticCatalogDiscovery { - return staticCatalogDiscovery{ - groups: []*metav1.APIGroup{testAPIGroup("", "v1")}, - resources: []*metav1.APIResourceList{{ - GroupVersion: "v1", - APIResources: []metav1.APIResource{ - {Name: "nodes", Kind: "Node", Verbs: metav1.Verbs{"get", "list", "watch"}}, - }, - }}, - } -} - -func newSnapshotDeliveryTestCatalog(t *testing.T) *APIResourceCatalog { - t.Helper() - catalog := NewAPIResourceCatalog() - _, err := catalog.Refresh(snapshotDeliveryTestDiscovery()) - require.NoError(t, err) - return catalog -} - -func snapshotDeliveryTestDiscoveryClient() func() (apiResourceDiscovery, error) { - return func() (apiResourceDiscovery, error) { - return snapshotDeliveryTestDiscovery(), nil - } -} diff --git a/internal/watch/manager.go b/internal/watch/manager.go index c7544ae7e..c1321d366 100644 --- a/internal/watch/manager.go +++ b/internal/watch/manager.go @@ -41,12 +41,15 @@ import ( "github.com/cespare/xxhash/v2" "github.com/go-logr/logr" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" "sigs.k8s.io/controller-runtime/pkg/client" configv1alpha1 "github.com/ConfigButler/gitops-reverser/api/v1alpha1" "github.com/ConfigButler/gitops-reverser/internal/events" "github.com/ConfigButler/gitops-reverser/internal/rulestore" "github.com/ConfigButler/gitops-reverser/internal/sanitize" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" "github.com/ConfigButler/gitops-reverser/internal/types" ) @@ -116,8 +119,67 @@ func (m *Manager) SnapshotEmitCount() int64 { } type ruleSetSnapshotTarget struct { + gitDest types.ResourceReference + hash uint64 + hasEntries bool +} + +// targetWatchPlan accumulates a single GitTarget's effective watch surface while +// currentRuleSetSnapshots walks the rule set. entries maps an entry key +// ("group/version/resource|scope|ns") to the union of operation tokens watched +// for it; dest holds the write destination. hash() folds both into a stable +// per-target hash. See currentRuleSetSnapshots for the rationale on what is and +// is not included. +type targetWatchPlan struct { gitDest types.ResourceReference - hash uint64 + entries map[string]map[string]struct{} + dest string +} + +// addEntry records that the target watches gvr in the given namespace ("" means +// all namespaces / cluster-scoped) for the given operations. An empty or +// wildcard operation set is canonicalised to "*" (all operations), and once "*" +// is present it subsumes any explicit operations during hashing. +func (p *targetWatchPlan) addEntry(gvr GVR, namespace string, ops []configv1alpha1.OperationType) { + entryKey := fmt.Sprintf("%s/%s/%s|scope=%s|ns=%s", + gvr.Group, gvr.Version, gvr.Resource, gvr.Scope, namespace) + opsSet := p.entries[entryKey] + if opsSet == nil { + opsSet = make(map[string]struct{}) + p.entries[entryKey] = opsSet + } + if len(ops) == 0 { + opsSet["*"] = struct{}{} + return + } + for _, op := range ops { + if op == configv1alpha1.OperationAll { + opsSet["*"] = struct{}{} + continue + } + opsSet[string(op)] = struct{}{} + } +} + +// hash returns a stable hash of the plan: destination plus each entry with its +// resolved operation set, sorted for determinism. +func (p *targetWatchPlan) hash() uint64 { + entryStrings := make([]string, 0, len(p.entries)) + for entryKey, opsSet := range p.entries { + var ops []string + if _, all := opsSet["*"]; all { + ops = []string{"*"} + } else { + ops = make([]string, 0, len(opsSet)) + for op := range opsSet { + ops = append(ops, op) + } + sort.Strings(ops) + } + entryStrings = append(entryStrings, entryKey+"|ops="+strings.Join(ops, ",")) + } + sort.Strings(entryStrings) + return xxhash.Sum64String(p.dest + "\x00" + strings.Join(entryStrings, "\x00")) } const ( @@ -702,7 +764,7 @@ func (m *Manager) ReconcileForRuleChange(ctx context.Context) error { activeCount := len(m.activeInformers) m.informersMu.Unlock() - targets := m.snapshotTargetsNeedingDelivery(len(added) > 0 || len(removed) > 0) + targets := m.snapshotTargetsNeedingDelivery() if len(added) == 0 && len(removed) == 0 && len(targets) == 0 { log.V(1).Info("No GVR changes detected, skipping reconciliation", "activeGVRs", activeCount) @@ -738,15 +800,23 @@ func (m *Manager) ReconcileForRuleChange(ctx context.Context) error { // Emit RequestClusterState for each affected GitTarget so that a single // "reconcile: sync N resources" commit is produced instead of N individual // [CREATE] commits from the informer ADDED events buffered above. - m.emitSnapshotForRuleChange(ctx, log, targets) + deliveryErr := m.emitSnapshotForRuleChange(ctx, log, targets, "rule_change") // Transition streams back to LIVE_PROCESSING and flush buffered events. // startInformersForGVRs already waited for cache sync (WaitForCacheSync), // so all initial ADDED events are guaranteed to be buffered before this point. // The flushed events are no-ops at the git level because the snapshot batch - // just wrote those files. + // just wrote those files. Run this even when a target failed delivery so the + // streams that did snapshot leave the buffering state. m.completeReconciliationForTargets(targets, log) + if deliveryErr != nil { + // A transient emit failure left at least one target pending. Surface it so + // the controller requeues with backoff and retries promptly instead of + // waiting for the next periodic reconcile. + return deliveryErr + } + log.V(1).Info("Watch manager reconciliation completed", "addedGVRs", len(added), "removedGVRs", len(removed)) @@ -1100,7 +1170,13 @@ func splitResourceKey(key string) []string { return parts } -func (m *Manager) snapshotTargetsNeedingDelivery(force bool) []ruleSetSnapshotTarget { +// snapshotTargetsNeedingDelivery returns the GitTargets whose effective watch +// plan hash differs from the one last delivered — i.e. targets whose watched +// resource surface actually changed. Selection is purely per-target: global GVR +// churn for an unrelated target no longer drags every target into a snapshot. +// Targets whose hash is unchanged keep processing live events. See +// docs/design/gittarget-isolation-on-rule-change.md. +func (m *Manager) snapshotTargetsNeedingDelivery() []ruleSetSnapshotTarget { current := m.currentRuleSetSnapshots() currentKeys := make(map[string]struct{}, len(current)) @@ -1112,10 +1188,11 @@ func (m *Manager) snapshotTargetsNeedingDelivery(force bool) []ruleSetSnapshotTa for _, target := range current { key := target.gitDest.Key() currentKeys[key] = struct{}{} - if !force { - if lastDelivered, ok := m.lastDeliveredRuleSetHash[key]; ok && lastDelivered == target.hash { - continue - } + if !target.hasEntries { + continue + } + if lastDelivered, ok := m.lastDeliveredRuleSetHash[key]; ok && lastDelivered == target.hash { + continue } m.pendingRuleSetHash[key] = target.hash targets = append(targets, target) @@ -1136,58 +1213,87 @@ func (m *Manager) snapshotTargetsNeedingDelivery(force bool) []ruleSetSnapshotTa return targets } +// currentRuleSetSnapshots returns, per GitTarget, a hash of that target's +// *effective watch plan* — what it actually watches after rule resolution and +// API discovery, not the literal rule text. The hash drives snapshot selection +// in snapshotTargetsNeedingDelivery: a target is re-snapshotted only when its +// plan hash changes. +// +// A plan is the set of (resolved GVR, namespace scope, union of operations) +// entries the target watches, plus the write destination (provider/branch/path). +// Deliberately excluded: source rule identity (namespace/name) and the raw +// apiGroups/apiVersions/resources patterns. Those are inputs to resolution, not +// the resolved surface — keeping them caused unrelated churn (a redundant +// duplicate rule) and missed real churn (a wildcard newly matching a CRD). See +// docs/design/gittarget-isolation-on-rule-change.md. +// +// Operations add up across rules: when two rules for the same target resolve to +// the same GVR, the entry's operation set is their union (see +// rulestore TestGetMatchingRules_OverlappingRulesUnionOperations). A target with +// rules that currently resolve to nothing is kept as an empty plan so transient +// discovery gaps do not look like rule removal. func (m *Manager) currentRuleSetSnapshots() []ruleSetSnapshotTarget { - entriesByTarget := make(map[string][]string) - refsByTarget := make(map[string]types.ResourceReference) + plans := make(map[string]*targetWatchPlan) + resolver := m.ruleGVRResolver() - for _, rule := range m.RuleStore.SnapshotWatchRules() { - ref := types.NewResourceReference(rule.GitTargetRef, rule.GitTargetNamespace) + plan := func(ref types.ResourceReference, providerNS, provider, branch, path string) *targetWatchPlan { key := ref.Key() - refsByTarget[key] = ref - entriesByTarget[key] = append(entriesByTarget[key], fmt.Sprintf( - "watch|source=%s/%s|target=%s/%s|provider=%s/%s|branch=%q|path=%q|rules=%#v", - rule.Source.Namespace, - rule.Source.Name, - rule.GitTargetNamespace, - rule.GitTargetRef, - rule.GitProviderNamespace, - rule.GitProviderRef, - rule.Branch, - rule.Path, - rule.ResourceRules, - )) + p := plans[key] + if p == nil { + p = &targetWatchPlan{gitDest: ref, entries: make(map[string]map[string]struct{})} + plans[key] = p + } + // Destination is a property of the GitTarget, so it is identical across + // that target's rules; recording it on each rule is harmless. + p.dest = fmt.Sprintf("provider=%s/%s|branch=%q|path=%q", providerNS, provider, branch, path) + return p + } + + // Namespaced WatchRules watch their own namespace (rule.Source.Namespace), + // so the namespace is part of the resolved scope even though the rule name is + // not part of the plan. + for _, rule := range m.RuleStore.SnapshotWatchRules() { + p := plan( + types.NewResourceReference(rule.GitTargetRef, rule.GitTargetNamespace), + rule.GitProviderNamespace, rule.GitProviderRef, rule.Branch, rule.Path, + ) + for _, rr := range rule.ResourceRules { + gvrs, _ := resolver.Resolve(rr.APIGroups, rr.APIVersions, rr.Resources, + configv1alpha1.ResourceScopeNamespaced) + for _, gvr := range gvrs { + p.addEntry(gvr, rule.Source.Namespace, rr.Operations) + } + } } + // ClusterWatchRules carry per-rule scope and watch all namespaces (or are + // cluster-scoped), so the plan namespace is empty. for _, rule := range m.RuleStore.SnapshotClusterWatchRules() { - ref := types.NewResourceReference(rule.GitTargetRef, rule.GitTargetNamespace) - key := ref.Key() - refsByTarget[key] = ref - entriesByTarget[key] = append(entriesByTarget[key], fmt.Sprintf( - "cluster|source=%s|target=%s/%s|provider=%s/%s|branch=%q|path=%q|rules=%#v", - rule.Source.Name, - rule.GitTargetNamespace, - rule.GitTargetRef, - rule.GitProviderNamespace, - rule.GitProviderRef, - rule.Branch, - rule.Path, - rule.Rules, - )) - } - - keys := make([]string, 0, len(entriesByTarget)) - for key := range entriesByTarget { + p := plan( + types.NewResourceReference(rule.GitTargetRef, rule.GitTargetNamespace), + rule.GitProviderNamespace, rule.GitProviderRef, rule.Branch, rule.Path, + ) + for _, rr := range rule.Rules { + gvrs, _ := resolver.Resolve(rr.APIGroups, rr.APIVersions, rr.Resources, rr.Scope) + for _, gvr := range gvrs { + p.addEntry(gvr, "", rr.Operations) + } + } + } + + keys := make([]string, 0, len(plans)) + for key := range plans { keys = append(keys, key) } sort.Strings(keys) targets := make([]ruleSetSnapshotTarget, 0, len(keys)) for _, key := range keys { - entries := entriesByTarget[key] - sort.Strings(entries) + p := plans[key] targets = append(targets, ruleSetSnapshotTarget{ - gitDest: refsByTarget[key], - hash: xxhash.Sum64String(strings.Join(entries, "\x00")), + gitDest: p.gitDest, + hash: p.hash(), + hasEntries: len(p.entries) > 0, }) } return targets @@ -1235,7 +1341,11 @@ func (m *Manager) MaybeReplaySnapshot(ctx context.Context, gitDest types.Resourc target := ruleSetSnapshotTarget{gitDest: gitDest, hash: hash} log := m.Log.WithName("reconcile") m.EventRouter.BeginReconciliationForStream(gitDest) - m.emitSnapshotForRuleChange(ctx, log, []ruleSetSnapshotTarget{target}) + if err := m.emitSnapshotForRuleChange(ctx, log, []ruleSetSnapshotTarget{target}, "startup_replay"); err != nil { + // The target stays pending (not marked delivered), so the next reconcile + // or replay retries it; just record why this attempt did not land. + log.Error(err, "snapshot replay did not complete, leaving target pending", "gitDest", gitDest.String()) + } m.EventRouter.CompleteReconciliationForStream(gitDest) } @@ -1252,20 +1362,31 @@ func (m *Manager) beginReconciliationForTargets(targets []ruleSetSnapshotTarget, // emitSnapshotForRuleChange emits fresh repo and cluster state requests for every affected // GitTarget so FolderReconciler diffs against current repository contents rather than a // stale cached repo snapshot from an earlier reconcile. +// +// A target is marked delivered and counted only once *both* its repo- and +// cluster-state requests land on the worker queue: a partial emission would have +// the reconciler diff against an incomplete cluster view. A transient failure on +// either request leaves that target pending and is returned as an error so the +// caller requeues with backoff and retries it promptly, rather than waiting out +// the 30s periodic reconcile — which matters for the per-pod restart gate that +// blocks on the reconcile counter reaching the new pod. Other targets in the +// batch are still attempted so one bad target cannot starve the rest. func (m *Manager) emitSnapshotForRuleChange( ctx context.Context, log logr.Logger, targets []ruleSetSnapshotTarget, -) { + trigger string, +) error { if m.EventRouter == nil { log.Info("EventRouter not set, skipping snapshot emission") if len(targets) > 0 { m.snapshotEmitCount.Add(1) } - return + return nil } log.Info("Emitting fresh repo and cluster state for affected GitTargets after rule change", "count", len(targets)) emitted := false + var errs []error for _, target := range targets { gitDest := target.gitDest if m.EventRouter.ReconcilerManager == nil { @@ -1283,6 +1404,7 @@ func (m *Manager) emitSnapshotForRuleChange( GitDest: gitDest, }); err != nil { log.Error(err, "failed to emit RequestRepoState for rule change", "gitDest", gitDest) + errs = append(errs, fmt.Errorf("emit RequestRepoState for %s: %w", gitDest, err)) continue } if err := m.EventRouter.ProcessControlEvent(ctx, events.ControlEvent{ @@ -1290,14 +1412,36 @@ func (m *Manager) emitSnapshotForRuleChange( GitDest: gitDest, }); err != nil { log.Error(err, "failed to emit RequestClusterState for rule change", "gitDest", gitDest) + errs = append(errs, fmt.Errorf("emit RequestClusterState for %s: %w", gitDest, err)) continue } m.markRuleSetSnapshotDelivered(target) + m.recordTargetReconcileCompleted(gitDest, trigger) emitted = true } if emitted { m.snapshotEmitCount.Add(1) } + return errors.Join(errs...) +} + +// recordTargetReconcileCompleted increments the per-GitTarget reconcile counter +// once its snapshot decision has been made and the resulting write request +// submitted to the branch worker, tagged with the trigger that drove the pass. +// On a controller restart the new pod's counter starts at 0, so a per-pod +// `{pod=""} > 0` reading is the signal that the new pod's snapshot +// reconcile reached the git write path — paired with a drained +// BranchWorkerQueueDepth it proves the post-restart snapshot has fully landed. +// No-op until the counter is registered. +func (m *Manager) recordTargetReconcileCompleted(gitDest types.ResourceReference, trigger string) { + if telemetry.TargetReconcileCompletedTotal == nil { + return + } + telemetry.TargetReconcileCompletedTotal.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("gittarget_namespace", gitDest.Namespace), + attribute.String("gittarget_name", gitDest.Name), + attribute.String("trigger", trigger), + )) } func (m *Manager) completeReconciliationForTargets(targets []ruleSetSnapshotTarget, log logr.Logger) { diff --git a/internal/watch/rule_change_snapshot_test.go b/internal/watch/rule_change_snapshot_test.go index 44fda13d3..d97876fe9 100644 --- a/internal/watch/rule_change_snapshot_test.go +++ b/internal/watch/rule_change_snapshot_test.go @@ -25,10 +25,9 @@ import ( "github.com/go-logr/logr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - dynamicfake "k8s.io/client-go/dynamic/fake" + k8stypes "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -36,6 +35,7 @@ import ( "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/reconcile" "github.com/ConfigButler/gitops-reverser/internal/rulestore" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" "github.com/ConfigButler/gitops-reverser/internal/types" ) @@ -49,17 +49,17 @@ import ( // Each test asserts the contract directly: the snapshot count goes up when the // rule-set changes, regardless of whether the GVR set changed. -// makeRuleChangeTestManager constructs a Manager configured for these tests: -// fake controller-runtime client, real RuleStore, a trusted-but-minimal API -// resource catalog that does not serve configmaps (so rule resolution plans no -// informer GVRs and no real informers start), and a target GitTarget already in -// the fake client. +// makeRuleChangeTestManager constructs a single-GitTarget Manager for these +// tests: fake controller-runtime client, real RuleStore, the common catalog +// (which serves configmaps and secrets, so configmaps rules resolve to a real +// GVR and the target's effective watch plan is non-empty), and a target +// GitTarget already in the fake client. // -// The configmaps GVR is also pre-populated in activeInformers with a no-op -// cancel so compareGVRs sees the GVR as "already watched". This is the -// steady-state we want to model: an informer for that GVR is logically -// running, so adding a second rule for it should not cause an informer churn, -// but *should* re-trigger a snapshot. +// The configmaps GVR is pre-populated in activeInformers (cluster-wide) so +// compareGVRs reports added=0, removed=0 for it: the informer is logically +// already running, and no real informer is started. This isolates the snapshot +// *decision* (driven by the per-target effective-watch-plan hash) from informer +// lifecycle. func makeRuleChangeTestManager(t *testing.T) (*Manager, *rulestore.RuleStore) { t.Helper() @@ -82,36 +82,27 @@ func makeRuleChangeTestManager(t *testing.T) (*Manager, *rulestore.RuleStore) { Build() store := rulestore.NewStore() - configMapGVR := GVR{ - Group: "", - Version: "v1", - Resource: "configmaps", - Scope: configv1alpha1.ResourceScopeNamespaced, - } manager := &Manager{ - Client: fakeClient, - Log: logr.Discard(), - RuleStore: store, - // The catalog does not serve configmaps, so these tests exercise - // snapshot delivery after rule-set changes without planning real - // informers. - resourceCatalog: newSnapshotDeliveryTestCatalog(t), - discoveryClient: snapshotDeliveryTestDiscoveryClient(), + Client: fakeClient, + Log: logr.Discard(), + RuleStore: store, + resourceCatalog: newCommonTestCatalog(t), + discoveryClient: commonTestDiscoveryClient(), // Pretend the configmaps informer is already running cluster-wide, // so compareGVRs reports added=0, removed=0 for that GVR. No real // informers are started. activeInformers: map[GVR]map[string]context.CancelFunc{ - configMapGVR: {"": func() {}}, + namespacedGVR("configmaps"): {"": func() {}}, }, } return manager, store } -// configMapRuleForTarget builds a ClusterWatchRule that selects core/v1 -// configmaps, scoped at the given GitTarget (always in the shared "test-ns" -// namespace used across these tests). -func configMapRuleForTarget(name, gitTargetName string) configv1alpha1.ClusterWatchRule { +// clusterRuleForResource builds a ClusterWatchRule that selects a single core/v1 +// namespaced resource, scoped at the given GitTarget (always in the shared +// "test-ns" namespace used across these tests). +func clusterRuleForResource(name, gitTargetName, resource string) configv1alpha1.ClusterWatchRule { return configv1alpha1.ClusterWatchRule{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: configv1alpha1.ClusterWatchRuleSpec{ @@ -122,67 +113,72 @@ func configMapRuleForTarget(name, gitTargetName string) configv1alpha1.ClusterWa Rules: []configv1alpha1.ClusterResourceRule{{ APIGroups: []string{""}, APIVersions: []string{"v1"}, - Resources: []string{"configmaps"}, + Resources: []string{resource}, Scope: configv1alpha1.ResourceScopeNamespaced, }}, }, } } -// TestReconcileForRuleChange_AddingSecondRuleSameGVR_EmitsSnapshot exercises -// the central case. After the first rule is registered and reconciled, the -// informer set is steady-state. Adding a *second* rule that names the same GVR -// does not change the GVR set — only the rule-set changes. The contract: any -// rule-set change must still emit a snapshot for affected GitTargets, so -// resources that already match the new rule are backfilled to git. -func TestReconcileForRuleChange_AddingSecondRuleSameGVR_EmitsSnapshot(t *testing.T) { - manager, store := makeRuleChangeTestManager(t) - ctx := context.Background() +// configMapRule builds the standard single-target configmaps ClusterWatchRule. +func configMapRule() configv1alpha1.ClusterWatchRule { + return clusterRuleForResource("rule-1", "test-target", "configmaps") +} + +// TestReconcileForRuleChange_TargetGainsAlreadyWatchedGVR_Snapshots is the +// issue #146 case under the per-target plan model. A GVR can already be watched +// globally (because another target's rule covers it) while a particular target +// has never received it. When that target gains a rule for the GVR, no informer +// churn happens (added=0, removed=0 for that GVR), yet the target's effective +// watch plan grows — so it must still snapshot to backfill the already-present +// resources into its repo. +// +// Here target-a watches configmaps and target-b watches secrets. configmaps is +// already active. target-b then also starts watching configmaps: target-b's +// plan grows and it must be selected, even though no new configmaps informer is +// needed. +func TestReconcileForRuleChange_TargetGainsAlreadyWatchedGVR_Snapshots(t *testing.T) { + manager, store := makeTwoTargetRuleChangeManager(t) - // First rule: register, reconcile once. This brings the manager into a - // stable state where the configmaps informer is logically active. store.AddOrUpdateClusterWatchRule( - configMapRuleForTarget("rule-1", "test-target"), - "test-target", "test-ns", - "test-provider", "test-ns", - "main", "test-path", + clusterRuleForResource("rule-a", "target-a", "configmaps"), + "target-a", "test-ns", "test-provider", "test-ns", "main", "path-a", ) - require.NoError(t, manager.ReconcileForRuleChange(ctx)) - - // Reset counter so the assertion targets only the *second* reconcile. - manager.snapshotEmitCount.Store(0) + store.AddOrUpdateClusterWatchRule( + clusterRuleForResource("rule-b", "target-b", "secrets"), + "target-b", "test-ns", "test-provider", "test-ns", "main", "path-b", + ) + seedDeliveredBaseline(manager) - // Second rule for the same GVR. The GVR diff is empty; only the rule-set - // changes — ReconcileForRuleChange must still emit a snapshot. + // target-b now also watches configmaps — already watched by target-a, so no + // new informer, but target-b's plan grows. store.AddOrUpdateClusterWatchRule( - configMapRuleForTarget("rule-2", "test-target"), - "test-target", "test-ns", - "test-provider", "test-ns", - "main", "test-path", + clusterRuleForResource("rule-b-configmaps", "target-b", "configmaps"), + "target-b", "test-ns", "test-provider", "test-ns", "main", "path-b", ) - require.NoError(t, manager.ReconcileForRuleChange(ctx)) - assert.Positive(t, - manager.SnapshotEmitCount(), - "adding a ClusterWatchRule should emit a snapshot even when no new GVR is introduced; "+ - "existing matching resources must be backfilled to git") + require.NoError(t, manager.ReconcileForRuleChange(ctx())) + + assert.True(t, targetPending(manager, "target-b"), + "target-b gained coverage of an already-watched GVR; it must snapshot to backfill "+ + "those resources even though no informer churn occurred (issue #146)") + assert.False(t, targetPending(manager, "target-a"), + "target-a's plan is unchanged and must not snapshot") } -// TestReconcileForRuleChange_NarrowingSelectorOnExistingRule_EmitsSnapshot -// covers a sibling gap: editing a rule's selectors without changing its GVR. -// A common operator action is "I added a label selector to my existing rule to -// scope it down" — the GVR set does not change, but the matched resource set -// does. Same root cause as the test above. Uses a distinct GitTarget name -// from the prior test so the same helper exercises more than one input. -func TestReconcileForRuleChange_NarrowingSelectorOnExistingRule_EmitsSnapshot(t *testing.T) { +// TestReconcileForRuleChange_NarrowingOperationsOnExistingRule_Snapshots covers +// editing a rule's operations without changing its GVR. The resolved GVR is +// unchanged, but operations are part of the effective watch plan, so the plan +// hash changes and the target must snapshot. +func TestReconcileForRuleChange_NarrowingOperationsOnExistingRule_Snapshots(t *testing.T) { manager, store := makeRuleChangeTestManager(t) ctx := context.Background() - // Initial rule: selects all configmaps via narrow-target. - initial := configMapRuleForTarget("rule-1", "narrow-target") + // Initial rule: all operations on configmaps. + initial := configMapRule() store.AddOrUpdateClusterWatchRule( initial, - "narrow-target", "test-ns", + "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", ) @@ -190,15 +186,15 @@ func TestReconcileForRuleChange_NarrowingSelectorOnExistingRule_EmitsSnapshot(t manager.snapshotEmitCount.Store(0) - // Edit the rule — narrow its selectors. The GVR (core/v1/configmaps) is - // unchanged; only the rule body changes. - narrowed := initial + // Narrow operations to UPDATE only. The GVR (core/v1/configmaps) is + // unchanged; only the operation set in the plan changes. + narrowed := configMapRule() narrowed.Spec.Rules[0].Operations = []configv1alpha1.OperationType{ configv1alpha1.OperationUpdate, } store.AddOrUpdateClusterWatchRule( narrowed, - "narrow-target", "test-ns", + "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", ) @@ -206,8 +202,8 @@ func TestReconcileForRuleChange_NarrowingSelectorOnExistingRule_EmitsSnapshot(t assert.Positive(t, manager.SnapshotEmitCount(), - "editing a ClusterWatchRule should emit a snapshot when the matched resource set may change, "+ - "even if the GVR set is unchanged") + "narrowing a rule's operations changes the effective watch plan and must emit a snapshot, "+ + "even though the resolved GVR set is unchanged") } // watchRuleForTarget builds a namespaced WatchRule selecting core/v1 @@ -226,63 +222,55 @@ func watchRuleForTarget(name, gitTargetName, namespace string) configv1alpha1.Wa } } -// TestReconcileForRuleChange_AddingSecondWatchRuleSameGVR_EmitsSnapshot is the -// namespaced-WatchRule analogue of the ClusterWatchRule test above. The -// underlying code path is the same (rules feed into ComputeRequestedGVRs and -// collectAffectedGitTargets via the shared RuleStore), so the same gap -// applies. Documenting it as a separate test makes the regression coverage -// for both rule kinds explicit. -func TestReconcileForRuleChange_AddingSecondWatchRuleSameGVR_EmitsSnapshot(t *testing.T) { +// TestCurrentRuleSetSnapshots_NamespacedWatchRulePlanByNamespace exercises the +// namespace dimension of the effective watch plan for namespaced WatchRules, +// directly at the hash level (no informer lifecycle). A WatchRule watches its +// own namespace, so the same GVR in a different namespace is a different watch +// surface and must change the target's plan hash — while a redundant duplicate +// WatchRule in an already-watched namespace must not. +func TestCurrentRuleSetSnapshots_NamespacedWatchRulePlanByNamespace(t *testing.T) { manager, store := makeRuleChangeTestManager(t) - ctx := context.Background() + hashForTarget := func() uint64 { + for _, target := range manager.currentRuleSetSnapshots() { + if target.gitDest.Name == "test-target" { + return target.hash + } + } + t.Fatal("test-target not present in rule-set snapshots") + return 0 + } + + // WatchRule watching configmaps in ns-a. store.AddOrUpdateWatchRule( - watchRuleForTarget("rule-1", "test-target", "test-ns"), - "test-target", "test-ns", - "test-provider", "test-ns", - "main", "test-path", + watchRuleForTarget("rule-a", "test-target", "ns-a"), + "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", ) - require.NoError(t, manager.ReconcileForRuleChange(ctx)) - - manager.snapshotEmitCount.Store(0) + hashA := hashForTarget() + // Second WatchRule in a different namespace (ns-b): the plan grows. store.AddOrUpdateWatchRule( - watchRuleForTarget("rule-2", "test-target", "test-ns"), - "test-target", "test-ns", - "test-provider", "test-ns", - "main", "test-path", + watchRuleForTarget("rule-b", "test-target", "ns-b"), + "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", ) - require.NoError(t, manager.ReconcileForRuleChange(ctx)) + hashAB := hashForTarget() + assert.NotEqual(t, hashA, hashAB, + "a WatchRule adding the same GVR in a new namespace expands the watch surface and must change the plan hash") - assert.Positive(t, - manager.SnapshotEmitCount(), - "adding a second WatchRule should emit a snapshot even when no new GVR is introduced; "+ - "the bug is symmetric across WatchRule and ClusterWatchRule") + // Redundant duplicate WatchRule in an already-watched namespace (ns-a). + store.AddOrUpdateWatchRule( + watchRuleForTarget("rule-a-dup", "test-target", "ns-a"), + "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", + ) + assert.Equal(t, hashAB, hashForTarget(), + "a redundant duplicate WatchRule in an already-watched namespace must not change the plan hash") } -// TestReconcileForRuleChange_RestartLikeBootstrap_NoSnapshotDrops models the -// "restart with already-Ready GitTarget" scenario. The user-visible symptom: -// -// - On controller restart, GitTarget.Status.SnapshotSynced is already True -// from the prior run, so GitTargetReconciler takes the early-return at -// gittarget_controller.go:384 and creates no FolderReconciler. -// - WatchManager.Start runs its initial ReconcileForRuleChange before the -// GitTargetReconciler reaches the target, so the snapshot is emitted -// while no reconciler is registered. RouteClusterStateEvent / RouteRepoStateEvent -// find no receiver and silently drop the snapshot output. -// - Live update events still flow through (informer is running, the eventual -// stream registration brings it to LiveProcessing), so the user sees -// "updates write, but pre-existing resources never land in git." -// -// This test guards the fix: it emits a snapshot with no pre-registered -// FolderReconciler and asserts the drop counter stays at zero, so snapshot -// emission stays coordinated with reconciler registration. -// -// The setup populates activeInformers with a stale GVR and uses a catalog that -// does not serve configmaps, so the rule's GVR resolves to nothing and -// compareGVRs reports added=[], removed=[stale-gvr]. That bypasses the -// early-return without requiring any real informers to start. -func TestReconcileForRuleChange_RestartLikeBootstrap_NoSnapshotDrops(t *testing.T) { +// TestReconcileForRuleChange_NoReconcilerLeavesSnapshotPending verifies that +// rule-change reconciliation does not emit state events until a FolderReconciler +// exists. The target remains pending so MaybeReplaySnapshot can retry when the +// GitTarget reconciler registers the receiver. +func TestReconcileForRuleChange_NoReconcilerLeavesSnapshotPending(t *testing.T) { scheme := runtime.NewScheme() require.NoError(t, clientgoscheme.AddToScheme(scheme)) require.NoError(t, configv1alpha1.AddToScheme(scheme)) @@ -294,16 +282,6 @@ func TestReconcileForRuleChange_RestartLikeBootstrap_NoSnapshotDrops(t *testing. Branch: "main", Path: "test-path", }, - Status: configv1alpha1.GitTargetStatus{ - Conditions: []metav1.Condition{ - // Simulate the post-restart state that triggers the - // gittarget_controller.go:384 early-return. - {Type: "SnapshotSynced", Status: metav1.ConditionTrue, Reason: "Completed"}, - }, - }, - } - preexistingCM := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "preexisting", Namespace: "test-ns"}, } fakeK8s := fake.NewClientBuilder(). @@ -313,7 +291,7 @@ func TestReconcileForRuleChange_RestartLikeBootstrap_NoSnapshotDrops(t *testing. store := rulestore.NewStore() store.AddOrUpdateClusterWatchRule( - configMapRuleForTarget("rule-1", "test-target"), + configMapRule(), "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", @@ -324,24 +302,19 @@ func TestReconcileForRuleChange_RestartLikeBootstrap_NoSnapshotDrops(t *testing. Scope: configv1alpha1.ResourceScopeNamespaced, } manager := &Manager{ - Client: fakeK8s, - Log: logr.Discard(), - RuleStore: store, - dynamicClient: dynamicfake.NewSimpleDynamicClient(scheme, preexistingCM), - // The catalog does not serve configmaps, so the rule's GVR resolves to - // nothing and does not hit startInformersForGVRs. Combined with a stale - // entry in activeInformers, compareGVRs reports removed > 0 and the - // snapshot-emit path runs. - resourceCatalog: newSnapshotDeliveryTestCatalog(t), - discoveryClient: snapshotDeliveryTestDiscoveryClient(), + Client: fakeK8s, + Log: logr.Discard(), + RuleStore: store, + resourceCatalog: newCommonTestCatalog(t), + discoveryClient: commonTestDiscoveryClient(), activeInformers: map[GVR]map[string]context.CancelFunc{ - staleGVR: {"": func() {}}, + namespacedGVR("configmaps"): {"": func() {}}, + staleGVR: {"": func() {}}, }, } - // Wire EventRouter with empty-but-non-nil sub-components. This is the - // state immediately after WatchManager.Start: workers and reconcilers - // have not yet been registered by their respective controllers. + // Wire EventRouter with no registered FolderReconciler. This is the state + // before the GitTarget controller has created the per-target receiver. manager.EventRouter = &EventRouter{ WorkerManager: git.NewWorkerManager(fakeK8s, logr.Discard(), 0, types.SensitiveResourcePolicy{}), ReconcilerManager: reconcile.NewReconcilerManager(nil, logr.Discard()), @@ -353,17 +326,338 @@ func TestReconcileForRuleChange_RestartLikeBootstrap_NoSnapshotDrops(t *testing. require.NoError(t, manager.ReconcileForRuleChange(ctx())) - assert.Zero(t, - manager.EventRouter.SnapshotDeliveryDrops(), - "on a restart-like bootstrap (no FolderReconciler registered yet), the snapshot is emitted "+ - "but RouteClusterStateEvent/RouteRepoStateEvent silently drop it; the system must coordinate "+ - "emission with reconciler registration, or retry, so pre-existing matching resources land in git") + assert.True(t, targetPending(manager, "test-target"), + "target must stay pending until a FolderReconciler exists") + assert.Zero(t, manager.EventRouter.SnapshotDeliveryDrops(), + "state events must not be emitted before a FolderReconciler exists") +} + +// makeTwoTargetRuleChangeManager builds a Manager with two GitTargets, +// "target-a" and "target-b", both in "test-ns". +// +// The common catalog serves configmaps and secrets, so rules for them resolve +// to real GVRs and each target's effective watch plan is non-empty. Those +// resolved GVRs are pre-populated in activeInformers, so compareGVRs reports +// them as already watched (added=0) and no real informer is started. A stale +// apps/v1 deployments GVR that no rule wants is also pre-populated: the next +// ReconcileForRuleChange drops it (removed>0), modelling unrelated *global* +// informer churn happening concurrently — again without starting any informer. +// +// EventRouter is left nil. snapshotTargetsNeedingDelivery records every selected +// target in pendingRuleSetHash *before* emission, and with no EventRouter the +// emit path is a no-op that never marks targets delivered, so pendingRuleSetHash +// is a faithful record of which targets were selected for a rule-change snapshot. +func makeTwoTargetRuleChangeManager(t *testing.T) (*Manager, *rulestore.RuleStore) { + t.Helper() + + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, configv1alpha1.AddToScheme(scheme)) + + targetA := &configv1alpha1.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "target-a", Namespace: "test-ns"}, + Spec: configv1alpha1.GitTargetSpec{ + ProviderRef: configv1alpha1.GitProviderReference{Name: "test-provider"}, + Branch: "main", + Path: "path-a", + }, + } + targetB := &configv1alpha1.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "target-b", Namespace: "test-ns"}, + Spec: configv1alpha1.GitTargetSpec{ + ProviderRef: configv1alpha1.GitProviderReference{Name: "test-provider"}, + Branch: "main", + Path: "path-b", + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(targetA, targetB). + Build() + + staleGVR := GVR{ + Group: "apps", Version: "v1", Resource: "deployments", + Scope: configv1alpha1.ResourceScopeNamespaced, + } + manager := &Manager{ + Client: fakeClient, + Log: logr.Discard(), + RuleStore: rulestore.NewStore(), + resourceCatalog: newCommonTestCatalog(t), + discoveryClient: commonTestDiscoveryClient(), + activeInformers: map[GVR]map[string]context.CancelFunc{ + namespacedGVR("configmaps"): {"": func() {}}, + namespacedGVR("secrets"): {"": func() {}}, + staleGVR: {"": func() {}}, + }, + } + + return manager, manager.RuleStore +} + +// namespacedGVR is a terse constructor for a core/v1 namespaced GVR. +func namespacedGVR(resource string) GVR { + return GVR{Group: "", Version: "v1", Resource: resource, Scope: configv1alpha1.ResourceScopeNamespaced} +} + +// seedDeliveredBaseline registers the current rule set's hashes as already +// delivered, modelling the steady state where every target's snapshot is up to +// date. After this, only a target whose hash *changes* should be re-selected. +func seedDeliveredBaseline(m *Manager) { + for _, target := range m.currentRuleSetSnapshots() { + m.markRuleSetSnapshotDelivered(target) + } +} + +// targetPending reports whether the given GitTarget (in the shared "test-ns" +// namespace these tests use) was selected for a rule-change snapshot on the last +// reconcile. +func targetPending(m *Manager, name string) bool { + key := types.NewResourceReference(name, "test-ns").Key() + m.ruleSetSnapshotMu.Lock() + defer m.ruleSetSnapshotMu.Unlock() + _, ok := m.pendingRuleSetHash[key] + return ok +} + +func deliveredHash(m *Manager, name string) (uint64, bool) { + key := types.NewResourceReference(name, "test-ns").Key() + m.ruleSetSnapshotMu.Lock() + defer m.ruleSetSnapshotMu.Unlock() + hash, ok := m.lastDeliveredRuleSetHash[key] + return hash, ok +} + +func targetDelivered(m *Manager, name string) bool { + _, ok := deliveredHash(m, name) + return ok +} + +// TestReconcileForRuleChange_UnrelatedTargetNotSnapshotted_OnGlobalGVRChurn is +// the GitTarget-isolation invariant from +// docs/design/gittarget-isolation-on-rule-change.md. +// +// Two GitTargets, A and B, both watch configmaps and are in steady state (their +// snapshots already delivered). Then *only* target B's effective watch plan +// changes — it starts watching secrets too — while at the same moment unrelated +// global informer churn occurs (a stale GVR is dropped). Target A's plan is +// untouched. +// +// Expected: only B is selected for a rule-change snapshot; A keeps processing +// live events and is not dragged into RECONCILING. +// +// This is the regression the design doc set out to remove: previously +// ReconcileForRuleChange passed `len(added) > 0 || len(removed) > 0` as a global +// `force` flag that bypassed the per-target hash check and selected *every* +// target, so unrelated churn dragged A into a snapshot too. Snapshot selection +// is now purely per-target, so this passes. +func TestReconcileForRuleChange_UnrelatedTargetNotSnapshotted_OnGlobalGVRChurn(t *testing.T) { + manager, store := makeTwoTargetRuleChangeManager(t) + + // Both targets watch configmaps. + store.AddOrUpdateClusterWatchRule( + clusterRuleForResource("rule-a", "target-a", "configmaps"), + "target-a", "test-ns", "test-provider", "test-ns", "main", "path-a", + ) + store.AddOrUpdateClusterWatchRule( + clusterRuleForResource("rule-b", "target-b", "configmaps"), + "target-b", "test-ns", "test-provider", "test-ns", "main", "path-b", + ) + + // Steady state: both snapshots already delivered. + seedDeliveredBaseline(manager) + + // Change ONLY target B's effective plan: it now also watches secrets. + // A is untouched. + store.AddOrUpdateClusterWatchRule( + clusterRuleForResource("rule-b-secrets", "target-b", "secrets"), + "target-b", "test-ns", "test-provider", "test-ns", "main", "path-b", + ) + + require.NoError(t, manager.ReconcileForRuleChange(ctx())) + + assert.True(t, targetPending(manager, "target-b"), + "target B's effective watch plan changed (now watches secrets), so B must be selected") + assert.False(t, targetPending(manager, "target-a"), + "target A's effective watch plan did not change; unrelated global GVR churn must not "+ + "drag A into a rule-change snapshot. GitTargets are an isolation boundary — see "+ + "docs/design/gittarget-isolation-on-rule-change.md") +} + +// TestReconcileForRuleChange_RedundantDuplicateRule_DoesNotSnapshot pins the +// flip side of the isolation fix: because the hash is over the *effective watch +// plan* (resolved GVR + scope + operations), a second rule that resolves to the +// same surface a target already watches does not change the plan and must not +// trigger a no-op snapshot. This is what dropping source rule identity buys. +func TestReconcileForRuleChange_RedundantDuplicateRule_DoesNotSnapshot(t *testing.T) { + manager, store := makeTwoTargetRuleChangeManager(t) + + store.AddOrUpdateClusterWatchRule( + clusterRuleForResource("rule-a", "target-a", "configmaps"), + "target-a", "test-ns", "test-provider", "test-ns", "main", "path-a", + ) + seedDeliveredBaseline(manager) + + // A second rule that resolves to the exact same surface (configmaps) for the + // same target — a redundant duplicate. The effective plan is unchanged. + store.AddOrUpdateClusterWatchRule( + clusterRuleForResource("rule-a-dup", "target-a", "configmaps"), + "target-a", "test-ns", "test-provider", "test-ns", "main", "path-a", + ) + + require.NoError(t, manager.ReconcileForRuleChange(ctx())) + + assert.False(t, targetPending(manager, "target-a"), + "a redundant duplicate rule does not change the effective watch plan and must not snapshot") +} + +// TestSnapshotTargetsNeedingDelivery_PerTargetHashIsolatesTargets exercises the +// selection function directly (no full reconcile): with two targets in steady +// state and only B's plan changing, only B is returned. This is the unit-level +// guard for the per-target hash diff that snapshot selection now relies on. +func TestSnapshotTargetsNeedingDelivery_PerTargetHashIsolatesTargets(t *testing.T) { + manager, store := makeTwoTargetRuleChangeManager(t) + + store.AddOrUpdateClusterWatchRule( + clusterRuleForResource("rule-a", "target-a", "configmaps"), + "target-a", "test-ns", "test-provider", "test-ns", "main", "path-a", + ) + store.AddOrUpdateClusterWatchRule( + clusterRuleForResource("rule-b", "target-b", "configmaps"), + "target-b", "test-ns", "test-provider", "test-ns", "main", "path-b", + ) + seedDeliveredBaseline(manager) + + store.AddOrUpdateClusterWatchRule( + clusterRuleForResource("rule-b-secrets", "target-b", "secrets"), + "target-b", "test-ns", "test-provider", "test-ns", "main", "path-b", + ) + + selected := manager.snapshotTargetsNeedingDelivery() + + keyA := types.NewResourceReference("target-a", "test-ns").Key() + keyB := types.NewResourceReference("target-b", "test-ns").Key() + selectedKeys := make(map[string]struct{}, len(selected)) + for _, target := range selected { + selectedKeys[target.gitDest.Key()] = struct{}{} + } + + assert.Contains(t, selectedKeys, keyB, + "B (whose effective plan changed) is correctly selected") + assert.NotContains(t, selectedKeys, keyA, + "A (whose effective plan is unchanged) is correctly skipped") +} + +func TestSnapshotTargets_DiscoveryUnavailableKeepsDeliveredHash(t *testing.T) { + manager, store := makeRuleChangeTestManager(t) + + store.AddOrUpdateClusterWatchRule( + configMapRule(), + "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", + ) + seedDeliveredBaseline(manager) + baseline, ok := deliveredHash(manager, "test-target") + require.True(t, ok, "expected delivered baseline") + + manager.resourceCatalog = NewAPIResourceCatalog() + selected := manager.snapshotTargetsNeedingDelivery() + + assert.Empty(t, selected, "an unavailable catalog produces an empty plan, not a snapshot") + current, ok := deliveredHash(manager, "test-target") + require.True(t, ok, "empty plan must not prune delivered state while rules still exist") + assert.Equal(t, baseline, current) + assert.False(t, targetPending(manager, "test-target"), "empty plans are not pending snapshots") +} + +func TestSnapshotTargets_DiscoveryRecoveryDoesNotResnapshotUnchangedPlan(t *testing.T) { + manager, store := makeRuleChangeTestManager(t) + + store.AddOrUpdateClusterWatchRule( + configMapRule(), + "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", + ) + seedDeliveredBaseline(manager) + + manager.resourceCatalog = NewAPIResourceCatalog() + require.Empty(t, manager.snapshotTargetsNeedingDelivery()) + + manager.resourceCatalog = newCommonTestCatalog(t) + assert.Empty(t, manager.snapshotTargetsNeedingDelivery(), + "the recovered plan matches the remembered delivered hash, so no resnapshot is needed") +} + +func TestSnapshotTargets_RuleRemovalPrunesDeliveredHash(t *testing.T) { + manager, store := makeRuleChangeTestManager(t) + + store.AddOrUpdateClusterWatchRule( + configMapRule(), + "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", + ) + seedDeliveredBaseline(manager) + require.True(t, targetDelivered(manager, "test-target"), "expected delivered baseline") + + store.DeleteClusterWatchRule(k8stypes.NamespacedName{Name: "rule-1"}) + assert.Empty(t, manager.snapshotTargetsNeedingDelivery()) + assert.False(t, targetDelivered(manager, "test-target"), + "when the target truly has no rules, delivered state is pruned") +} + +// A transient emit failure (here: RequestRepoState fails because no worker is +// registered) must NOT mark the target delivered or bump the reconcile counter, +// and must be returned to the caller so it requeues and retries promptly. Before +// the fix the error was swallowed (counter and delivery were skipped silently), +// so the per-pod restart gate could wait out its 90s timeout on a one-off error. +func TestEmitSnapshotForRuleChange_TransientFailureReturnsErrorAndStaysPending(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, configv1alpha1.AddToScheme(scheme)) + + gitTarget := &configv1alpha1.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "test-target", Namespace: "test-ns"}, + Spec: configv1alpha1.GitTargetSpec{ + ProviderRef: configv1alpha1.GitProviderReference{Name: "test-provider"}, + Branch: "main", + Path: "test-path", + }, + } + fakeK8s := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gitTarget).Build() + + manager := &Manager{Client: fakeK8s, Log: logr.Discard()} + gitDest := types.NewResourceReference("test-target", "test-ns") + + // A FolderReconciler exists (so emission is attempted), but no BranchWorker is + // registered, so RequestRepoState fails with "no worker". + reconcilerManager := reconcile.NewReconcilerManager(nil, logr.Discard()) + reconcilerManager.CreateReconciler(ctx(), gitDest, nil) + manager.EventRouter = &EventRouter{ + WorkerManager: git.NewWorkerManager(fakeK8s, logr.Discard(), 0, types.SensitiveResourcePolicy{}), + ReconcilerManager: reconcilerManager, + WatchManager: manager, + Client: fakeK8s, + Log: logr.Discard(), + gitTargetStreams: map[string]*reconcile.GitTargetEventStream{}, + } + + target := ruleSetSnapshotTarget{gitDest: gitDest, hash: 0xABCD} + emitErr := manager.emitSnapshotForRuleChange(ctx(), logr.Discard(), []ruleSetSnapshotTarget{target}, "rule_change") + + require.Error(t, emitErr, "a failed emission must be returned so the caller requeues") + + // The target was not marked delivered, so the next reconcile retries it. + manager.ruleSetSnapshotMu.Lock() + delivered, ok := manager.lastDeliveredRuleSetHash[gitDest.Key()] + manager.ruleSetSnapshotMu.Unlock() + assert.False(t, ok && delivered == target.hash, "a failed target must stay pending, not be marked delivered") + + // The reconcile counter did not fire for this failed pass. + _, counted := telemetry.CollectInt64Sum(reader, targetReconcileCompletedMetric, + map[string]string{"gittarget_namespace": "test-ns", "gittarget_name": "test-target", "trigger": "rule_change"}) + assert.False(t, counted, "the reconcile counter must not fire when emission failed") } // ctx returns a background context. Wrapped for terse use in test setup. func ctx() context.Context { return context.Background() } - -// Compile-time sanity: corev1 is used by tests that may extend this file with -// pre-populated ConfigMap objects to assert backfill content. Today only the -// snapshot-emit counter is asserted; keep the import for cheap follow-up tests. -var _ = corev1.ConfigMap{} diff --git a/internal/watch/target_reconcile_metric_test.go b/internal/watch/target_reconcile_metric_test.go new file mode 100644 index 000000000..a67667bef --- /dev/null +++ b/internal/watch/target_reconcile_metric_test.go @@ -0,0 +1,64 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watch + +import ( + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +const targetReconcileCompletedMetric = "gitopsreverser_target_reconcile_completed_total" + +// recordTargetReconcileCompleted must increment the per-GitTarget counter and +// carry the gittarget_* and trigger labels. +func TestRecordTargetReconcileCompleted_IncrementsPerTrigger(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + m := &Manager{Log: logr.Discard()} + gitDest := types.NewResourceReference("my-target", "my-ns") + + m.recordTargetReconcileCompleted(gitDest, "rule_change") + + value, ok := telemetry.CollectInt64Sum(reader, targetReconcileCompletedMetric, + map[string]string{"gittarget_namespace": "my-ns", "gittarget_name": "my-target", "trigger": "rule_change"}) + require.True(t, ok, "expected a target_reconcile_completed_total sample") + assert.Equal(t, int64(1), value) + + // A second pass increments the counter, proving a delta over a baseline + // distinguishes a fresh reconcile from a stale latched value. + m.recordTargetReconcileCompleted(gitDest, "rule_change") + value, ok = telemetry.CollectInt64Sum(reader, targetReconcileCompletedMetric, + map[string]string{"gittarget_namespace": "my-ns", "gittarget_name": "my-target", "trigger": "rule_change"}) + require.True(t, ok) + assert.Equal(t, int64(2), value) + + // A different trigger is a distinct series, not folded into the first. + m.recordTargetReconcileCompleted(gitDest, "startup_replay") + value, ok = telemetry.CollectInt64Sum(reader, targetReconcileCompletedMetric, + map[string]string{"gittarget_namespace": "my-ns", "gittarget_name": "my-target", "trigger": "startup_replay"}) + require.True(t, ok) + assert.Equal(t, int64(1), value) +} diff --git a/internal/webhook/audit_joiner_deletecollection_test.go b/internal/webhook/audit_joiner_deletecollection_test.go new file mode 100644 index 000000000..90bf912dc --- /dev/null +++ b/internal/webhook/audit_joiner_deletecollection_test.go @@ -0,0 +1,250 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +// These tests pin the shallow-detector and joiner behaviour for aggregated-API +// `deletecollection`, using exact copies of real audit events captured by +// apiservice-audit-proxy (see its docs/examples). deletecollection is the +// awkward verb: the kube-apiserver's own (Lane A) event for an aggregated +// resource is hollow — it has the objectRef but no request/response body — and +// the only body comes from the proxy (Lane B). Two flavours matter: +// +// - "raw" — `kubectl delete flunder --all`; the proxy captures a real +// FlunderList responseObject. +// - "teardown" — namespace deletion; the namespace-controller issues the +// deletecollection and the proxy only ever sees a DeleteOptions +// requestObject (no list body at all). +// +// The teardown flavour is the trap: its only proxy body is DeleteOptions. These +// tests assert that such an event is still NOT dropped as shallow (the proxy +// body is enough to take the merge/emit path), while a deletecollection with no +// proxy body at all IS correctly dropped. + +import ( + "context" + "os" + "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" + "sigs.k8s.io/yaml" +) + +const ( + deletecollectionOfficialTeardownFixture = "testdata/audit-events/audit-deletecollection-official-teardown-hollow.json" + deletecollectionOfficialRawFixture = "testdata/audit-events/audit-deletecollection-official-raw-hollow.json" + deletecollectionProxyTeardownFixture = "testdata/audit-events/audit-deletecollection-proxy-teardown-deleteoptions.json" + deletecollectionProxyRawFixture = "testdata/audit-events/audit-deletecollection-proxy-raw-listbody.json" +) + +// loadAuditEventListItem reads an audit.k8s.io/v1 EventList fixture and returns +// its single item. yaml.Unmarshal handles JSON input and populates the +// runtime.Unknown Raw bodies the same way the live decoder does. +func loadAuditEventListItem(t *testing.T, path string) auditv1.Event { + t.Helper() + raw, err := os.ReadFile(path) + require.NoError(t, err) + + var list auditv1.EventList + require.NoError(t, yaml.Unmarshal(raw, &list)) + require.Len(t, list.Items, 1, "fixture %s must contain exactly one audit event", path) + return list.Items[0] +} + +// TestClassifyAuditEventQuality_DeleteCollectionFixtures documents how the +// shallow detector classifies each real deletecollection event in isolation. +func TestClassifyAuditEventQuality_DeleteCollectionFixtures(t *testing.T) { + tests := []struct { + name string + fixture string + source AuditSource + want AuditEventQuality + }{ + { + // Lane A, namespace teardown: hollow. Alone it has no body, so it is + // correctly identity-shallow — it must rely on a parked proxy body. + name: "official teardown (hollow) is identity-shallow", + fixture: deletecollectionOfficialTeardownFixture, + source: AuditSourceOfficial, + want: AuditEventQualityIdentityShallow, + }, + { + name: "official raw (hollow) is identity-shallow", + fixture: deletecollectionOfficialRawFixture, + source: AuditSourceOfficial, + want: AuditEventQualityIdentityShallow, + }, + { + // Lane B, namespace teardown: only a DeleteOptions requestObject. It + // must still register as a real body (Collection), not Malformed — + // otherwise the proxy contribution is discarded and the official + // event has nothing to merge and gets dropped. + name: "proxy teardown (DeleteOptions body) is collection, not malformed", + fixture: deletecollectionProxyTeardownFixture, + source: AuditSourceAdditional, + want: AuditEventQualityCollection, + }, + { + // Lane B, --all: a real FlunderList responseObject. + name: "proxy raw (FlunderList body) is collection", + fixture: deletecollectionProxyRawFixture, + source: AuditSourceAdditional, + want: AuditEventQualityCollection, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := loadAuditEventListItem(t, tt.fixture) + assert.Equal(t, tt.want, classifyAuditEventQuality(tt.source, &event)) + }) + } +} + +// TestRedisAuditEventJoiner_DeleteCollectionTeardownNotDroppedAsShallow is the +// core namespace-deletion check: the hollow official deletecollection plus the +// proxy's DeleteOptions-only body must be emitted, not dropped as shallow. +func TestRedisAuditEventJoiner_DeleteCollectionTeardownNotDroppedAsShallow(t *testing.T) { + mr := miniredis.RunT(t) + joiner := newTestJoiner(t, mr) + ctx := context.Background() + + proxy := loadAuditEventListItem(t, deletecollectionProxyTeardownFixture) + parked, err := decide(ctx, t, joiner, AuditSourceAdditional, &proxy) + require.NoError(t, err) + require.Equal(t, AuditJoinActionParked, parked.Action, "proxy teardown body must park") + + official := loadAuditEventListItem(t, deletecollectionOfficialTeardownFixture) + require.Equal(t, proxy.AuditID, official.AuditID, "fixtures must share an auditID to join") + + decision, err := decide(ctx, t, joiner, AuditSourceOfficial, &official) + require.NoError(t, err) + + require.Equal(t, AuditJoinActionEmit, decision.Action, + "a deletecollection with a parked proxy body must be emitted, not dropped as shallow") + // The objectRef identifies the collection to remove (resource + namespace, + // no name). The DeleteOptions body is intentionally not merged onto a + // deletecollection, so the emitted event stays bodyless and the consumer + // acts on the objectRef. + require.NotNil(t, decision.Event.ObjectRef) + assert.Equal(t, "flunders", decision.Event.ObjectRef.Resource) + assert.Equal(t, "example-teardown", decision.Event.ObjectRef.Namespace) + assert.Empty(t, decision.Event.ObjectRef.Name) + assert.Nil(t, decision.Event.RequestObject, "DeleteOptions must not be merged onto a deletecollection") + assert.Nil(t, decision.Event.ResponseObject) +} + +// TestRedisAuditEventJoiner_DeleteCollectionRawNotDroppedAsShallow covers the +// `kubectl delete flunder --all` flavour, where the proxy supplies a real +// FlunderList responseObject. +func TestRedisAuditEventJoiner_DeleteCollectionRawNotDroppedAsShallow(t *testing.T) { + mr := miniredis.RunT(t) + joiner := newTestJoiner(t, mr) + ctx := context.Background() + + proxy := loadAuditEventListItem(t, deletecollectionProxyRawFixture) + parked, err := decide(ctx, t, joiner, AuditSourceAdditional, &proxy) + require.NoError(t, err) + require.Equal(t, AuditJoinActionParked, parked.Action, "proxy raw body must park") + + official := loadAuditEventListItem(t, deletecollectionOfficialRawFixture) + require.Equal(t, proxy.AuditID, official.AuditID, "fixtures must share an auditID to join") + + decision, err := decide(ctx, t, joiner, AuditSourceOfficial, &official) + require.NoError(t, err) + + require.Equal(t, AuditJoinActionEmit, decision.Action, + "a deletecollection with a parked proxy body must be emitted, not dropped as shallow") + require.NotNil(t, decision.Event.ObjectRef) + assert.Equal(t, "flunders", decision.Event.ObjectRef.Resource) + assert.Equal(t, "example-raw", decision.Event.ObjectRef.Namespace) + assert.Empty(t, decision.Event.ObjectRef.Name) +} + +// TestRedisAuditEventJoiner_DeleteCollectionWithoutProxyBodyIsShallow is the +// negative control: with no proxy (Lane B) body parked, the hollow official +// deletecollection has nothing to act on and is correctly dropped as shallow. +// This is the pre-0.6.0 / proxy-missing behaviour and the boundary the proxy +// fix moves the teardown case across. +func TestRedisAuditEventJoiner_DeleteCollectionWithoutProxyBodyIsShallow(t *testing.T) { + mr := miniredis.RunT(t) + joiner := newTestJoiner(t, mr) + + official := loadAuditEventListItem(t, deletecollectionOfficialTeardownFixture) + decision, err := decide(context.Background(), t, joiner, AuditSourceOfficial, &official) + require.NoError(t, err) + + assert.Equal(t, AuditJoinActionDrop, decision.Action, + "a hollow deletecollection with no proxy body must be dropped as shallow") + assert.False(t, mr.Exists(bodyKey(string(official.AuditID))), "shallow official must not park") +} + +// TestClassifyAuditEventQuality_DeleteCollectionBodyShapeIndependent guards the +// fact that the responseObject a backend returns for a deletecollection is +// aggregated-API-specific: wardle returns a FlunderList, but another apiserver +// may return a metav1.Status, an empty object, or only echo a DeleteOptions +// requestObject. The detector keys off "is any body present" (hasAuditV1ObjectBody) +// and the objectRef, never the body shape — so every non-empty variant must +// classify as Collection, and only a truly bodyless event is shallow. The base +// event is the real hollow official teardown fixture with its body swapped, so +// these stay synthetic-shape variants of a real event rather than invented data. +func TestClassifyAuditEventQuality_DeleteCollectionBodyShapeIndependent(t *testing.T) { + statusBody := `{"kind":"Status","apiVersion":"v1","status":"Success","code":200}` + listBody := `{"kind":"FlunderList","apiVersion":"wardle.example.com/v1alpha1","items":[]}` + deleteOptionsBody := `{"kind":"DeleteOptions","apiVersion":"v1","propagationPolicy":"Background"}` + + tests := []struct { + name string + requestRaw string + responseRaw string + want AuditEventQuality + }{ + {name: "metav1.Status response body", responseRaw: statusBody, want: AuditEventQualityCollection}, + {name: "List response body", responseRaw: listBody, want: AuditEventQualityCollection}, + {name: "empty-object response body", responseRaw: `{}`, want: AuditEventQualityCollection}, + {name: "DeleteOptions request body only", requestRaw: deleteOptionsBody, want: AuditEventQualityCollection}, + // A bodyless proxy (additional-source) contribution is useless and is + // discarded as Malformed before it can park, so the hollow official has + // nothing to merge and is later dropped as shallow. Either way: not + // actionable. (For the official source the same empty event is + // IdentityShallow; the label differs, the "dropped" outcome does not.) + {name: "no body at all is not actionable", want: AuditEventQualityMalformed}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Start from a real hollow official deletecollection (objectRef set, + // no body) and attach only the body shape under test. + event := loadAuditEventListItem(t, deletecollectionOfficialTeardownFixture) + require.Nil(t, event.RequestObject) + require.Nil(t, event.ResponseObject) + if tt.requestRaw != "" { + event.RequestObject = &runtime.Unknown{Raw: []byte(tt.requestRaw)} + } + if tt.responseRaw != "" { + event.ResponseObject = &runtime.Unknown{Raw: []byte(tt.responseRaw)} + } + + assert.Equal(t, tt.want, classifyAuditEventQuality(AuditSourceAdditional, &event)) + }) + } +} diff --git a/internal/webhook/testdata/audit-events/audit-deletecollection-official-raw-hollow.json b/internal/webhook/testdata/audit-events/audit-deletecollection-official-raw-hollow.json new file mode 100644 index 000000000..6870b4365 --- /dev/null +++ b/internal/webhook/testdata/audit-events/audit-deletecollection-official-raw-hollow.json @@ -0,0 +1,46 @@ +{ + "apiVersion": "audit.k8s.io/v1", + "kind": "EventList", + "metadata": {}, + "items": [ + { + "level": "RequestResponse", + "auditID": "5b41e1ef-d0a5-4df0-b0bd-d5c317cbad8b", + "stage": "ResponseComplete", + "requestURI": "/apis/wardle.example.com/v1alpha1/namespaces/example-raw/flunders", + "verb": "deletecollection", + "user": { + "username": "system:admin", + "groups": [ + "system:masters", + "system:authenticated" + ], + "extra": { + "authentication.kubernetes.io/credential-id": [ + "X509SHA256=fb5afb7b73ca3bc9fac7563a91d609a917d56708dd1cb3bbc32abaab6cd48fd4" + ] + } + }, + "sourceIPs": [ + "172.20.0.6" + ], + "userAgent": "kubectl/v1.36.1 (linux/amd64) kubernetes/7569396", + "objectRef": { + "resource": "flunders", + "namespace": "example-raw", + "apiGroup": "wardle.example.com", + "apiVersion": "v1alpha1" + }, + "responseStatus": { + "metadata": {}, + "code": 200 + }, + "requestReceivedTimestamp": "2026-06-02T11:16:29.161836Z", + "stageTimestamp": "2026-06-02T11:16:29.174537Z", + "annotations": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "" + } + } + ] +} diff --git a/internal/webhook/testdata/audit-events/audit-deletecollection-official-teardown-hollow.json b/internal/webhook/testdata/audit-events/audit-deletecollection-official-teardown-hollow.json new file mode 100644 index 000000000..8bfbbcc5b --- /dev/null +++ b/internal/webhook/testdata/audit-events/audit-deletecollection-official-teardown-hollow.json @@ -0,0 +1,48 @@ +{ + "apiVersion": "audit.k8s.io/v1", + "kind": "EventList", + "metadata": {}, + "items": [ + { + "level": "RequestResponse", + "auditID": "097dcb8c-c4d1-4e72-94f9-794cc2681e24", + "stage": "ResponseComplete", + "requestURI": "/apis/wardle.example.com/v1alpha1/namespaces/example-teardown/flunders", + "verb": "deletecollection", + "user": { + "username": "system:serviceaccount:kube-system:namespace-controller", + "uid": "9766626a-31eb-44ee-9dfe-59242cbf2a75", + "groups": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ], + "extra": { + "authentication.kubernetes.io/credential-id": [ + "JTI=a124b540-01b8-444a-8f37-791b30052ae4" + ] + } + }, + "sourceIPs": [ + "127.0.0.1" + ], + "userAgent": "k3s/v1.35.2+k3s1 (linux/amd64) kubernetes/13563fe/system:serviceaccount:kube-system:namespace-controller", + "objectRef": { + "resource": "flunders", + "namespace": "example-teardown", + "apiGroup": "wardle.example.com", + "apiVersion": "v1alpha1" + }, + "responseStatus": { + "metadata": {}, + "code": 200 + }, + "requestReceivedTimestamp": "2026-06-02T11:16:44.602001Z", + "stageTimestamp": "2026-06-02T11:16:44.605469Z", + "annotations": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:controller:namespace-controller\" of ClusterRole \"system:controller:namespace-controller\" to ServiceAccount \"namespace-controller/kube-system\"" + } + } + ] +} diff --git a/internal/webhook/testdata/audit-events/audit-deletecollection-proxy-raw-listbody.json b/internal/webhook/testdata/audit-events/audit-deletecollection-proxy-raw-listbody.json new file mode 100644 index 000000000..0a0bc22fb --- /dev/null +++ b/internal/webhook/testdata/audit-events/audit-deletecollection-proxy-raw-listbody.json @@ -0,0 +1,129 @@ +{ + "apiVersion": "audit.k8s.io/v1", + "kind": "EventList", + "metadata": {}, + "items": [ + { + "level": "RequestResponse", + "auditID": "5b41e1ef-d0a5-4df0-b0bd-d5c317cbad8b", + "stage": "ResponseComplete", + "requestURI": "/apis/wardle.example.com/v1alpha1/namespaces/example-raw/flunders", + "verb": "deletecollection", + "user": { + "username": "system:admin", + "groups": [ + "system:masters", + "system:authenticated" + ], + "extra": { + "authentication.kubernetes.io/credential-id": [ + "X509SHA256=fb5afb7b73ca3bc9fac7563a91d609a917d56708dd1cb3bbc32abaab6cd48fd4" + ] + } + }, + "sourceIPs": [ + "172.20.0.6", + "10.42.0.0" + ], + "userAgent": "kubectl/v1.36.1 (linux/amd64) kubernetes/7569396", + "objectRef": { + "resource": "flunders", + "namespace": "example-raw", + "apiGroup": "wardle.example.com", + "apiVersion": "v1alpha1", + "resourceVersion": "50" + }, + "responseStatus": { + "metadata": {}, + "code": 200 + }, + "responseObject": { + "kind": "FlunderList", + "apiVersion": "wardle.example.com/v1alpha1", + "metadata": { + "resourceVersion": "50" + }, + "items": [ + { + "metadata": { + "name": "member-0", + "namespace": "example-raw", + "uid": "e1b076ff-f430-4b82-ad7b-c170c4095fe3", + "resourceVersion": "49", + "creationTimestamp": "2026-06-02T11:16:29Z", + "annotations": { + "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"wardle.example.com/v1alpha1\",\"kind\":\"Flunder\",\"metadata\":{\"annotations\":{},\"name\":\"member-0\",\"namespace\":\"example-raw\"},\"spec\":{\"reference\":\"example\"}}\n" + }, + "managedFields": [ + { + "manager": "kubectl-client-side-apply", + "operation": "Update", + "apiVersion": "wardle.example.com/v1alpha1", + "time": "2026-06-02T11:16:29Z", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:annotations": { + ".": {}, + "f:kubectl.kubernetes.io/last-applied-configuration": {} + } + }, + "f:spec": { + "f:reference": {}, + "f:referenceType": {} + } + } + } + ] + }, + "spec": { + "reference": "example", + "referenceType": "Flunder" + }, + "status": {} + }, + { + "metadata": { + "name": "member-1", + "namespace": "example-raw", + "uid": "5ce03312-bab3-4e72-af37-e0ff893a7b76", + "resourceVersion": "50", + "creationTimestamp": "2026-06-02T11:16:29Z", + "annotations": { + "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"wardle.example.com/v1alpha1\",\"kind\":\"Flunder\",\"metadata\":{\"annotations\":{},\"name\":\"member-1\",\"namespace\":\"example-raw\"},\"spec\":{\"reference\":\"example\"}}\n" + }, + "managedFields": [ + { + "manager": "kubectl-client-side-apply", + "operation": "Update", + "apiVersion": "wardle.example.com/v1alpha1", + "time": "2026-06-02T11:16:29Z", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:annotations": { + ".": {}, + "f:kubectl.kubernetes.io/last-applied-configuration": {} + } + }, + "f:spec": { + "f:reference": {}, + "f:referenceType": {} + } + } + } + ] + }, + "spec": { + "reference": "example", + "referenceType": "Flunder" + }, + "status": {} + } + ] + }, + "requestReceivedTimestamp": "2026-06-02T11:16:29.162706Z", + "stageTimestamp": "2026-06-02T11:16:29.174126Z" + } + ] +} diff --git a/internal/webhook/testdata/audit-events/audit-deletecollection-proxy-teardown-deleteoptions.json b/internal/webhook/testdata/audit-events/audit-deletecollection-proxy-teardown-deleteoptions.json new file mode 100644 index 000000000..8c2ff24a1 --- /dev/null +++ b/internal/webhook/testdata/audit-events/audit-deletecollection-proxy-teardown-deleteoptions.json @@ -0,0 +1,49 @@ +{ + "apiVersion": "audit.k8s.io/v1", + "kind": "EventList", + "metadata": {}, + "items": [ + { + "level": "RequestResponse", + "auditID": "097dcb8c-c4d1-4e72-94f9-794cc2681e24", + "stage": "ResponseComplete", + "requestURI": "/apis/wardle.example.com/v1alpha1/namespaces/example-teardown/flunders", + "verb": "deletecollection", + "user": { + "username": "system:serviceaccount:kube-system:namespace-controller", + "groups": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ], + "extra": { + "authentication.kubernetes.io/credential-id": [ + "JTI=a124b540-01b8-444a-8f37-791b30052ae4" + ] + } + }, + "sourceIPs": [ + "127.0.0.1", + "10.42.0.0" + ], + "userAgent": "k3s/v1.35.2+k3s1 (linux/amd64) kubernetes/13563fe/system:serviceaccount:kube-system:namespace-controller", + "objectRef": { + "resource": "flunders", + "namespace": "example-teardown", + "apiGroup": "wardle.example.com", + "apiVersion": "v1alpha1" + }, + "responseStatus": { + "metadata": {}, + "code": 200 + }, + "requestObject": { + "kind": "DeleteOptions", + "apiVersion": "v1", + "propagationPolicy": "Background" + }, + "requestReceivedTimestamp": "2026-06-02T11:16:44.602749Z", + "stageTimestamp": "2026-06-02T11:16:44.605108Z" + } + ] +} diff --git a/test/e2e/E2E_DEBUGGING.md b/test/e2e/E2E_DEBUGGING.md index 4b573cc82..9c37dbf8c 100644 --- a/test/e2e/E2E_DEBUGGING.md +++ b/test/e2e/E2E_DEBUGGING.md @@ -126,6 +126,33 @@ Kind Cluster └─ Controller pods (2 replicas, HTTPS metrics) ``` +## Spec Timings + +Every e2e target writes a Ginkgo JSON report to +`.stamps/cluster///ginkgo-report-.json` (e.g. +`ginkgo-report-smoke.json` for `task test-e2e`). Render the per-spec +duration table from the latest run with: + +```bash +go run ./test/e2e/tools/spec-timings \ + .stamps/cluster/k3d-gitops-reverser-test-e2e/gitops-reverser/ginkgo-report-smoke.json +``` + +The companion `test/e2e/tools/ts` reads stdin and prefixes each line with +elapsed seconds + wallclock. Useful for decorating cold-run logs that +don't have structured JSON yet: + +```bash +task test-e2e 2>&1 | go run ./test/e2e/tools/ts | tee /tmp/e2e.log +``` + +If output looks batched (long pauses then a burst of timestamps), prefix +with `stdbuf -oL -eL` to force line-buffering on `task`/`go test`: + +```bash +stdbuf -oL -eL task test-e2e 2>&1 | go run ./test/e2e/tools/ts | tee /tmp/e2e.log +``` + ## Debugging Failed Tests 1. **Ensure port-forwards are running:** diff --git a/test/e2e/Taskfile.yml b/test/e2e/Taskfile.yml index 1852151c1..720d5a5e8 100644 --- a/test/e2e/Taskfile.yml +++ b/test/e2e/Taskfile.yml @@ -91,8 +91,21 @@ vars: FLUX_INSTANCE_FILE: '{{.FLUX_INSTANCE_FILE | default "test/e2e/setup/flux-operator/flux-instance.yaml"}}' FLUX_OPERATOR_DIR: '{{.FLUX_OPERATOR_DIR | default "test/e2e/setup/flux-operator"}}' FLUX_WAIT_TIMEOUT: '{{.FLUX_WAIT_TIMEOUT | default "500s"}}' - FLUX_SERVICES_WAIT_TIMEOUT: '{{.FLUX_SERVICES_WAIT_TIMEOUT | default "600s"}}' + FLUX_SERVICES_WAIT_TIMEOUT: '{{.FLUX_SERVICES_WAIT_TIMEOUT | default "120s"}}' E2E_GO_TEST_TIMEOUT: '{{.E2E_GO_TEST_TIMEOUT | default "15m"}}' + ALLURE_RESULTS_DIR: '{{.ALLURE_RESULTS_DIR | default ".stamps/allure-results"}}' + ALLURE_REPORT_DIR: '{{.ALLURE_REPORT_DIR | default ".stamps/allure-report"}}' + ALLURE_CLI_DIR: '{{.ALLURE_CLI_DIR | default ".stamps/allure-cli"}}' + ALLURE_VERSION: '{{.ALLURE_VERSION | default "2.42.0"}}' + ALLURE_CONTAINER_IMAGE: '{{.ALLURE_CONTAINER_IMAGE | default "eclipse-temurin:17-jre"}}' + ALLURE_PORT: '{{.ALLURE_PORT | default "19081"}}' + # Number of parallel Ginkgo processes for the e2e suite. >1 enables Ginkgo + # parallelism (Phase 2.5 of docs/design/e2e-speedup-plan.md). Files that touch + # shared cluster-scoped state are marked Serial (see docs/design/e2e-serial-registry.md). + # Default 4 locally: GitTarget isolation on rule changes is now per-target + # (docs/design/gittarget-isolation-on-rule-change.md), so higher parallelism no + # longer trips cross-target snapshot coupling. + E2E_GINKGO_PROCS: '{{.E2E_GINKGO_PROCS | default "4"}}' FLUX_SERVICES_DIR: '{{.FLUX_SERVICES_DIR | default "test/e2e/setup/flux"}}' DEMO_MANIFESTS_DIR: 'test/e2e/setup/demo-only' DEMO_TUNNEL_CREDENTIALS: '{{.DEMO_MANIFESTS_DIR}}/cloudflared-public/tunnel-credentials.yaml' @@ -105,16 +118,11 @@ vars: tasks: prepare-e2e: desc: Prepare E2E prerequisites for Go tests + # portforward-ensure runs *after* _prepare-e2e-ready, not in parallel + # with it: _webhook-tls-ready inside the prepare graph restarts the + # k3d server node, which would kill any kubectl port-forwards started + # concurrently with it. cmds: - - task: install - - task: _project-image-ready - - task: _image-loaded - - task: _controller-deployed - - task: _age-key - - task: _sops-secret-yaml - - task: _sops-secret-applied - - task: _webhook-tls-ready - - task: _aggregated-api-ready - task: _prepare-e2e-ready - task: portforward-ensure @@ -126,11 +134,9 @@ tasks: portforward-ensure: desc: Ensure port-forwards are running + deps: + - _services-ready cmds: - - task: _cluster-ready - - task: _flux-installed - - task: _flux-setup-ready - - task: _services-ready - | export E2E_KUBECONTEXT="{{.CTX}}" export GITEA_PORT="{{.GITEA_PORT}}" @@ -142,74 +148,146 @@ tasks: install: desc: Install using the configured INSTALL_MODE cmds: + # Validate INSTALL_MODE before dispatching. The actual dispatch uses + # in-process task: invocation so the full dependency graph stays in + # one go-task run (the previous shell-subprocess form lost CTX and + # other task vars when not also exported as env). - | case "{{.INSTALL_MODE}}" in - helm) - task install-helm - ;; - plain-manifests-file) - task install-plain-manifests-file - ;; - config-dir) - task install-config-dir - ;; + helm|config-dir|plain-manifests-file) ;; *) echo "ERROR: INSTALL_MODE must be one of [config-dir helm plain-manifests-file]" >&2 exit 2 ;; esac + - task: 'install-{{.INSTALL_MODE}}' install-helm: - deps: - - helm-sync desc: Install via Helm - cmds: - - task: _cluster-ready - - task: _flux-installed - - task: _flux-setup-ready - - task: _services-ready - - task: _install-helm + deps: + - _install-helm install-plain-manifests-file: desc: Install via dist/install.yaml deps: - - dist-install - cmds: - - task: _cluster-ready - - task: _flux-installed - - task: _flux-setup-ready - - task: _services-ready - - task: _install-plain-manifests-file + - _install-plain-manifests-file install-config-dir: desc: Install via config directory - cmds: - - task: _cluster-ready - - task: _flux-installed - - task: _flux-setup-ready - - task: _services-ready - - task: manifests - - task: _install-config-dir + deps: + - _install-config-dir test-e2e: desc: Run the smoke e2e suite + # Ginkgo parallelism (--procs) is orchestrated by the ginkgo CLI, not by + # `go test`. Run it via `go run` so the CLI version matches the pinned + # github.com/onsi/ginkgo/v2 module (a globally-installed ginkgo can differ + # and refuse to run on a version mismatch). cmds: - | export CTX="{{.CTX}}" export INSTALL_MODE="{{.INSTALL_MODE}}" export NAMESPACE="{{.NAMESPACE}}" export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt" - go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=smoke + go run github.com/onsi/ginkgo/v2/ginkgo \ + --procs={{.E2E_GINKGO_PROCS}} --timeout="{{.E2E_GO_TEST_TIMEOUT}}" -v \ + --label-filter=smoke \ + --output-dir="{{.CS}}/{{.NAMESPACE}}" --json-report=ginkgo-report-smoke.json \ + ./test/e2e/ test-e2e-full: desc: Run the full e2e suite, including slow and specialized scenarios + vars: + # 46 specs at ~22s avg + ~6m cold prepare ≈ 25m; the smoke-tuned 15m default + # times out mid-run and gives a goroutine dump instead of a useful report. + E2E_FULL_TIMEOUT: '{{.E2E_FULL_TIMEOUT | default "30m"}}' cmds: - | export CTX="{{.CTX}}" export INSTALL_MODE="{{.INSTALL_MODE}}" export NAMESPACE="{{.NAMESPACE}}" export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt" - go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v + go run github.com/onsi/ginkgo/v2/ginkgo \ + --procs={{.E2E_GINKGO_PROCS}} --timeout="{{.E2E_FULL_TIMEOUT}}" -v \ + --output-dir="{{.CS}}/{{.NAMESPACE}}" --json-report=ginkgo-report-full.json \ + ./test/e2e/ + + allure-e2e-results: + desc: Convert local e2e Ginkgo JSON reports into Allure results + aliases: + - e2e-allure-results + cmds: + - | + reports_file="$(mktemp)" + trap 'rm -f "${reports_file}"' EXIT + if [ -n "${ALLURE_GINKGO_REPORTS:-}" ]; then + printf '%s\n' ${ALLURE_GINKGO_REPORTS} | sort > "${reports_file}" + else + reports_root="${ALLURE_GINKGO_REPORTS_DIR:-{{.CS}}/{{.NAMESPACE}}}" + if [ -d "${reports_root}" ]; then + if [ "${ALLURE_INCLUDE_ALL_GINKGO_REPORTS:-}" = "true" ]; then + find "${reports_root}" -maxdepth 1 -name "ginkgo-report-*.json" -type f | sort > "${reports_file}" + else + find "${reports_root}" -maxdepth 1 -name "ginkgo-report-*.json" -type f -printf '%T@ %p\n' \ + | sort -rn \ + | sed -n '1s/^[^ ]* //p' > "${reports_file}" + fi + fi + fi + if [ ! -s "${reports_file}" ]; then + echo "No Ginkgo JSON reports found for the current e2e context/namespace. Run an e2e task first." >&2 + exit 1 + fi + + rm -rf "{{.ALLURE_RESULTS_DIR}}" + mkdir -p "{{.ALLURE_RESULTS_DIR}}" + xargs go run ./test/e2e/tools/ginkgo-allure --output-dir "{{.ALLURE_RESULTS_DIR}}" < "${reports_file}" + echo "Wrote Allure results to {{.ALLURE_RESULTS_DIR}}" + + allure-e2e-report: + desc: Generate local Allure HTML from e2e Ginkgo reports + aliases: + - e2e-allure-report + cmds: + - task: allure-e2e-results + - | + if ! command -v docker >/dev/null 2>&1; then + echo "Docker is required to render Allure without adding Java to the devcontainer." >&2 + exit 1 + fi + + workspace_host="${HOST_PROJECT_PATH:-$(pwd)}" + workspace_container="$(pwd)" + mkdir -p "{{.ALLURE_CLI_DIR}}" + docker run --rm \ + --user "$(id -u):$(id -g)" \ + -e HOME=/tmp \ + -e ALLURE_CLI_DIR="{{.ALLURE_CLI_DIR}}" \ + -e ALLURE_REPORT_DIR="{{.ALLURE_REPORT_DIR}}" \ + -e ALLURE_RESULTS_DIR="{{.ALLURE_RESULTS_DIR}}" \ + -e ALLURE_VERSION="{{.ALLURE_VERSION}}" \ + -v "${workspace_host}:${workspace_container}" \ + -w "${workspace_container}" \ + "{{.ALLURE_CONTAINER_IMAGE}}" \ + sh -eu -c ' + allure_home="${ALLURE_CLI_DIR}/allure-${ALLURE_VERSION}" + if [ ! -x "${allure_home}/bin/allure" ]; then + rm -rf "${allure_home}" + mkdir -p "${ALLURE_CLI_DIR}" + curl -fsSL "https://github.com/allure-framework/allure2/releases/download/${ALLURE_VERSION}/allure-${ALLURE_VERSION}.tgz" \ + | tar -xz -C "${ALLURE_CLI_DIR}" + fi + "${allure_home}/bin/allure" generate "${ALLURE_RESULTS_DIR}" --clean -o "${ALLURE_REPORT_DIR}" + ' + echo "Wrote Allure HTML report to {{.ALLURE_REPORT_DIR}}" + + allure-e2e-open: + desc: Generate and serve the local e2e Allure report + aliases: + - e2e-allure-open + cmds: + - task: allure-e2e-report + - python3 -m http.server "{{.ALLURE_PORT}}" --bind 0.0.0.0 --directory "{{.ALLURE_REPORT_DIR}}" test-e2e-manager: desc: Run manager-focused e2e scenarios @@ -219,7 +297,11 @@ tasks: export INSTALL_MODE="{{.INSTALL_MODE}}" export NAMESPACE="{{.NAMESPACE}}" export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt" - go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=manager + go run github.com/onsi/ginkgo/v2/ginkgo \ + --procs={{.E2E_GINKGO_PROCS}} --timeout="{{.E2E_GO_TEST_TIMEOUT}}" -v \ + --label-filter=manager \ + --output-dir="{{.CS}}/{{.NAMESPACE}}" --json-report=ginkgo-report-manager.json \ + ./test/e2e/ test-e2e-signing: desc: Run commit-signing e2e scenarios @@ -229,7 +311,8 @@ tasks: export INSTALL_MODE="{{.INSTALL_MODE}}" export NAMESPACE="{{.NAMESPACE}}" export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt" - go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=signing + go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=signing \ + -ginkgo.json-report="{{.CS}}/{{.NAMESPACE}}/ginkgo-report-signing.json" test-image-refresh: desc: Validate the image-refresh dependency chain @@ -239,7 +322,8 @@ tasks: export INSTALL_MODE="{{.INSTALL_MODE}}" export NAMESPACE="{{.NAMESPACE}}" export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt" - go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=image-refresh + go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=image-refresh \ + -ginkgo.json-report="{{.CS}}/{{.NAMESPACE}}/ginkgo-report-image-refresh.json" test-e2e-quickstart-helm: desc: Run quickstart smoke test with Helm install @@ -251,7 +335,8 @@ tasks: export E2E_ENABLE_QUICKSTART_FRAMEWORK=true export E2E_QUICKSTART_MODE=helm export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt" - go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework + go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework \ + -ginkgo.json-report="{{.CS}}/{{.NAMESPACE}}/ginkgo-report-quickstart-helm.json" test-e2e-quickstart-manifest: desc: Run quickstart smoke test with manifest install @@ -263,7 +348,8 @@ tasks: export E2E_ENABLE_QUICKSTART_FRAMEWORK=true export E2E_QUICKSTART_MODE=plain-manifests-file export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt" - go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework + go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework \ + -ginkgo.json-report="{{.CS}}/{{.NAMESPACE}}/ginkgo-report-quickstart-manifest.json" playground-bootstrap: desc: Bootstrap a reusable repo-backed playground @@ -435,16 +521,6 @@ tasks: fi "{{.KUBECTL}}" --context "${ctx}" -n "${ns}" delete secret "${name}" --ignore-not-found=true - test-e2e-audit-redis: - desc: Run dedicated audit to Redis enqueue scenario - cmds: - - | - export CTX="{{.CTX}}" - export INSTALL_MODE="{{.INSTALL_MODE}}" - export NAMESPACE="{{.NAMESPACE}}" - export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt" - go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=audit-redis - test-e2e-aggregated-api: desc: Run aggregated API server e2e scenarios cmds: @@ -453,8 +529,11 @@ tasks: export INSTALL_MODE="{{.INSTALL_MODE}}" export NAMESPACE="{{.NAMESPACE}}" export E2E_AGE_KEY_FILE="{{.CS}}/age-key.txt" - go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=aggregated-api + go test -timeout "{{.E2E_GO_TEST_TIMEOUT}}" ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=aggregated-api \ + -ginkgo.json-report="{{.CS}}/{{.NAMESPACE}}/ginkgo-report-aggregated-api.json" + # test-e2e-demo intentionally has no -ginkgo.json-report flag: this target + # is a demo-setup runner, not a measurable test suite. test-e2e-demo: desc: Prepare a reusable demo repo and leave demo resources in place cmds: @@ -534,13 +613,16 @@ tasks: clean-cluster: desc: Tear down the E2E cluster used for tests and remove its stamps cmds: + # Delete unconditionally rather than guarding on `k3d cluster list`. An + # interrupted create can leave orphaned artifacts (e.g. a `-tools` + # container and `-images` volume) that `k3d cluster list` does not show + # but `k3d cluster create` still rejects as "a cluster with that name + # already exists". `k3d cluster delete` cleans up that partial state and + # exits 0 when there is genuinely nothing to delete, so it is safe to run + # every time. - | - if {{.K3D}} cluster list 2>/dev/null | awk '{print $1}' | grep -q "^{{.CLUSTER_NAME}}$"; then - echo "Deleting k3d cluster '{{.CLUSTER_NAME}}'" - {{.K3D}} cluster delete "{{.CLUSTER_NAME}}" - else - echo "k3d cluster '{{.CLUSTER_NAME}}' does not exist; skipping cleanup" - fi + echo "Deleting k3d cluster '{{.CLUSTER_NAME}}' (including any orphaned partial state)" + {{.K3D}} cluster delete "{{.CLUSTER_NAME}}" - rm -rf "{{.CS}}" _cluster-ready: @@ -568,6 +650,8 @@ tasks: _flux-installed: method: timestamp + deps: + - _cluster-ready sources: - '{{.CS}}/ready' - '{{.FLUX_OPERATOR_DIR}}/**/*.yaml' @@ -583,7 +667,6 @@ tasks: -n "{{.FLUX_NAMESPACE}}" \ get fluxinstance "{{.FLUX_INSTANCE_NAME}}" >/dev/null cmds: - - task: _cluster-ready - | {{.FLUX_OPERATOR}} install \ --kube-context "{{.CTX}}" \ @@ -615,6 +698,8 @@ tasks: _flux-setup-ready: method: timestamp + deps: + - _flux-installed sources: - '{{.CS}}/flux.installed' - '{{.FLUX_SERVICES_DIR}}/**/*' @@ -623,7 +708,6 @@ tasks: status: - test -f "{{.CS}}/flux-setup.ready" cmds: - - task: _flux-installed - | kubectl --context "{{.CTX}}" apply -k "{{.FLUX_SERVICES_DIR}}" CTX="{{.CTX}}" FLUX_SERVICES_WAIT_TIMEOUT="{{.FLUX_SERVICES_WAIT_TIMEOUT}}" \ @@ -632,6 +716,8 @@ tasks: _services-ready: method: timestamp + deps: + - _flux-setup-ready sources: - '{{.CS}}/flux-setup.ready' - 'test/e2e/setup/manifests/**/*.yaml' @@ -640,7 +726,6 @@ tasks: status: - test -f "{{.CS}}/services.ready" cmds: - - task: _flux-setup-ready - | kubectl --context "{{.CTX}}" wait --for=condition=Established \ crd/prometheuses.monitoring.coreos.com \ @@ -689,6 +774,8 @@ tasks: _project-image-ready-local: method: timestamp + deps: + - _controller-image-id sources: - '{{.IS}}/controller.id' generates: @@ -697,7 +784,6 @@ tasks: - test -f "{{.IS}}/project-image.ready" - test "$(<"{{.IS}}/project-image.ready")" = "{{.PROJECT_IMAGE}}" cmds: - - task: _controller-image-id - mkdir -p "{{.IS}}" - printf '%s\n' "{{.PROJECT_IMAGE}}" > "{{.IS}}/project-image.ready" @@ -716,17 +802,17 @@ tasks: printf '%s\n' "{{.PROJECT_IMAGE}}" > "{{.IS}}/project-image.ready" _project-image-ready: + # In-process dispatch so the leaf task runs inside the same go-task + # graph as its caller — preserves CTX and lets it run in parallel with + # the cluster/Flux chain via deps on _image-loaded. cmds: - - | - if [ -n "{{.PROJECT_IMAGE_PROVIDED}}" ]; then - task _project-image-ready-provided - else - task _controller-image-id - task _project-image-ready-local - fi + - task: '{{if .PROJECT_IMAGE_PROVIDED}}_project-image-ready-provided{{else}}_project-image-ready-local{{end}}' _image-loaded: method: timestamp + deps: + - _cluster-ready + - _project-image-ready sources: - '{{.CS}}/ready' - '{{.IS}}/project-image.ready' @@ -735,7 +821,6 @@ tasks: status: - test -f "{{.CS}}/image.loaded" cmds: - - task: _cluster-ready - mkdir -p "{{.CS}}" - | CTX="{{.CTX}}" CLUSTER_NAME="{{.CLUSTER_NAME}}" PROJECT_IMAGE="{{.PROJECT_IMAGE}}" \ @@ -745,7 +830,39 @@ tasks: CONTAINER_TOOL="{{.CONTAINER_TOOL}}" K3D="{{.K3D}}" STAMP_FILE="{{.CS}}/image.loaded" \ bash hack/e2e/load-image.sh + _install-cleanup: + # Wipe prior install state so a new install mode can take over cleanly. + # Runs cleanup-installs.sh, which deletes any existing install.yaml-owned + # resources, drops the install namespace, and rm -rf's every stamp under + # .stamps/cluster/// except git-* fixtures. + # + # This used to live inline at the top of every _install- task. As + # its own DAG node it can be a parallel-safe prerequisite for both the + # active _install- and for tasks like _sops-secret-yaml that write + # into the same namespace dir. + # + # Status: if the install.yaml for the *current* INSTALL_MODE already + # exists we're either warm or just re-running the same mode, and the + # in-place apply/upgrade does not need a wipe. Cleanup is only required + # when switching modes or on a brand-new CTX where the file is missing. + method: timestamp + deps: + - _cluster-ready + generates: + - '{{.CS}}/install-cleanup.done' + status: + - test -f "{{.CS}}/{{.NAMESPACE}}/{{.INSTALL_MODE}}/install.yaml" + cmds: + - | + CTX="{{.CTX}}" KUBECTL="{{.KUBECTL}}" bash hack/cleanup-installs.sh + mkdir -p "{{.CS}}" + touch "{{.CS}}/install-cleanup.done" + _install-helm: + deps: + - _services-ready + - helm-sync + - _install-cleanup sources: - '{{.CS}}/services.ready' - charts/gitops-reverser/Chart.yaml @@ -757,7 +874,6 @@ tasks: - '{{.CS}}/{{.NAMESPACE}}/helm/install.yaml' cmds: - | - CTX="{{.CTX}}" KUBECTL="{{.KUBECTL}}" bash hack/cleanup-installs.sh mkdir -p "{{.CS}}/{{.NAMESPACE}}/helm" skip_crds_arg="" if kubectl --context "{{.CTX}}" get crd \ @@ -789,6 +905,10 @@ tasks: fi _install-config-dir: + deps: + - _services-ready + - manifests + - _install-cleanup sources: - '{{.CS}}/services.ready' - config/**/* @@ -796,7 +916,6 @@ tasks: - '{{.CS}}/{{.NAMESPACE}}/config-dir/install.yaml' cmds: - | - CTX="{{.CTX}}" KUBECTL="{{.KUBECTL}}" bash hack/cleanup-installs.sh mkdir -p "{{.CS}}/{{.NAMESPACE}}/config-dir" CTX="{{.CTX}}" NAMESPACE="{{.NAMESPACE}}" PROJECT_IMAGE="{{.PROJECT_IMAGE}}" \ E2E_VALKEY_PASSWORD="{{.E2E_VALKEY_PASSWORD}}" \ @@ -806,6 +925,10 @@ tasks: _install-plain-manifests-file: method: timestamp + deps: + - _services-ready + - dist-install + - _install-cleanup sources: - '{{.CS}}/services.ready' - dist/install.yaml @@ -813,7 +936,6 @@ tasks: - '{{.CS}}/{{.NAMESPACE}}/plain-manifests-file/install.yaml' cmds: - | - CTX="{{.CTX}}" KUBECTL="{{.KUBECTL}}" bash hack/cleanup-installs.sh mkdir -p "{{.CS}}/{{.NAMESPACE}}/plain-manifests-file" CTX="{{.CTX}}" NAMESPACE="{{.NAMESPACE}}" E2E_VALKEY_PASSWORD="{{.E2E_VALKEY_PASSWORD}}" \ bash hack/e2e/ensure-valkey-auth.sh @@ -827,6 +949,9 @@ tasks: _controller-deployed: method: timestamp + deps: + - install + - _image-loaded sources: - '{{.CS}}/{{.NAMESPACE}}/{{.INSTALL_MODE}}/install.yaml' - '{{.CS}}/image.loaded' @@ -850,6 +975,8 @@ tasks: _webhook-tls-ready: method: timestamp + deps: + - _controller-deployed sources: - '{{.CS}}/{{.NAMESPACE}}/controller.deployed' generates: @@ -865,6 +992,9 @@ tasks: _aggregated-api-webhook-kubeconfig-ready: method: timestamp + deps: + - _services-ready + - _webhook-tls-ready sources: - '{{.CS}}/services.ready' - '{{.CS}}/{{.NAMESPACE}}/webhook-tls.ready' @@ -875,8 +1005,6 @@ tasks: status: - test -f "{{.CS}}/aggregated-api-webhook-kubeconfig.ready" cmds: - - task: _services-ready - - task: _webhook-tls-ready - | CTX="{{.CTX}}" NAMESPACE="{{.NAMESPACE}}" PROXY_NAMESPACE="aggregated-api" \ bash hack/e2e/prepare-aggregated-api-webhook-kubeconfig.sh @@ -884,6 +1012,8 @@ tasks: _aggregated-api-ready: method: timestamp + deps: + - _aggregated-api-webhook-kubeconfig-ready sources: - '{{.CS}}/services.ready' - '{{.CS}}/aggregated-api-webhook-kubeconfig.ready' @@ -893,8 +1023,6 @@ tasks: status: - test -f "{{.CS}}/aggregated-api.ready" cmds: - - task: _services-ready - - task: _aggregated-api-webhook-kubeconfig-ready - | kubectl --context "{{.CTX}}" -n aggregated-api wait certificate/audit-pass-through-proxy-serving-cert \ --for=condition=Ready --timeout=180s @@ -934,6 +1062,15 @@ tasks: _sops-secret-yaml: method: timestamp + # _install-cleanup runs first: it wipes everything under + # .stamps/cluster/// except git-* fixtures, including this + # task's output. Without this dep, status: was evaluated in parallel + # with the cleanup, found a stale file from a prior run, reported "up + # to date", and then _sops-secret-applied tried to kubectl apply a + # file the cleanup had just deleted. + deps: + - _age-key + - _install-cleanup sources: - '{{.CS}}/age-key.txt' - test/e2e/tools/gen-age-key/main.go @@ -958,6 +1095,9 @@ tasks: _sops-secret-applied: method: timestamp + deps: + - _sops-secret-yaml + - install sources: - '{{.CS}}/{{.NAMESPACE}}/sops-secret.yaml' - '{{.CS}}/{{.NAMESPACE}}/{{.INSTALL_MODE}}/install.yaml' @@ -972,6 +1112,11 @@ tasks: _prepare-e2e-ready: method: timestamp + deps: + - _controller-deployed + - _webhook-tls-ready + - _sops-secret-applied + - _aggregated-api-ready sources: - '{{.CS}}/{{.NAMESPACE}}/{{.INSTALL_MODE}}/install.yaml' - '{{.CS}}/image.loaded' @@ -1033,6 +1178,14 @@ tasks: crd/quizsessions.examples.configbutler.ai \ crd/quizsubmissions.examples.configbutler.ai \ --timeout=180s + # Traefik and kro live in the demo path (not the core bring-up): the + # demo podinfo flows below need kro's ResourceGraphDefinition CRD and + # traefik's IngressRoutes. Install them here and wait before applying + # anything that depends on them. + kubectl --context "{{.CTX}}" apply -k "{{.DEMO_MANIFESTS_DIR}}/flux" + kubectl --context "{{.CTX}}" -n flux-system wait \ + helmrelease/traefik helmrelease/kro \ + --for=condition=Ready --timeout=300s kubectl --context "{{.CTX}}" apply -f "{{.DEMO_MANIFESTS_DIR}}/podinfo-kro-template.yaml" kubectl --context "{{.CTX}}" wait --for=condition=Ready \ resourcegraphdefinition/podinfo-app --timeout=120s diff --git a/test/e2e/aggregated_apiserver_e2e_test.go b/test/e2e/aggregated_apiserver_e2e_test.go index e9ed2617b..38bae82fd 100644 --- a/test/e2e/aggregated_apiserver_e2e_test.go +++ b/test/e2e/aggregated_apiserver_e2e_test.go @@ -19,7 +19,6 @@ limitations under the License. package e2e import ( - "context" "encoding/json" "fmt" "os" @@ -33,6 +32,9 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) +// Not Serial: the wardle APIService is installed once at cluster setup (Flux) +// and only read here; everything this spec mutates is name-isolated. See +// docs/design/e2e-serial-registry.md. var _ = Describe("Aggregated API server", Label("aggregated-api"), Ordered, func() { var ( testNs string @@ -52,6 +54,10 @@ var _ = Describe("Aggregated API server", Label("aggregated-api"), Ordered, func _, _ = kubectlRun("create", "namespace", testNs) + By("setting up the Prometheus client for the shallow-drop metric check") + setupPrometheusClient() + verifyPrometheusAvailable() + repo = SetupRepo( resolveE2EContext(), testNs, @@ -165,58 +171,31 @@ spec: }, 30*time.Second, time.Second).Should(Succeed()) }) - It("should recover Flunder audit request and response bodies through the proxy", func() { - streamName := defaultAuditRedisStream - configMapName := fmt.Sprintf("aggregated-audit-compare-cm-%d", GinkgoRandomSeed()) - flunderName := fmt.Sprintf("aggregated-audit-compare-flunder-%d", GinkgoRandomSeed()) - - By("connecting to Valkey through the e2e port-forward") - client := newE2EValkeyClient() - defer func() { - _ = client.Close() - }() - - Eventually(func(g Gomega) { - g.Expect(client.Ping(context.Background()).Err()).NotTo(HaveOccurred()) - }, 30*time.Second, 2*time.Second).Should(Succeed()) - - By("recording the latest audit stream entry before creating comparison resources") - baselineID, err := latestAuditStreamID(context.Background(), client, streamName) - Expect(err).NotTo(HaveOccurred()) - - configMapManifest := fmt.Sprintf(`apiVersion: v1 -kind: ConfigMap -metadata: - name: %s - namespace: %s -data: - compare: configmap -`, configMapName, testNs) + It("should never drop aggregated audit events as shallow", func() { + // A "shallow" audit event is one whose request/response body never + // arrived, so it cannot drive a high-quality Git write and is dropped, + // incrementing gitopsreverser_audit_shallow_dropped_total. In a correctly + // configured environment every audit event is paired with its body, so + // that counter must stay at zero — for this scenario and for every other + // test sharing the cluster. Asserting it here is the externally + // observable equivalent of confirming the proxy recovers full bodies. + ensureFlunderWatchRuleReady() + flunderName := fmt.Sprintf("aggregated-shallow-flunder-%d", GinkgoRandomSeed()) + repoPath := path.Join( + aggregatedPath, + fmt.Sprintf("wardle.example.com/v1alpha1/flunders/%s/%s.yaml", testNs, flunderName), + ) + repoFile := filepath.Join(repo.CheckoutDir, repoPath) flunderManifest := fmt.Sprintf(`apiVersion: wardle.example.com/v1alpha1 kind: Flunder metadata: name: %s namespace: %s spec: - reference: compare-reference + reference: shallow-check-reference `, flunderName, testNs) - By("creating a ConfigMap and Flunder via the same kubectl apply path") - _, err = kubectlRunWithStdin(testNs, configMapManifest, "apply", "-f", "-") - Expect(err).NotTo(HaveOccurred()) - _, err = kubectlRunWithStdin(testNs, flunderManifest, "apply", "-f", "-") - Expect(err).NotTo(HaveOccurred()) - - DeferCleanup(func() { - if skipCleanupBecauseResourcesArePreserved( - fmt.Sprintf("ConfigMap %s/%s", testNs, configMapName), - testNs, - ) { - return - } - _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", configMapName, "--ignore-not-found=true") - }) DeferCleanup(func() { if skipCleanupBecauseResourcesArePreserved(fmt.Sprintf("Flunder %s/%s", testNs, flunderName), testNs) { return @@ -224,62 +203,31 @@ spec: _, _ = kubectlRunInNamespace(testNs, "delete", "flunder", flunderName, "--ignore-not-found=true") }) - By("capturing the corresponding raw audit payloads from the Valkey stream") + By("creating a Flunder through the aggregated API to exercise the audit body-join path") + _, err := kubectlRunWithStdin(testNs, flunderManifest, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred()) + + By("waiting for the create commit, confirming the audit body was recovered (not shallow)") Eventually(func(g Gomega) { - configMapAudit, findErr := findAuditPayloadSince( - context.Background(), - client, - streamName, - baselineID, - 300, - func(payload map[string]interface{}) bool { - return auditPayloadMatches(payload, "", "configmaps", testNs, configMapName, "create") - }, - ) - g.Expect(findErr).NotTo(HaveOccurred()) - - flunderAudit, findErr := findAuditPayloadSince( - context.Background(), - client, - streamName, - baselineID, - 300, - func(payload map[string]interface{}) bool { - return auditPayloadMatches(payload, "wardle.example.com", "flunders", testNs, flunderName, "create") - }, - ) - g.Expect(findErr).NotTo(HaveOccurred()) - - g.Expect(auditObjectRefName(configMapAudit.Payload)).To(Equal(configMapName)) - g.Expect(auditPayloadHasObject(configMapAudit.Payload, "requestObject")).To(BeTrue()) - g.Expect(auditPayloadHasObject(configMapAudit.Payload, "responseObject")).To(BeTrue()) - g.Expect(auditObjectRefName(flunderAudit.Payload)).To(Equal(flunderName)) - g.Expect(auditPayloadHasObject(flunderAudit.Payload, "requestObject")).To(BeTrue()) - g.Expect(auditPayloadHasObject(flunderAudit.Payload, "responseObject")).To(BeTrue()) - flunderAuditID := auditPayloadID(flunderAudit.Payload) - g.Expect(flunderAuditID).NotTo(BeEmpty()) - flunderCount, countErr := countAuditPayloadsByAuditIDSince( - context.Background(), - client, - streamName, - baselineID, - flunderAuditID, - 300, - ) - g.Expect(countErr).NotTo(HaveOccurred()) - g.Expect(flunderCount).To(Equal(1), "expected one canonical stream entry for the Flunder auditID") + pullLatestRepoState(g, repo.CheckoutDir) - _, _ = fmt.Fprintf( - GinkgoWriter, - "ConfigMap create audit payload for comparison:\n%s\n", - prettyAuditPayload(configMapAudit.Payload), - ) - _, _ = fmt.Fprintf( - GinkgoWriter, - "Flunder create audit payload for comparison:\n%s\n", - prettyAuditPayload(flunderAudit.Payload), - ) - }, 30*time.Second, 2*time.Second).Should(Succeed()) + content, readErr := os.ReadFile(repoFile) + g.Expect(readErr).NotTo(HaveOccurred(), "expected Flunder file to exist after create") + g.Expect(string(content)).To(ContainSubstring("reference: shallow-check-reference")) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + By("verifying no audit events were dropped as shallow, cluster-wide") + // Zero is the only acceptable value here. We never want a shallow drop: + // not for this Flunder, and not as a side effect of any other test + // sharing this cluster. Any non-zero count means a request/response body + // failed to join its official event, so this assertion stays absolute + // (not a baseline delta) on purpose. + Consistently(func(g Gomega) { + dropped, queryErr := queryPrometheus("sum(gitopsreverser_audit_shallow_dropped_total) or vector(0)") + g.Expect(queryErr).NotTo(HaveOccurred()) + g.Expect(dropped).To(BeZero(), + "gitopsreverser_audit_shallow_dropped_total must stay at zero in a healthy e2e environment") + }, 10*time.Second, 2*time.Second).Should(Succeed()) }) It("should accept an aggregated flunder WatchRule and mark it ready", func() { @@ -289,7 +237,6 @@ spec: It("should attribute Flunder commits to the impersonated user", func() { ensureFlunderWatchRuleReady() - streamName := defaultAuditRedisStream flunderName := fmt.Sprintf("aggregated-impersonation-flunder-%d", GinkgoRandomSeed()) repoPath := path.Join( aggregatedPath, @@ -305,20 +252,6 @@ spec: reference: impersonated-reference `, flunderName, testNs) - By("connecting to Valkey through the e2e port-forward") - client := newE2EValkeyClient() - defer func() { - _ = client.Close() - }() - - Eventually(func(g Gomega) { - g.Expect(client.Ping(context.Background()).Err()).NotTo(HaveOccurred()) - }, 30*time.Second, 2*time.Second).Should(Succeed()) - - By("recording the latest audit stream entry before the impersonated create") - baselineID, err := latestAuditStreamID(context.Background(), client, streamName) - Expect(err).NotTo(HaveOccurred()) - DeferCleanup(func() { if skipCleanupBecauseResourcesArePreserved(fmt.Sprintf("Flunder %s/%s", testNs, flunderName), testNs) { return @@ -327,26 +260,9 @@ spec: }) By("creating a Flunder through the aggregated API as an impersonated user") - _, err = kubectlRunWithStdin(testNs, flunderManifest, "--as=jane@acme.com", "apply", "-f", "-") + _, err := kubectlRunWithStdin(testNs, flunderManifest, "--as=jane@acme.com", "apply", "-f", "-") Expect(err).NotTo(HaveOccurred()) - By("observing the impersonated user in the audit payload") - Eventually(func(g Gomega) { - flunderAudit, findErr := findAuditPayloadSince( - context.Background(), - client, - streamName, - baselineID, - 300, - func(payload map[string]interface{}) bool { - return auditPayloadMatches(payload, "wardle.example.com", "flunders", testNs, flunderName, "create") - }, - ) - g.Expect(findErr).NotTo(HaveOccurred()) - g.Expect(auditObjectRefName(flunderAudit.Payload)).To(Equal(flunderName)) - g.Expect(auditEffectiveUsername(flunderAudit.Payload)).To(Equal("jane@acme.com")) - }, 30*time.Second, 2*time.Second).Should(Succeed()) - By("verifying the resulting git commit uses the impersonated user as author") Eventually(func(g Gomega) { pullLatestRepoState(g, repo.CheckoutDir) @@ -465,32 +381,3 @@ spec: }, 2*time.Minute, 2*time.Second).Should(Succeed()) }) }) - -func auditEffectiveUsername(payload map[string]interface{}) string { - if impersonated := nestedAuditString(payload, "impersonatedUser", "username"); impersonated != "" { - return impersonated - } - - return nestedAuditString(payload, "user", "username") -} - -func nestedAuditString(payload map[string]interface{}, fields ...string) string { - current := any(payload) - for _, field := range fields { - nextMap, ok := current.(map[string]interface{}) - if !ok { - return "" - } - current, ok = nextMap[field] - if !ok { - return "" - } - } - - value, ok := current.(string) - if !ok { - return "" - } - - return strings.TrimSpace(value) -} diff --git a/test/e2e/audit_helpers_test.go b/test/e2e/audit_helpers_test.go deleted file mode 100644 index 9264d350d..000000000 --- a/test/e2e/audit_helpers_test.go +++ /dev/null @@ -1,241 +0,0 @@ -/* -SPDX-License-Identifier: Apache-2.0 - -Copyright 2025 ConfigButler - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - "encoding/json" - "fmt" - "strings" - - "github.com/redis/go-redis/v9" -) - -type auditPayloadEntry struct { - ID string - Payload map[string]interface{} - RawJSON string -} - -func newE2EValkeyClient() *redis.Client { - return redis.NewClient(&redis.Options{ - Addr: valkeyPortForwardAddr(), - Password: valkeyPortForwardPassword(), - }) -} - -func latestAuditStreamID(ctx context.Context, client *redis.Client, stream string) (string, error) { - entries, err := client.XRevRangeN(ctx, stream, "+", "-", 1).Result() - if err != nil { - return "", err - } - if len(entries) == 0 { - return "", nil - } - - return entries[0].ID, nil -} - -func findAuditPayloadSince( - ctx context.Context, - client *redis.Client, - stream, afterID string, - limit int64, - match func(map[string]interface{}) bool, -) (auditPayloadEntry, error) { - start := "-" - if afterID != "" { - start = "(" + afterID - } - - entries, err := client.XRangeN(ctx, stream, start, "+", limit).Result() - if err != nil { - return auditPayloadEntry{}, err - } - - for _, entry := range entries { - rawJSON, ok := entry.Values["payload_json"].(string) - if !ok || rawJSON == "" { - continue - } - - var payload map[string]interface{} - if err := json.Unmarshal([]byte(rawJSON), &payload); err != nil { - return auditPayloadEntry{}, fmt.Errorf("decode payload_json for stream entry %s: %w", entry.ID, err) - } - if match(payload) { - return auditPayloadEntry{ - ID: entry.ID, - Payload: payload, - RawJSON: rawJSON, - }, nil - } - } - - return auditPayloadEntry{}, fmt.Errorf( - "no matching audit payload found after scanning %d entries from %q: %s", - len(entries), - start, - summarizeAuditPayloadEntries(entries, 12), - ) -} - -func countAuditPayloadsByAuditIDSince( - ctx context.Context, - client *redis.Client, - stream, afterID, auditID string, - limit int64, -) (int, error) { - start := "-" - if afterID != "" { - start = "(" + afterID - } - - entries, err := client.XRangeN(ctx, stream, start, "+", limit).Result() - if err != nil { - return 0, err - } - - count := 0 - for _, entry := range entries { - value, _ := entry.Values["audit_id"].(string) - if value == auditID { - count++ - } - } - return count, nil -} - -func auditPayloadID(payload map[string]interface{}) string { - value, _ := payload["auditID"].(string) - return value -} - -func auditPayloadMatches( - payload map[string]interface{}, - apiGroup, resource, namespace, name, verb string, -) bool { - objectRef, _ := payload["objectRef"].(map[string]interface{}) - if objectRef == nil { - return false - } - if value, _ := payload["verb"].(string); verb != "" && value != verb { - return false - } - if value, _ := objectRef["apiGroup"].(string); apiGroup != value { - return false - } - if value, _ := objectRef["resource"].(string); resource != value { - return false - } - if value, _ := objectRef["namespace"].(string); namespace != value { - return false - } - if name == "" { - return true - } - - value, _ := objectRef["name"].(string) - return name == value -} - -func auditObjectRefName(payload map[string]interface{}) string { - objectRef, _ := payload["objectRef"].(map[string]interface{}) - if objectRef == nil { - return "" - } - - value, _ := objectRef["name"].(string) - return value -} - -func auditPayloadHasObject(payload map[string]interface{}, field string) bool { - object, _ := payload[field].(map[string]interface{}) - return len(object) > 0 -} - -func summarizeAuditPayloadEntries(entries []redis.XMessage, maxEntries int) string { - if len(entries) == 0 { - return "" - } - if maxEntries <= 0 { - maxEntries = len(entries) - } - start := 0 - if len(entries) > maxEntries { - start = len(entries) - maxEntries - } - - var builder strings.Builder - if start > 0 { - _, _ = fmt.Fprintf(&builder, "... %d earlier entries omitted; ", start) - } - for i, entry := range entries[start:] { - if i > 0 { - builder.WriteString("; ") - } - rawJSON, _ := entry.Values["payload_json"].(string) - if rawJSON == "" { - _, _ = fmt.Fprintf(&builder, "%s ", entry.ID) - continue - } - - var payload map[string]interface{} - if err := json.Unmarshal([]byte(rawJSON), &payload); err != nil { - _, _ = fmt.Fprintf(&builder, "%s ", entry.ID, err) - continue - } - _, _ = fmt.Fprintf( - &builder, - "%s auditID=%s verb=%s ref=%s/%s ns=%s name=%s requestObject=%t responseObject=%t", - entry.ID, - auditPayloadID(payload), - auditPayloadString(payload, "verb"), - auditPayloadObjectRefString(payload, "apiGroup"), - auditPayloadObjectRefString(payload, "resource"), - auditPayloadObjectRefString(payload, "namespace"), - auditPayloadObjectRefString(payload, "name"), - auditPayloadHasObject(payload, "requestObject"), - auditPayloadHasObject(payload, "responseObject"), - ) - } - return builder.String() -} - -func auditPayloadString(payload map[string]interface{}, field string) string { - value, _ := payload[field].(string) - return value -} - -func auditPayloadObjectRefString(payload map[string]interface{}, field string) string { - objectRef, _ := payload["objectRef"].(map[string]interface{}) - if objectRef == nil { - return "" - } - value, _ := objectRef[field].(string) - return value -} - -func prettyAuditPayload(payload map[string]interface{}) string { - formatted, err := json.MarshalIndent(payload, "", " ") - if err != nil { - return fmt.Sprintf("", err) - } - return string(formatted) -} diff --git a/test/e2e/audit_redis_e2e_test.go b/test/e2e/audit_redis_e2e_test.go deleted file mode 100644 index 138b22649..000000000 --- a/test/e2e/audit_redis_e2e_test.go +++ /dev/null @@ -1,376 +0,0 @@ -/* -SPDX-License-Identifier: Apache-2.0 - -Copyright 2025 ConfigButler - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "os" - "os/exec" - "path" - "path/filepath" - "strings" - "sync" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/redis/go-redis/v9" -) - -const ( - defaultAuditRedisStream = "gitopsreverser.audit.events.v1" - defaultE2EValkeyPassword = "e2e-valkey-password" -) - -var ( - // auditRedisRepo holds the file-local repo fixtures for both audit-redis Describe blocks. - auditRedisRepo *RepoArtifacts - auditRedisRepoOnce sync.Once -) - -func ensureAuditRedisRepo() *RepoArtifacts { - GinkgoHelper() - - auditRedisRepoOnce.Do(func() { - consumerNs := testNamespaceFor("audit-consumer") - - By("creating the audit consumer namespace for shared repo fixtures") - _, _ = kubectlRun("create", "namespace", consumerNs) - - By("setting up the shared Gitea repo for audit-redis tests") - auditRedisRepo = SetupRepo( - resolveE2EContext(), - consumerNs, - fmt.Sprintf("e2e-audit-redis-%d", GinkgoRandomSeed()), - ) - }) - - Expect(auditRedisRepo).NotTo(BeNil(), "expected audit Redis repo fixtures to be initialised") - return auditRedisRepo -} - -var _ = Describe("Audit Redis Queue", Label("audit-redis", "smoke"), Ordered, func() { - var testNs string - - BeforeAll(func() { - By("creating producer test namespace") - testNs = testNamespaceFor("audit-redis") - _, _ = kubectlRun("create", "namespace", testNs) // idempotent; ignore AlreadyExists - - repo := ensureAuditRedisRepo() - - By("applying git secrets to producer test namespace") - _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) - Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to producer test namespace") - }) - - AfterAll(func() { - cleanupNamespace(testNs) - }) - - It("should enqueue incoming audit webhook events into a Redis stream", func() { - streamName := defaultAuditRedisStream - testConfigMapName := fmt.Sprintf("audit-redis-test-cm-%d", GinkgoRandomSeed()) - - By("connecting to Valkey through the e2e port-forward") - client := redis.NewClient(&redis.Options{ - Addr: valkeyPortForwardAddr(), - Password: valkeyPortForwardPassword(), - }) - Eventually(func(g Gomega) { - g.Expect(client.Ping(context.Background()).Err()).NotTo(HaveOccurred()) - }, 30*time.Second, 2*time.Second).Should(Succeed()) - - By("triggering kube-apiserver audit events with a ConfigMap create/update") - _, err := kubectlRunInNamespace( - testNs, - "delete", - "configmap", - testConfigMapName, - "--ignore-not-found=true", - ) - Expect(err).NotTo(HaveOccurred()) - - _, err = kubectlRunInNamespace( - testNs, - "create", - "configmap", - testConfigMapName, - "--from-literal=test=audit-redis", - ) - Expect(err).NotTo(HaveOccurred()) - - _, err = kubectlRunInNamespace( - testNs, - "patch", - "configmap", - testConfigMapName, - "--type=merge", - "--patch", - `{"data":{"test":"audit-redis-updated"}}`, - ) - Expect(err).NotTo(HaveOccurred()) - - By("verifying Redis stream receives an entry for the ConfigMap audit event") - Eventually(func(g Gomega) { - entries, readErr := client.XRange(context.Background(), streamName, "-", "+").Result() - g.Expect(readErr).NotTo(HaveOccurred()) - g.Expect(entries).NotTo(BeEmpty()) - - found := false - for _, entry := range entries { - resource, _ := entry.Values["resource"].(string) - name, _ := entry.Values["name"].(string) - payload, _ := entry.Values["payload_json"].(string) - if resource == "configmaps" && name == testConfigMapName && payload != "" { - found = true - break - } - } - g.Expect(found).To(BeTrue(), "expected ConfigMap audit event to be enqueued in Redis stream") - }, 2*time.Minute, 2*time.Second).Should(Succeed()) - - By("cleaning up test ConfigMap") - _, _ = kubectlRunInNamespace( - testNs, - "delete", - "configmap", - testConfigMapName, - "--ignore-not-found=true", - ) - }) -}) - -// auditConsumerDescribe holds the consumer-path e2e test. -// Label: audit-redis (runs together with the producer test via task test-e2e-audit-redis). -var _ = Describe("Audit Redis Consumer", Label("audit-redis", "smoke"), Ordered, func() { - var ( - testNs string - valkeyClient *redis.Client - gitProvName string - gitTargetName string - watchRuleName string - cmName string - ) - - BeforeAll(func() { - By("creating consumer test namespace and applying git secrets") - testNs = testNamespaceFor("audit-consumer") - repo := ensureAuditRedisRepo() - _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) - Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to consumer test namespace") - applySOPSAgeKeyToNamespace(testNs) - - By("connecting to Valkey through the e2e port-forward") - valkeyClient = redis.NewClient(&redis.Options{ - Addr: valkeyPortForwardAddr(), - Password: valkeyPortForwardPassword(), - }) - Eventually(func(g Gomega) { - g.Expect(valkeyClient.Ping(context.Background()).Err()).NotTo(HaveOccurred()) - }, 30*time.Second, 2*time.Second).Should(Succeed()) - - By("setting up GitProvider, GitTarget and WatchRule for consumer test") - seed := GinkgoRandomSeed() - gitProvName = fmt.Sprintf("audit-consumer-gitprovider-%d", seed) - gitTargetName = fmt.Sprintf("audit-consumer-gittarget-%d", seed) - watchRuleName = fmt.Sprintf("audit-consumer-watchrule-%d", seed) - cmName = fmt.Sprintf("audit-consumer-cm-%d", seed) - - createGitProviderWithURLInNamespace( - gitProvName, - testNs, - auditRedisRepo.GitSecretHTTP, - auditRedisRepo.RepoURLHTTP, - ) - verifyResourceStatus("gitprovider", gitProvName, testNs, "True", "Ready", "") - - createGitTarget(gitTargetName, testNs, gitProvName, "e2e/audit-consumer-test", "main") - verifyResourceStatus("gittarget", gitTargetName, testNs, "True", "Ready", "") - - data := struct { - Name string - Namespace string - DestinationName string - }{ - Name: watchRuleName, - Namespace: testNs, - DestinationName: gitTargetName, - } - err = applyFromTemplate("test/e2e/templates/manager/watchrule-configmap.tmpl", data, testNs) - Expect(err).NotTo(HaveOccurred(), "failed to apply WatchRule") - verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") - }) - - AfterAll(func() { - cleanupWatchRule(watchRuleName, testNs) - cleanupGitTarget(gitTargetName, testNs) - cleanupNamespacedResource(testNs, "gitprovider", gitProvName) - cleanupNamespace(testNs) - if valkeyClient != nil { - _ = valkeyClient.Close() - } - }) - - It("should produce a Git commit from an audit stream event consumed by the consumer", func() { - streamName := defaultAuditRedisStream - - By("creating a ConfigMap to produce an audit event") - cmData := struct { - Name string - Namespace string - }{Name: cmName, Namespace: testNs} - err := applyFromTemplate("test/e2e/templates/manager/configmap.tmpl", cmData, testNs) - Expect(err).NotTo(HaveOccurred(), "failed to create test ConfigMap") - - By("waiting for the audit stream entry to appear (producer path)") - // cmName includes GinkgoRandomSeed so it uniquely identifies this test run's entry. - Eventually(func(g Gomega) { - entries, readErr := valkeyClient.XRange(context.Background(), streamName, "-", "+").Result() - g.Expect(readErr).NotTo(HaveOccurred()) - found := false - for _, entry := range entries { - resource, _ := entry.Values["resource"].(string) - name, _ := entry.Values["name"].(string) - if resource == "configmaps" && name == cmName { - found = true - break - } - } - g.Expect(found).To(BeTrue(), "audit stream entry for ConfigMap should appear") - }, 2*time.Minute, 2*time.Second).Should(Succeed()) - - By("waiting for the Git commit to appear (consumer path)") - expectedFile := filepath.Join(auditRedisRepo.CheckoutDir, - "e2e/audit-consumer-test", - fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, cmName)) - - Eventually(func(g Gomega) { - pullLatestRepoState(g, auditRedisRepo.CheckoutDir) - - info, statErr := os.Stat(expectedFile) - g.Expect(statErr).NotTo(HaveOccurred(), - fmt.Sprintf("ConfigMap file should exist at %s", expectedFile)) - g.Expect(info.Size()).To(BeNumerically(">", 0)) - - expectedRepoPath := path.Join( - "e2e/audit-consumer-test", - fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, cmName), - ) - assertLatestCommitTouchesOnly(g, auditRedisRepo.CheckoutDir, []string{expectedRepoPath}) - assertLatestCommitTouchesNoNamespaces( - g, - auditRedisRepo.CheckoutDir, - "e2e/audit-consumer-test/v1/configmaps", - []string{ - "gitops-reverser", - "flux-system", - "kube-system", - "tilt-playground", - }, - ) - - authorCmd := exec.Command("git", "log", "-1", "--pretty=%an") - authorCmd.Dir = auditRedisRepo.CheckoutDir - authorOut, authorErr := authorCmd.CombinedOutput() - g.Expect(authorErr).NotTo(HaveOccurred(), - fmt.Sprintf("git log author failed: %s", string(authorOut))) - g.Expect(string(authorOut)).To(ContainSubstring("system:admin")) - }, 3*time.Minute, 3*time.Second).Should(Succeed()) - - By("cleaning up test ConfigMap") - _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", cmName, "--ignore-not-found=true") - }) - - It("should attribute the commit to the OIDC display name and email from user.extra", func() { - const ( - oidcDisplayName = "Simon Koudijs" - oidcEmail = "something@configbutler.ai" - ) - oidcCMName := fmt.Sprintf("audit-oidc-cm-%d", GinkgoRandomSeed()) - - By("creating a ConfigMap while impersonating an OIDC user carrying name/email extras") - // Impersonation makes the real kube-apiserver emit an audit event whose - // impersonatedUser.Extra carries these keys, mimicking the claims a - // structured authentication config maps into user.extra for an OIDC - // login. system:masters keeps the impersonated create authorized - // without provisioning per-user RBAC. - err := createConfigMapAsImpersonatedUser( - testNs, - oidcCMName, - "oidc-simon", - []string{"system:masters"}, - map[string][]string{ - "configbutler.ai/claims/display-name": {oidcDisplayName}, - "configbutler.ai/claims/email": {oidcEmail}, - }, - ) - Expect(err).NotTo(HaveOccurred(), "failed to create impersonated ConfigMap") - - repoPath := path.Join( - "e2e/audit-consumer-test", - fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, oidcCMName), - ) - expectedFile := filepath.Join(auditRedisRepo.CheckoutDir, repoPath) - - By("waiting for the commit and asserting its author is the OIDC identity") - Eventually(func(g Gomega) { - pullLatestRepoState(g, auditRedisRepo.CheckoutDir) - - info, statErr := os.Stat(expectedFile) - g.Expect(statErr).NotTo(HaveOccurred(), - fmt.Sprintf("ConfigMap file should exist at %s", expectedFile)) - g.Expect(info.Size()).To(BeNumerically(">", 0)) - - // Scope the log to this run's unique file path so the author read - // back is unambiguously the commit produced by the impersonated - // create, regardless of commit-window coalescing. - authorCmd := exec.Command( - "git", "log", "-1", "--pretty=%an <%ae>", "--", repoPath, - ) - authorCmd.Dir = auditRedisRepo.CheckoutDir - authorOut, authorErr := authorCmd.CombinedOutput() - g.Expect(authorErr).NotTo(HaveOccurred(), - fmt.Sprintf("git log author failed: %s", string(authorOut))) - g.Expect(strings.TrimSpace(string(authorOut))).To( - Equal(fmt.Sprintf("%s <%s>", oidcDisplayName, oidcEmail)), - "commit author should be the OIDC display name and email from user.extra") - }, 3*time.Minute, 3*time.Second).Should(Succeed()) - - By("cleaning up the test ConfigMap") - _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", oidcCMName, "--ignore-not-found=true") - }) -}) - -func valkeyPortForwardAddr() string { - port := strings.TrimSpace(os.Getenv("VALKEY_PORT")) - if port == "" { - port = "16379" - } - return "127.0.0.1:" + port -} - -func valkeyPortForwardPassword() string { - password := strings.TrimSpace(os.Getenv("E2E_VALKEY_PASSWORD")) - if password == "" { - return defaultE2EValkeyPassword - } - return password -} diff --git a/test/e2e/bi_directional_e2e_test.go b/test/e2e/bi_directional_e2e_test.go index cf153ea99..20d9a4aef 100644 --- a/test/e2e/bi_directional_e2e_test.go +++ b/test/e2e/bi_directional_e2e_test.go @@ -95,6 +95,16 @@ type iceCreamScoop struct { // biDirectionalRepo holds the file-local repo fixtures for the Bi Directional describe block. var biDirectionalRepo *RepoArtifacts +// Not Serial: a whole-cluster bidirectional behavioral test (Flux applies from +// git while gitops-reverser mirrors live state back) that asserts on *exact* +// commit counts to prove no commit loop. It owns a dedicated Gitea repo and its +// own per-file CRD group, so the only writer to its main branch is its own +// GitTarget. The historical +2-commits-under-parallelism flake came from +// cluster-wide GVR catalog churn (another spec installing/deleting a CRD) +// dragging unrelated targets into a resnapshot; that is fixed — a target now +// only resnapshots when its *resolved* plan hash changes (manager.go) — so a +// foreign catalog refresh no longer perturbs this repo's commit count. See +// docs/design/e2e-serial-registry.md. var _ = Describe("Bi Directional", Label("bi-directional"), Ordered, func() { var run biDirectionalRun var testNs string @@ -127,7 +137,7 @@ var _ = Describe("Bi Directional", Label("bi-directional"), Ordered, func() { // Namespace deletion is enough for ordinary namespaced leftovers like test Secrets and // IceCreamOrder instances. Keep explicit cleanup only for Flux resources and cluster-scoped // resources that live outside the test namespace. - cleanupClusterResource("crd", "icecreamorders.shop.example.com") + cleanupClusterResource("crd", iceCreamCRDName(crdGroupBiDirectional)) cleanupNamespace(testNs) }) @@ -167,10 +177,12 @@ var _ = Describe("Bi Directional", Label("bi-directional"), Ordered, func() { Name string Namespace string DestinationName string + Group string }{ Name: run.watchRuleName, Namespace: testNs, DestinationName: run.gitTargetName, + Group: crdGroupBiDirectional, }, testNs) Expect(err).NotTo(HaveOccurred(), "failed to apply bi-directional WatchRule") verifyResourceStatus("gitprovider", run.gitProviderName, testNs, "True", "Ready", "") @@ -217,7 +229,7 @@ var _ = Describe("Bi Directional", Label("bi-directional"), Ordered, func() { _, err = kubectlRunInNamespace( testNs, "patch", - "icecreamorder", + iceCreamCRDName(crdGroupBiDirectional), run.firstOrderName, "--type=merge", "--patch", @@ -377,7 +389,7 @@ func (r biDirectionalRun) waitForCRDEstablished() { output, err := kubectlRun( "get", "crd", - "icecreamorders.shop.example.com", + iceCreamCRDName(crdGroupBiDirectional), "-o", "jsonpath={.status.conditions[?(@.type=='Established')].status}", ) @@ -394,9 +406,7 @@ func (r biDirectionalRun) repoPath(parts ...string) string { func (r biDirectionalRun) liveOrderPath(name string) string { return r.repoPath( r.livePath, - "shop.example.com", - "v1", - "icecreamorders", + iceCreamInstanceDir(crdGroupBiDirectional), r.testNs, name+".yaml", ) @@ -413,8 +423,11 @@ func (r biDirectionalRun) liveSecretPath(name string) string { } func (r biDirectionalRun) writeCRDToRepo() { - content, err := os.ReadFile("test/e2e/templates/icecreamorder-crd.yaml") - Expect(err).NotTo(HaveOccurred(), "failed to read IceCreamOrder CRD fixture") + content, err := renderTemplate( + "test/e2e/templates/icecreamorder-crd.tmpl", + struct{ Group string }{Group: crdGroupBiDirectional}, + ) + Expect(err).NotTo(HaveOccurred(), "failed to render IceCreamOrder CRD fixture") Expect( os.MkdirAll( r.repoPath(r.crdPath, "apiextensions.k8s.io", "v1", "customresourcedefinitions"), @@ -428,9 +441,9 @@ func (r biDirectionalRun) writeCRDToRepo() { "apiextensions.k8s.io", "v1", "customresourcedefinitions", - "icecreamorders.shop.example.com.yaml", + iceCreamCRDMirrorFile(crdGroupBiDirectional), ), - content, + []byte(content), 0o644, ), ).To(Succeed()) @@ -446,6 +459,7 @@ func (r biDirectionalRun) writeLiveOrder(order iceCreamOrderFile) { Container string Scoops []iceCreamScoop Toppings []string + Group string }{ Name: order.Name, Namespace: order.Namespace, @@ -455,6 +469,7 @@ func (r biDirectionalRun) writeLiveOrder(order iceCreamOrderFile) { Container: order.Container, Scoops: order.Scoops, Toppings: order.Toppings, + Group: crdGroupBiDirectional, }) Expect(err).NotTo(HaveOccurred(), "failed to render IceCreamOrder manifest") Expect(os.MkdirAll(filepath.Dir(r.liveOrderPath(order.Name)), 0o755)).To(Succeed()) @@ -831,7 +846,7 @@ func (r biDirectionalRun) waitForOrderSpec(name, container, flavor, topping stri output, err := kubectlRunInNamespace( r.testNs, "get", - "icecreamorder", + iceCreamCRDName(crdGroupBiDirectional), name, "-o", "json", @@ -1039,7 +1054,7 @@ func (r biDirectionalRun) waitForSecretValue(name, key, expectedValue string) { func (r biDirectionalRun) waitForOrderDeleted(name string) { Eventually(func(g Gomega) { - _, err := kubectlRunInNamespace(r.testNs, "get", "icecreamorder", name, "-o", "json") + _, err := kubectlRunInNamespace(r.testNs, "get", iceCreamCRDName(crdGroupBiDirectional), name, "-o", "json") g.Expect(err).To(HaveOccurred()) g.Expect(strings.ToLower(err.Error())).To(ContainSubstring("not found")) }, biEventuallyTimeout, biPollInterval).Should(Succeed()) diff --git a/test/e2e/cluster/start-cluster.sh b/test/e2e/cluster/start-cluster.sh index b71970fd9..1a0df850f 100644 --- a/test/e2e/cluster/start-cluster.sh +++ b/test/e2e/cluster/start-cluster.sh @@ -11,6 +11,7 @@ DISABLE_K3S_TRAEFIK="${DISABLE_K3S_TRAEFIK:-true}" DISABLE_K3S_SERVICELB="${DISABLE_K3S_SERVICELB:-true}" KUBE_APISERVER_MAX_REQUESTS_INFLIGHT="${KUBE_APISERVER_MAX_REQUESTS_INFLIGHT:-800}" KUBE_APISERVER_MAX_MUTATING_REQUESTS_INFLIGHT="${KUBE_APISERVER_MAX_MUTATING_REQUESTS_INFLIGHT:-400}" +K3D_AGENT_COUNT="${K3D_AGENT_COUNT:-3}" AUDIT_DIR_REL="${AUDIT_DIR_REL:-test/e2e/cluster/audit}" K3D_CREATE_LOG_FILE="${TMPDIR:-/tmp}/k3d-create-${CLUSTER_NAME}.log" REPO_PWD="$(pwd -P)" @@ -74,7 +75,41 @@ cluster_is_healthy() { return 1 fi - kubectl --context "${context_name}" --request-timeout=10s get ns >/dev/null 2>&1 + kubectl --context "${context_name}" --request-timeout=10s get ns >/dev/null 2>&1 || return 1 + + local expected_nodes + expected_nodes="$((K3D_AGENT_COUNT + 1))" + kubectl --context "${context_name}" --request-timeout=10s get nodes \ + -o jsonpath='{range .items[*]}{.metadata.name}{" "}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\n"}{end}' \ + 2>/dev/null \ + | awk -v expected="${expected_nodes}" ' + BEGIN { total = 0; ready = 0 } + NF >= 2 { + total++ + if ($2 == "True") { + ready++ + } + } + END { + exit !(total == expected && ready == expected) + } + ' +} + +wait_cluster_healthy() { + local context_name attempt + context_name="$(cluster_context_name)" + + for attempt in $(seq 1 60); do + if cluster_is_healthy; then + return 0 + fi + sleep 2 + done + + echo "Current node status for ${context_name}:" >&2 + kubectl --context "${context_name}" get nodes -o wide >&2 || true + return 1 } ensure_k3d_stat_compat_path() { local host_project_path="$1" @@ -172,7 +207,7 @@ create_cluster() { k3d cluster create "${CLUSTER_NAME}" --image rancher/k3s:v1.35.2-k3s1 --servers 1 - --agents 1 + --agents "${K3D_AGENT_COUNT}" --kubeconfig-update-default --kubeconfig-switch-context -v "${audit_host_dir}:/etc/kubernetes/audit@server:0" @@ -263,7 +298,7 @@ main() { merge_kubeconfig rewrite_kubeconfig_for_devcontainer - if ! cluster_is_healthy; then + if ! wait_cluster_healthy; then echo "❌ Existing k3d cluster '${CLUSTER_NAME}' is unhealthy." >&2 echo "Refusing to auto-recreate it because cluster-scoped stamps may now be inconsistent." >&2 echo "" >&2 diff --git a/test/e2e/commit_author_attribution_e2e_test.go b/test/e2e/commit_author_attribution_e2e_test.go new file mode 100644 index 000000000..cac8bb352 --- /dev/null +++ b/test/e2e/commit_author_attribution_e2e_test.go @@ -0,0 +1,177 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Commit Author Attribution proves the end-to-end identity chain: a write made +// by an impersonated user whose audit event carries OIDC name/email claims in +// user.extra ends up as the Git commit author. The kube-apiserver emits the +// real audit event, the controller's audit pipeline consumes it, and the +// resulting commit must be attributed to that identity. The author-mapping +// *logic* is unit-tested (internal/queue redis_audit_consumer_test.go +// TestResolveUserInfo); this spec is the only place that proves the full +// apiserver → audit webhook → consumer → git-author wiring carries the keys +// through. +// +// Not Serial: the dedicated GitProvider uses a 0s commit window, so every +// watched event is committed immediately as its own commit (no batching). Each +// assertion reads the author scoped to its own unique file path, so concurrent +// audit traffic from other specs lands in separate commits and cannot change +// the author of this spec's commit. +var _ = Describe("Commit Author Attribution", Label("manager", "smoke"), Ordered, func() { + var ( + testNs string + repo *RepoArtifacts + gitProvName string + gitTargetName string + watchRuleName string + ) + + const basePath = "e2e/commit-author-test" + + BeforeAll(func() { + By("creating commit-author test namespace") + testNs = testNamespaceFor("commit-author") + _, _ = kubectlRun("create", "namespace", testNs) // idempotent; ignore AlreadyExists + + By("setting up the Gitea repo and credentials") + repo = SetupRepo( + resolveE2EContext(), + testNs, + fmt.Sprintf("e2e-commit-author-%d", GinkgoRandomSeed()), + ) + _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") + applySOPSAgeKeyToNamespace(testNs) + + By("setting up GitProvider (0s commit window), GitTarget and WatchRule") + seed := GinkgoRandomSeed() + gitProvName = fmt.Sprintf("commit-author-gitprovider-%d", seed) + gitTargetName = fmt.Sprintf("commit-author-gittarget-%d", seed) + watchRuleName = fmt.Sprintf("commit-author-watchrule-%d", seed) + + // createGitProviderWithURLInNamespace uses a 0s commit window, so each + // event commits immediately and never coalesces with another's. + createGitProviderWithURLInNamespace(gitProvName, testNs, repo.GitSecretHTTP, repo.RepoURLHTTP) + verifyResourceStatus("gitprovider", gitProvName, testNs, "True", "Ready", "") + + createGitTarget(gitTargetName, testNs, gitProvName, basePath, "main") + verifyResourceStatus("gittarget", gitTargetName, testNs, "True", "Ready", "") + + data := struct { + Name string + Namespace string + DestinationName string + }{ + Name: watchRuleName, + Namespace: testNs, + DestinationName: gitTargetName, + } + err = applyFromTemplate("test/e2e/templates/manager/watchrule-configmap.tmpl", data, testNs) + Expect(err).NotTo(HaveOccurred(), "failed to apply WatchRule") + verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") + }) + + AfterAll(func() { + cleanupWatchRule(watchRuleName, testNs) + cleanupGitTarget(gitTargetName, testNs) + cleanupNamespacedResource(testNs, "gitprovider", gitProvName) + cleanupNamespace(testNs) + }) + + // assertCommitAuthorFromOIDCClaims creates a ConfigMap while impersonating a + // user that carries the given OIDC display-name/email claims in user.extra, + // then asserts the commit for that ConfigMap's unique path is authored by + // " ". + assertCommitAuthorFromOIDCClaims := func(cmName, asUser, displayName, email string) { + GinkgoHelper() + + By(fmt.Sprintf("creating ConfigMap %q while impersonating %q with OIDC name/email extras", cmName, asUser)) + // Impersonation makes the real kube-apiserver emit an audit event whose + // impersonatedUser.Extra carries these keys, mimicking the claims a + // structured authentication config maps into user.extra for an OIDC + // login. system:masters keeps the impersonated create authorized + // without provisioning per-user RBAC. + err := createConfigMapAsImpersonatedUser( + testNs, + cmName, + asUser, + []string{"system:masters"}, + map[string][]string{ + "configbutler.ai/claims/display-name": {displayName}, + "configbutler.ai/claims/email": {email}, + }, + ) + Expect(err).NotTo(HaveOccurred(), "failed to create impersonated ConfigMap") + + repoPath := path.Join(basePath, fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, cmName)) + expectedFile := filepath.Join(repo.CheckoutDir, repoPath) + + By("waiting for the commit and asserting its author is the OIDC identity") + Eventually(func(g Gomega) { + pullLatestRepoState(g, repo.CheckoutDir) + + info, statErr := os.Stat(expectedFile) + g.Expect(statErr).NotTo(HaveOccurred(), + fmt.Sprintf("ConfigMap file should exist at %s", expectedFile)) + g.Expect(info.Size()).To(BeNumerically(">", 0)) + + // Scope the log to this ConfigMap's unique path so the author read + // back is unambiguously the commit produced by this impersonated + // create, independent of any concurrent audit traffic. + out, logErr := gitRun(repo.CheckoutDir, "log", "-1", "--pretty=%an <%ae>", "--", repoPath) + g.Expect(logErr).NotTo(HaveOccurred(), fmt.Sprintf("git log author failed: %s", out)) + g.Expect(strings.TrimSpace(out)).To( + Equal(fmt.Sprintf("%s <%s>", displayName, email)), + "commit author should be the OIDC display name and email from user.extra") + }, 3*time.Minute, 3*time.Second).Should(Succeed()) + + By("cleaning up the test ConfigMap") + _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", cmName, "--ignore-not-found=true") + } + + It("attributes a commit to the OIDC display name and email from user.extra", func() { + assertCommitAuthorFromOIDCClaims( + fmt.Sprintf("commit-author-oidc-cm-%d", GinkgoRandomSeed()), + "oidc-simon", + "Simon Koudijs", + "something@configbutler.ai", + ) + }) + + It("attributes a second commit to a different OIDC identity (claims are data-driven)", func() { + assertCommitAuthorFromOIDCClaims( + fmt.Sprintf("commit-author-oidc-cm2-%d", GinkgoRandomSeed()), + "oidc-ada", + "Ada Lovelace", + "ada@configbutler.ai", + ) + }) +}) diff --git a/test/e2e/commit_request_e2e_test.go b/test/e2e/commit_request_e2e_test.go index dec1a3364..f667ad6ad 100644 --- a/test/e2e/commit_request_e2e_test.go +++ b/test/e2e/commit_request_e2e_test.go @@ -19,7 +19,6 @@ limitations under the License. package e2e import ( - "context" "fmt" "os" "path/filepath" @@ -36,9 +35,17 @@ import ( // a deliberately long commitWindow so that, without the CommitRequest, the // edit would not be committed for minutes — observing the commit promptly is // what proves the commit-request path works. -var _ = Describe("Commit Request", Label("commit-request", "audit-redis", "smoke"), Ordered, func() { +// +// Not Serial: this spec owns a dedicated Gitea repo (its own GitProvider → +// GitTarget → namespace-scoped WatchRule), so the only writer to its main +// branch is its own GitTarget, fed exclusively by audit events from its own +// namespace. The HEAD/SHA assertions below therefore read back only this spec's +// own commit; concurrent audit traffic for other GitTargets lands in other +// repos and cannot move this repo's HEAD. See docs/design/e2e-serial-registry.md. +var _ = Describe("Commit Request", Label("commit-request", "audit-consumer", "smoke"), Ordered, func() { var ( testNs string + repo *RepoArtifacts gitProvName string gitTargetName string watchRuleName string @@ -52,7 +59,11 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-redis", "smoke By("creating commit-request test namespace and applying git secrets") testNs = testNamespaceFor("commit-request") _, _ = kubectlRun("create", "namespace", testNs) - repo := ensureAuditRedisRepo() + repo = SetupRepo( + resolveE2EContext(), + testNs, + fmt.Sprintf("e2e-commit-request-%d", GinkgoRandomSeed()), + ) _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to namespace") applySOPSAgeKeyToNamespace(testNs) @@ -91,7 +102,6 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-redis", "smoke }) It("finalizes the open commit window on demand and reports the resulting SHA", func() { - repo := auditRedisRepo basePath := "e2e/commit-request-test" seed := GinkgoRandomSeed() cmName := fmt.Sprintf("commit-request-cm-%d", seed) @@ -101,26 +111,6 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-redis", "smoke By("editing a ConfigMap to open a commit window") applyConfigMap(testNs, cmName) - By("waiting for the ConfigMap audit event to reach the Redis stream") - // Confirming the producer side guarantees the edit is observable by the - // consumer before the CommitRequest's own (later) audit event. - valkeyClient := newE2EValkeyClient() - defer func() { _ = valkeyClient.Close() }() - Eventually(func(g Gomega) { - entries, readErr := valkeyClient.XRange(context.Background(), defaultAuditRedisStream, "-", "+").Result() - g.Expect(readErr).NotTo(HaveOccurred()) - found := false - for _, entry := range entries { - resource, _ := entry.Values["resource"].(string) - name, _ := entry.Values["name"].(string) - if resource == "configmaps" && name == cmName { - found = true - break - } - } - g.Expect(found).To(BeTrue(), "ConfigMap audit event should appear in the stream") - }, 2*time.Minute, 2*time.Second).Should(Succeed()) - By("confirming the edit has NOT been committed yet (long commitWindow)") Consistently(func(g Gomega) { pullLatestRepoState(g, repo.CheckoutDir) @@ -177,7 +167,6 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-redis", "smoke // from responseObject. Without the fix the CommitRequest stays stuck in // WaitingForAuditEvent forever. It("finalizes a CommitRequest created with metadata.generateName", func() { - repo := auditRedisRepo basePath := "e2e/commit-request-test" seed := GinkgoRandomSeed() cmName := fmt.Sprintf("commit-request-gen-cm-%d", seed) diff --git a/test/e2e/commit_window_batching_e2e_test.go b/test/e2e/commit_window_batching_e2e_test.go index a0eda207a..3aa466e24 100644 --- a/test/e2e/commit_window_batching_e2e_test.go +++ b/test/e2e/commit_window_batching_e2e_test.go @@ -32,134 +32,147 @@ import ( // Commit-window batching exercises the grouped-commit path: multiple events // arriving within the rolling silence window collapse into one grouped commit -// and one push. The audit-redis consumer pipeline is the real path under test -// — events flow kubectl → audit webhook → Valkey stream → consumer → +// and one push. The audit consumer pipeline is the real path under test: +// events flow kubectl → audit webhook → consumer → // BranchWorker.commitGroups → BranchWorker.pushPendingCommits. -var _ = Describe("Commit Window Batching", Label("commit-window-batching", "audit-redis", "smoke"), Ordered, func() { - var ( - testNs string - gitProvName string - gitTargetName string - watchRuleName string - ) - - const commitWindow = "3s" - - BeforeAll(func() { - By("creating commit-window-batching test namespace and applying git secrets") - testNs = testNamespaceFor("commit-window-batching") - _, _ = kubectlRun("create", "namespace", testNs) - repo := ensureAuditRedisRepo() - _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) - Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to namespace") - applySOPSAgeKeyToNamespace(testNs) - - seed := GinkgoRandomSeed() - gitProvName = fmt.Sprintf("commit-window-gitprovider-%d", seed) - gitTargetName = fmt.Sprintf("commit-window-gittarget-%d", seed) - watchRuleName = fmt.Sprintf("commit-window-watchrule-%d", seed) - - By(fmt.Sprintf("creating GitProvider with commitWindow=%s", commitWindow)) - createGitProviderWithCommitWindow( - gitProvName, - testNs, - repo.GitSecretHTTP, - repo.RepoURLHTTP, - commitWindow, +// +// Not Serial: this spec owns a dedicated Gitea repo, so the commit-count delta +// it asserts on counts only its own GitTarget's commits — fed exclusively by +// audit events from its own namespace. The batching property (a same-author +// burst inside one commit window collapses into one commit) is a per-GitTarget +// branch-worker behaviour: each GitTarget runs its own commit window, so events +// routed to other GitTargets cannot land in this spec's grouped commit. See +// docs/design/e2e-serial-registry.md. +var _ = Describe("Commit Window Batching", + Label("commit-window-batching", "audit-consumer", "smoke"), Ordered, func() { + var ( + testNs string + repo *RepoArtifacts + gitProvName string + gitTargetName string + watchRuleName string ) - verifyResourceStatus("gitprovider", gitProvName, testNs, "True", "Ready", "") - - createGitTarget(gitTargetName, testNs, gitProvName, "e2e/commit-window-test", "main") - verifyResourceStatus("gittarget", gitTargetName, testNs, "True", "Ready", "") - - watchRuleData := struct { - Name string - Namespace string - DestinationName string - }{ - Name: watchRuleName, - Namespace: testNs, - DestinationName: gitTargetName, - } - err = applyFromTemplate("test/e2e/templates/manager/watchrule-configmap.tmpl", watchRuleData, testNs) - Expect(err).NotTo(HaveOccurred(), "failed to apply WatchRule") - verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") - }) - - AfterAll(func() { - cleanupWatchRule(watchRuleName, testNs) - cleanupGitTarget(gitTargetName, testNs) - cleanupNamespacedResource(testNs, "gitprovider", gitProvName) - cleanupNamespace(testNs) - }) - It("collapses a burst of events into one grouped commit and one push", func() { - const burstSize = 4 - - repo := auditRedisRepo - basePath := "e2e/commit-window-test" - seed := GinkgoRandomSeed() - burstPrefix := fmt.Sprintf("commit-window-burst-%d", seed) - - // Seed: ensure the repo has at least one commit for this target so - // commit-count math is meaningful and not racing against bootstrap. - seedCM := fmt.Sprintf("%s-seed", burstPrefix) - applyConfigMap(testNs, seedCM) - Eventually(func(g Gomega) { - pullLatestRepoState(g, repo.CheckoutDir) - expected := filepath.Join(repo.CheckoutDir, basePath, - "v1", "configmaps", testNs, seedCM+".yaml") - _, statErr := os.Stat(expected) - g.Expect(statErr).NotTo(HaveOccurred(), - fmt.Sprintf("seed ConfigMap should land before the burst: %s", expected)) - }, 90*time.Second, 2*time.Second).Should(Succeed()) - - commitsBefore := mustCommitCount(repo.CheckoutDir) - - By(fmt.Sprintf("creating %d ConfigMaps in rapid succession", burstSize)) - burstStart := time.Now() - burstNames := make([]string, burstSize) - for i := range burstSize { - name := fmt.Sprintf("%s-%d", burstPrefix, i) - burstNames[i] = name - applyConfigMap(testNs, name) - } - burstSubmitted := time.Since(burstStart) - // Burst must fit comfortably inside the commit window so the - // batching property is what we are actually exercising. - Expect(burstSubmitted).To(BeNumerically("<", 2*time.Second), - "burst should be submitted faster than the commit window") - - By("waiting for every burst ConfigMap to land in the repository") - Eventually(func(g Gomega) { - pullLatestRepoState(g, repo.CheckoutDir) - for _, name := range burstNames { + const commitWindow = "3s" + + BeforeAll(func() { + By("creating commit-window-batching test namespace and applying git secrets") + testNs = testNamespaceFor("commit-window-batching") + _, _ = kubectlRun("create", "namespace", testNs) + repo = SetupRepo( + resolveE2EContext(), + testNs, + fmt.Sprintf("e2e-commit-window-%d", GinkgoRandomSeed()), + ) + _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to namespace") + applySOPSAgeKeyToNamespace(testNs) + + seed := GinkgoRandomSeed() + gitProvName = fmt.Sprintf("commit-window-gitprovider-%d", seed) + gitTargetName = fmt.Sprintf("commit-window-gittarget-%d", seed) + watchRuleName = fmt.Sprintf("commit-window-watchrule-%d", seed) + + By(fmt.Sprintf("creating GitProvider with commitWindow=%s", commitWindow)) + createGitProviderWithCommitWindow( + gitProvName, + testNs, + repo.GitSecretHTTP, + repo.RepoURLHTTP, + commitWindow, + ) + verifyResourceStatus("gitprovider", gitProvName, testNs, "True", "Ready", "") + + createGitTarget(gitTargetName, testNs, gitProvName, "e2e/commit-window-test", "main") + verifyResourceStatus("gittarget", gitTargetName, testNs, "True", "Ready", "") + + watchRuleData := struct { + Name string + Namespace string + DestinationName string + }{ + Name: watchRuleName, + Namespace: testNs, + DestinationName: gitTargetName, + } + err = applyFromTemplate("test/e2e/templates/manager/watchrule-configmap.tmpl", watchRuleData, testNs) + Expect(err).NotTo(HaveOccurred(), "failed to apply WatchRule") + verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") + }) + + AfterAll(func() { + cleanupWatchRule(watchRuleName, testNs) + cleanupGitTarget(gitTargetName, testNs) + cleanupNamespacedResource(testNs, "gitprovider", gitProvName) + cleanupNamespace(testNs) + }) + + It("collapses a burst of events into one grouped commit and one push", func() { + const burstSize = 4 + + basePath := "e2e/commit-window-test" + seed := GinkgoRandomSeed() + burstPrefix := fmt.Sprintf("commit-window-burst-%d", seed) + + // Seed: ensure the repo has at least one commit for this target so + // commit-count math is meaningful and not racing against bootstrap. + seedCM := fmt.Sprintf("%s-seed", burstPrefix) + applyConfigMap(testNs, seedCM) + Eventually(func(g Gomega) { + pullLatestRepoState(g, repo.CheckoutDir) expected := filepath.Join(repo.CheckoutDir, basePath, - "v1", "configmaps", testNs, name+".yaml") + "v1", "configmaps", testNs, seedCM+".yaml") _, statErr := os.Stat(expected) g.Expect(statErr).NotTo(HaveOccurred(), - fmt.Sprintf("ConfigMap file should exist: %s", expected)) + fmt.Sprintf("seed ConfigMap should land before the burst: %s", expected)) + }, 90*time.Second, 2*time.Second).Should(Succeed()) + + commitsBefore := mustCommitCount(repo.CheckoutDir) + + By(fmt.Sprintf("creating %d ConfigMaps in rapid succession", burstSize)) + burstStart := time.Now() + burstNames := make([]string, burstSize) + for i := range burstSize { + name := fmt.Sprintf("%s-%d", burstPrefix, i) + burstNames[i] = name + applyConfigMap(testNs, name) } - }, 90*time.Second, 2*time.Second).Should(Succeed()) - - commitsAfter := mustCommitCount(repo.CheckoutDir) - commitsAdded := commitsAfter - commitsBefore - - // Phase 3 groups same-author events in the window into one commit. The - // batching property is observable as one new remote commit touching the - // whole burst after the commitWindow expires. - Expect(commitsAdded).To(Equal(1), - fmt.Sprintf("expected 1 grouped commit for the burst, got %d", commitsAdded)) - - assertBurstFilesAreGroupedIntoLatestCommit(repo.CheckoutDir, burstNames, basePath, testNs) - - By("cleaning up burst ConfigMaps") - for _, name := range burstNames { - _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", name, "--ignore-not-found=true") - } - _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", seedCM, "--ignore-not-found=true") + burstSubmitted := time.Since(burstStart) + // Burst must fit comfortably inside the commit window so the + // batching property is what we are actually exercising. + Expect(burstSubmitted).To(BeNumerically("<", 2*time.Second), + "burst should be submitted faster than the commit window") + + By("waiting for every burst ConfigMap to land in the repository") + Eventually(func(g Gomega) { + pullLatestRepoState(g, repo.CheckoutDir) + for _, name := range burstNames { + expected := filepath.Join(repo.CheckoutDir, basePath, + "v1", "configmaps", testNs, name+".yaml") + _, statErr := os.Stat(expected) + g.Expect(statErr).NotTo(HaveOccurred(), + fmt.Sprintf("ConfigMap file should exist: %s", expected)) + } + }, 90*time.Second, 2*time.Second).Should(Succeed()) + + commitsAfter := mustCommitCount(repo.CheckoutDir) + commitsAdded := commitsAfter - commitsBefore + + // Phase 3 groups same-author events in the window into one commit. The + // batching property is observable as one new remote commit touching the + // whole burst after the commitWindow expires. + Expect(commitsAdded).To(Equal(1), + fmt.Sprintf("expected 1 grouped commit for the burst, got %d", commitsAdded)) + + assertBurstFilesAreGroupedIntoLatestCommit(repo.CheckoutDir, burstNames, basePath, testNs) + + By("cleaning up burst ConfigMaps") + for _, name := range burstNames { + _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", name, "--ignore-not-found=true") + } + _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", seedCM, "--ignore-not-found=true") + }) }) -}) // applyConfigMap renders the standard manager configmap template and applies // it. The template carries the ConfigMap data the burst-event audit pipeline diff --git a/test/e2e/controller_basics_e2e_test.go b/test/e2e/controller_basics_e2e_test.go new file mode 100644 index 000000000..70ac73a02 --- /dev/null +++ b/test/e2e/controller_basics_e2e_test.go @@ -0,0 +1,227 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/ConfigButler/gitops-reverser/test/utils" +) + +var _ = Describe("Manager Controller Basics", Label("manager"), Ordered, func() { + var controllerPodName string // Name of first controller pod for logging + + BeforeAll(func() { + By("setting up Prometheus client for metrics testing") + setupPrometheusClient() + verifyPrometheusAvailable() + }) + + SetDefaultEventuallyTimeout(30 * time.Second) + SetDefaultEventuallyPollingInterval(time.Second) + + It("should run successfully", Label("smoke"), func() { + By("validating that the gitops-reverser pods are running as expected") + verifyControllerUp := func(g Gomega) { + // Get the names of the gitops-reverser pods + podOutput, err := kubectlRunInNamespace( + namespace, + "get", + "pods", + "-l", + "control-plane=gitops-reverser", + "-o", "go-template={{ range .items }}"+ + "{{ if not .metadata.deletionTimestamp }}"+ + "{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}", + ) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve gitops-reverser pod information") + podNames := utils.GetNonEmptyLines(podOutput) + g.Expect(podNames).To(HaveLen(1), "expected exactly 1 controller pod running") + controllerPodName = podNames[0] // Use first pod for logging + g.Expect(controllerPodName).To(ContainSubstring("gitops-reverser")) + + // Validate all pods' status + for _, podName := range podNames { + output, err := kubectlRunInNamespace( + namespace, + "get", + "pods", + podName, + "-o", + "jsonpath={.status.phase}", + ) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running"), fmt.Sprintf("Incorrect status for pod %s", podName)) + } + } + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should expose the controller service", Label("smoke"), func() { + By("verifying controller service exists") + _, err := kubectlRunInNamespace(namespace, "get", "svc", controllerServiceName) + Expect(err).NotTo(HaveOccurred(), "Controller service should exist") + + By("verifying controller service routes to the controller pod") + Eventually(func(g Gomega) { + expectServiceRoutesToPod(g, controllerServiceName, controllerPodName) + }, 30*time.Second).Should(Succeed()) + }) + + It("should expose the audit service separately", func() { + By("verifying audit service exists") + _, err := kubectlRunInNamespace(namespace, "get", "svc", auditServiceName) + Expect(err).NotTo(HaveOccurred(), "Audit service should exist") + + By("verifying audit service routes to the controller pod") + Eventually(func(g Gomega) { + expectServiceRoutesToPod(g, auditServiceName, controllerPodName) + }, 30*time.Second).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", Label("smoke"), func() { + By("validating that the controller service is available for metrics") + _, err := kubectlRunInNamespace(namespace, "get", "service", controllerServiceName) + Expect(err).NotTo(HaveOccurred(), "Controller service should exist") + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + output, err := kubectlRunInNamespace(namespace, "get", "endpoints", controllerServiceName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + output, err := kubectlRunInNamespace(namespace, "logs", controllerPodName) + g.Expect(err).NotTo(HaveOccurred()) + jsonMetricsLogLine := "\"logger\":\"controller-runtime.metrics\"," + + "\"msg\":\"Serving metrics server\"" + g.Expect(output).To( + SatisfyAny( + ContainSubstring("controller-runtime.metrics\tServing metrics server"), + ContainSubstring(jsonMetricsLogLine), + ), + "Metrics server not yet started", + ) + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("waiting for Prometheus to scrape controller metrics") + waitForMetric("sum(up{job='gitops-reverser'})", + func(v float64) bool { return v == 1 }, + "metrics endpoint should be up") + + By("verifying basic process metrics are exposed") + waitForMetric("sum(process_cpu_seconds_total{job='gitops-reverser'})", + func(v float64) bool { return v > 0 }, + "process metrics should exist") + + By("verifying metrics from the controller pod") + podCount, err := queryPrometheus("sum(up{job='gitops-reverser'})") + Expect(err).NotTo(HaveOccurred()) + Expect(podCount).To(BeEquivalentTo(1), "Should scrape from 1 controller pod") + + fmt.Printf("✅ Metrics collection verified from %.0f pods\n", podCount) + fmt.Printf("📊 Inspect metrics: %s\n", getPrometheusURL()) + }) + + It("should receive audit webhook events from kube-apiserver", Label("smoke"), func() { + By("recording baseline audit event count") + baselineAuditEvents, err := queryPrometheus("sum(gitopsreverser_audit_events_received_total) or vector(0)") + Expect(err).NotTo(HaveOccurred()) + fmt.Printf("📊 Baseline audit events: %.0f\n", baselineAuditEvents) + baselineOfficialAuditEvents, err := queryPrometheus( + "sum(gitopsreverser_audit_events_received_total{source='official'}) or vector(0)", + ) + Expect(err).NotTo(HaveOccurred()) + fmt.Printf("📊 Baseline official audit events: %.0f\n", baselineOfficialAuditEvents) + + By("creating a ConfigMap to trigger audit events") + _, err = kubectlRunInNamespace( + namespace, + "create", + "configmap", + "audit-test-cm", + "--from-literal=test=audit", + ) + Expect(err).NotTo(HaveOccurred(), "ConfigMap creation should succeed") + _, err = kubectlRunInNamespace( + namespace, + "patch", + "configmap", + "audit-test-cm", + "--type=merge", + "--patch", + `{"data":{"test":"audit-updated"}}`, + ) + Expect(err).NotTo(HaveOccurred(), "ConfigMap update should succeed") + + By("waiting for audit event metric to increment") + waitForMetricWithTimeout("sum(gitopsreverser_audit_events_received_total) or vector(0)", + func(v float64) bool { return v > baselineAuditEvents }, + "audit events should increment", 2*time.Minute) + waitForMetricWithTimeout( + "sum(gitopsreverser_audit_events_received_total{source='official'}) or vector(0)", + func(v float64) bool { return v > baselineOfficialAuditEvents }, + "audit events should increment for source=official", 2*time.Minute, + ) + + By("verifying audit events were received") + currentAuditEvents, err := queryPrometheus("sum(gitopsreverser_audit_events_received_total) or vector(0)") + Expect(err).NotTo(HaveOccurred()) + Expect(currentAuditEvents).To(BeNumerically(">", baselineAuditEvents), + "Should have received audit events from kube-apiserver") + + By("verifying the EventList ingress metric appears for source=official") + waitForMetricWithTimeout( + "sum(gitopsreverser_audit_eventlists_total{source='official'}) or vector(0)", + func(v float64) bool { return v > 0 }, + "EventList ingress requests should be counted for source=official", 2*time.Minute, + ) + + By("verifying no removed cluster/gvr/action label remains on any audit series") + for _, removed := range []string{"cluster", "gvr", "action"} { + stale, queryErr := queryPrometheus(fmt.Sprintf( + "sum({__name__=~'gitopsreverser_audit_.+', %s=~'.+'}) or vector(0)", removed)) + Expect(queryErr).NotTo(HaveOccurred()) + Expect(stale).To(BeNumerically("==", 0), + fmt.Sprintf("no audit series should still carry the %q label", removed)) + } + + newEvents := currentAuditEvents - baselineAuditEvents + fmt.Printf("✅ Received %.0f new audit events from kube-apiserver\n", newEvents) + fmt.Printf("📊 Total audit events: %.0f\n", currentAuditEvents) + + By("cleaning up audit test resources") + _, _ = kubectlRunInNamespace( + namespace, + "delete", + "configmap", + "audit-test-cm", + "--ignore-not-found=true", + ) + }) +}) diff --git a/test/e2e/crd_lifecycle_e2e_test.go b/test/e2e/crd_lifecycle_e2e_test.go new file mode 100644 index 000000000..0fbea3768 --- /dev/null +++ b/test/e2e/crd_lifecycle_e2e_test.go @@ -0,0 +1,693 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Manager CRD Lifecycle", Label("manager"), Ordered, func() { + var ( + testNs string + crdLifecycleRepo *RepoArtifacts + ) + + BeforeAll(func() { + By("creating CRD lifecycle test namespace") + testNs = testNamespaceFor("manager-crd") + _, _ = kubectlRun("create", "namespace", testNs) // idempotent; ignore AlreadyExists + + By("setting up Gitea repo and credentials for CRD lifecycle tests") + crdLifecycleRepo = SetupRepo( + resolveE2EContext(), + testNs, + fmt.Sprintf("e2e-manager-crd-%d", GinkgoRandomSeed()), + ) + + By("applying git secrets to test namespace") + _, err := kubectlRunInNamespace(testNs, "apply", "-f", crdLifecycleRepo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") + applySOPSAgeKeyToNamespace(testNs) + + By("creating shared gitprovider-normal for CRD lifecycle specs") + createGitProviderWithURLInNamespace( + "gitprovider-normal", + testNs, + crdLifecycleRepo.GitSecretHTTP, + crdLifecycleRepo.RepoURLHTTP, + ) + verifyResourceStatus( + "gitprovider", "gitprovider-normal", testNs, + "True", "Ready", "Repository connectivity validated", + ) + }) + + AfterAll(func() { + By("cleaning up CRD lifecycle test resources") + + // Clean up IceCreamOrder CRD (cluster-scoped — must delete explicitly) + _, _ = kubectlRun( + "delete", + "crd", + iceCreamCRDName(crdGroupCRDLifecycle), + "--ignore-not-found=true", + ) + + cleanupNamespace(testNs) + + By("test infrastructure still running for debugging") + fmt.Printf("\n") + fmt.Printf("═══════════════════════════════════════════════════════════\n") + fmt.Printf("📊 E2E Infrastructure kept running for debugging purposes:\n") + fmt.Printf("═══════════════════════════════════════════════════════════\n") + fmt.Printf(" Prometheus: http://localhost:19090\n") + fmt.Printf(" Gitea: http://localhost:13000\n") + fmt.Printf(" Flux UI: http://localhost:19080\n") + fmt.Printf("\n") + fmt.Printf("═══════════════════════════════════════════════════════════\n") + fmt.Printf("\n") + }) + + SetDefaultEventuallyTimeout(30 * time.Second) + SetDefaultEventuallyPollingInterval(time.Second) + + It("should create Git commit when IceCreamOrder CRD is installed via ClusterWatchRule", Label("smoke"), func() { + gitProviderName := "gitprovider-normal" + clusterWatchRuleName := "clusterwatchrule-crd-install" + crdName := iceCreamCRDName(crdGroupCRDLifecycle) + + By("creating ClusterWatchRule with Cluster scope for CRDs") + destName := clusterWatchRuleName + "-dest" + createGitTarget(destName, testNs, gitProviderName, "e2e/crd-install-test", "main") + + clusterWatchRuleData := struct { + Name string + DestinationName string + Namespace string + }{ + Name: clusterWatchRuleName, + DestinationName: destName, + Namespace: testNs, + } + + err := applyFromTemplate("test/e2e/templates/manager/clusterwatchrule-crd.tmpl", clusterWatchRuleData, "") + Expect(err).NotTo(HaveOccurred(), "Failed to apply ClusterWatchRule for CRDs") + + By("verifying ClusterWatchRule is ready") + verifyResourceStatus("clusterwatchrule", clusterWatchRuleName, "", "True", "Ready", "") + verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") + + By("installing the IceCreamOrder CRD to trigger Git commit") + err = applyIceCreamCRD(crdGroupCRDLifecycle) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRD") + + By("waiting for CRD to be established") + verifyCRDEstablished := func(g Gomega) { + output, err := kubectlRun( + "get", + "crd", + crdName, + "-o", + "jsonpath={.status.conditions[?(@.type=='Established')].status}", + ) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("True")) + } + Eventually(verifyCRDEstablished).Should(Succeed()) + + By("verifying CRD YAML file exists in Git repository (NO namespace in path - cluster resource)") + verifyGitCommit := func(g Gomega) { + pullLatestRepoState(g, crdLifecycleRepo.CheckoutDir) + + // CRDs are cluster-scoped, so path should NOT include namespace + expectedFile := filepath.Join(crdLifecycleRepo.CheckoutDir, + "e2e/crd-install-test", + "apiextensions.k8s.io/v1/customresourcedefinitions/"+iceCreamCRDMirrorFile(crdGroupCRDLifecycle)) + fileInfo, statErr := os.Stat(expectedFile) + g.Expect(statErr).NotTo(HaveOccurred(), fmt.Sprintf("CRD file should exist at %s", expectedFile)) + g.Expect(fileInfo.Size()).To(BeNumerically(">", 0), "CRD file should not be empty") + + // Verify file content + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred()) + g.Expect(string(content)).To(ContainSubstring("kind: CustomResourceDefinition"), + "File should contain CRD kind") + g.Expect(string(content)).To(ContainSubstring("name: "+iceCreamCRDName(crdGroupCRDLifecycle)), + "File should contain CRD name") + } + Eventually(verifyGitCommit). + WithTimeout(60 * time.Second). + WithPolling(2 * time.Second). + Should(Succeed()) + + By("cleaning up test resources") + cleanupClusterWatchRule(clusterWatchRuleName) + cleanupGitTarget(destName, testNs) + // Keep CRD installed for subsequent tests + + By("✅ CRD installation via ClusterWatchRule E2E test passed") + }) + + It("should create Git commit when IceCreamOrder is added via WatchRule", func() { + gitProviderName := "gitprovider-normal" + watchRuleName := "watchrule-icecream-orders" + + By("installing the IceCreamOrder CRD first (needed for custom resource tests)") + err := applyIceCreamCRD(crdGroupCRDLifecycle) + Expect(err).NotTo(HaveOccurred(), "Failed to install sample CRD") + + By("waiting for CRD to be established") + verifyCRDEstablished := func(g Gomega) { + output, err := kubectlRun( + "get", + "crd", + iceCreamCRDName(crdGroupCRDLifecycle), + "-o", + "jsonpath={.status.conditions[?(@.type=='Established')].status}", + ) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("True")) + } + Eventually(verifyCRDEstablished, 30*time.Second, time.Second).Should(Succeed()) + + // Established=True only means the apiserver registered the CRD; the + // controller's reflector may still be in "Failed to watch" backoff for + // a few seconds. If we create a CR before that recovers, the create + // event is missed and the next spec sees no file in git. Block until + // the resource is actually listable end-to-end. + Eventually(func(g Gomega) { + _, err := kubectlRun("get", iceCreamCRDName(crdGroupCRDLifecycle), "-A") + g.Expect(err).NotTo(HaveOccurred(), "icecreamorders must be servable before creating instances") + }, 30*time.Second, time.Second).Should(Succeed()) + + crdInstanceName := "alices-order" + uniqueRepoName := crdLifecycleRepo.RepoName + + By("creating WatchRule that monitors IceCreamOrder resources") + destName := watchRuleName + "-dest" + createGitTarget(destName, testNs, gitProviderName, "e2e/icecream-test", "main") + + data := struct { + Name string + Namespace string + DestinationName string + Group string + }{ + Name: watchRuleName, + Namespace: testNs, + DestinationName: destName, + Group: crdGroupCRDLifecycle, + } + + err2 := applyFromTemplate("test/e2e/templates/watchrule-crd.tmpl", data, testNs) + Expect(err2).NotTo(HaveOccurred(), "Failed to apply WatchRule for CRDs") + + By("verifying WatchRule is ready") + verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") + verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") + + By("creating CR with labels and annotations to trigger Git commit") + crdInstanceData := struct { + Name string + Namespace string + Labels map[string]string + Annotations map[string]string + CustomerName string + Container string + Scoops []struct { + Flavor string + Quantity int + } + Toppings []string + Group string + }{ + Name: crdInstanceName, + Namespace: testNs, + Labels: map[string]string{ + "environment": "test", + "team": "engineering", + }, + Annotations: map[string]string{ + "description": "Alice's favorite ice cream", + "priority": "high", + "kubectl.kubernetes.io/last-applied-configuration": "should-be-filtered", + "deployment.kubernetes.io/revision": "should-also-be-filtered", + }, + Group: crdGroupCRDLifecycle, + CustomerName: "Alice", + Container: "Cone", + Scoops: []struct { + Flavor string + Quantity int + }{ + {Flavor: "Vanilla", Quantity: 2}, + {Flavor: "Chocolate", Quantity: 1}, + }, + Toppings: []string{"Sprinkles", "HotFudge"}, + } + + err3 := applyFromTemplate("test/e2e/templates/icecreamorder-instance.tmpl", crdInstanceData, testNs) + Expect(err3).NotTo(HaveOccurred(), "Failed to apply CRD instance") + + By("waiting for controller reconciliation of CRD instance event") + verifyReconciliationLogs := func(g Gomega) { + output, err := kubectlRunInNamespace( + namespace, + "logs", + "-l", + "control-plane=gitops-reverser", + "--tail=500", + "--prefix=true", + ) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("git commit"), + "Should see git commit operation in logs") + } + Eventually(verifyReconciliationLogs, 45*time.Second, 2*time.Second).Should(Succeed()) + + By("verifying CRD instance YAML file exists in Gitea repository") + verifyGitCommit := func(g Gomega) { + pullLatestRepoState(g, crdLifecycleRepo.CheckoutDir) + + expectedFile := filepath.Join(crdLifecycleRepo.CheckoutDir, + "e2e/icecream-test", + fmt.Sprintf("%s/%s/%s.yaml", iceCreamInstanceDir(crdGroupCRDLifecycle), testNs, crdInstanceName)) + fileInfo, statErr := os.Stat(expectedFile) + g.Expect(statErr). + NotTo(HaveOccurred(), fmt.Sprintf("CRD instance file should exist at %s", expectedFile)) + g.Expect(fileInfo.Size()).To(BeNumerically(">", 0), "CRD instance file should not be empty") + + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred()) + contentStr := string(content) + g.Expect(contentStr).To(ContainSubstring("kind: IceCreamOrder"), + "CRD instance file should contain IceCreamOrder kind") + g.Expect(contentStr).To(ContainSubstring("customerName: Alice"), + "CRD instance file should contain customer name") + g.Expect(contentStr).To(ContainSubstring("container: Cone"), + "CRD instance file should contain container type") + g.Expect(contentStr).To(ContainSubstring("flavor: Vanilla"), + "CRD instance file should contain ice cream flavors") + + // Verify labels are present + g.Expect(contentStr).To(ContainSubstring("environment: test"), + "CRD instance file should contain environment label") + g.Expect(contentStr).To(ContainSubstring("team: engineering"), + "CRD instance file should contain team label") + + // Verify user annotations are present + g.Expect(contentStr).To(ContainSubstring("description: Alice's favorite ice cream"), + "CRD instance file should contain description annotation") + g.Expect(contentStr).To(ContainSubstring("priority: high"), + "CRD instance file should contain priority annotation") + + // Verify filtered annotations are NOT present + g.Expect(contentStr).NotTo(ContainSubstring("kubectl.kubernetes.io/last-applied-configuration"), + "CRD instance file should NOT contain kubectl annotation") + g.Expect(contentStr).NotTo(ContainSubstring("deployment.kubernetes.io/revision"), + "CRD instance file should NOT contain deployment annotation") + + // Verify status field is NOT present in Git + g.Expect(contentStr).NotTo(ContainSubstring("status:"), + "CRD instance file should NOT contain status field") + } + Eventually(verifyGitCommit). + WithTimeout(60 * time.Second). + WithPolling(2 * time.Second). + Should(Succeed()) + + By("verifying the IceCreamOrder still exists before status update") + verifyCRDInstanceExists := func(g Gomega) { + _, err := kubectlRunInNamespace(testNs, "get", iceCreamCRDName(crdGroupCRDLifecycle), crdInstanceName) + g.Expect(err).NotTo(HaveOccurred(), "IceCreamOrder should exist before status patch") + } + Eventually(verifyCRDInstanceExists, 15*time.Second, time.Second).Should(Succeed()) + + By("applying status update to the IceCreamOrder CR") + statusPatch := `{"status":{"phase":"Preparing","cost":12.5,"message":"Queued for pickup"}}` + statusOutput, statusErr := kubectlRunInNamespace( + testNs, + "patch", + iceCreamCRDName(crdGroupCRDLifecycle), + crdInstanceName, + "--type=merge", + "--subresource=status", + "-p", + statusPatch, + ) + Expect(statusErr).NotTo(HaveOccurred(), "Status subresource patch should succeed") + By(fmt.Sprintf("status patched successfully: %s", statusOutput)) + + By("getting current git commit hash") + gitRevCmd := exec.Command("git", "rev-parse", "HEAD") + gitRevCmd.Dir = crdLifecycleRepo.CheckoutDir + beforeStatusCommit, _ := gitRevCmd.Output() + + By("waiting to ensure no new commit is created from status update") + time.Sleep(10 * time.Second) + + By("verifying no new commit was created and status is not in Git") + verifyStatusNotCommitted := func(g Gomega) { + pullLatestRepoState(g, crdLifecycleRepo.CheckoutDir) + + // Check that commit hash hasn't changed + gitRevCmd := exec.Command("git", "rev-parse", "HEAD") + gitRevCmd.Dir = crdLifecycleRepo.CheckoutDir + afterStatusCommit, err := gitRevCmd.Output() + g.Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("Commit before status: %s", string(beforeStatusCommit))) + By(fmt.Sprintf("Commit after status: %s", string(afterStatusCommit))) + + // Read the file again to ensure status is still not present + expectedFile := filepath.Join(crdLifecycleRepo.CheckoutDir, + "e2e/icecream-test", + fmt.Sprintf("%s/%s/%s.yaml", iceCreamInstanceDir(crdGroupCRDLifecycle), testNs, crdInstanceName)) + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred()) + g.Expect(string(content)).NotTo(ContainSubstring("status:"), + "CRD instance file should still NOT contain status field after status update") + g.Expect(string(content)).NotTo(ContainSubstring("phase:"), + "CRD instance file should NOT contain status phase content") + g.Expect(string(content)).NotTo(ContainSubstring("Queued for pickup"), + "CRD instance file should NOT contain status content") + } + Eventually(verifyStatusNotCommitted).Should(Succeed()) + + By("✅ Status update verified - no Git commit created and status not in file") + + By("cleaning up IceCreamOrder instance") + _, _ = kubectlRunInNamespace(testNs, "delete", iceCreamCRDName(crdGroupCRDLifecycle), crdInstanceName) + + By("Note: GitTarget, WatchRule, GitProvider, and CRD kept for subsequent tests") + + By("✅ IceCreamOrder to Git commit E2E test passed") + fmt.Printf("✅ IceCreamOrder '%s' successfully triggered Git commit in repo '%s'\n", + crdInstanceName, uniqueRepoName) + }) + + It("should update Git file when IceCreamOrder is modified via WatchRule", func() { + crdInstanceName := "bobs-order" + uniqueRepoName := crdLifecycleRepo.RepoName + + By("creating initial IceCreamOrder instance") + crdInstanceData := struct { + Name string + Namespace string + Labels map[string]string + Annotations map[string]string + CustomerName string + Container string + Scoops []struct { + Flavor string + Quantity int + } + Toppings []string + Group string + }{ + Name: crdInstanceName, + Namespace: testNs, + Labels: nil, + Annotations: nil, + Group: crdGroupCRDLifecycle, + CustomerName: "Bob", + Container: "Cup", + Scoops: []struct { + Flavor string + Quantity int + }{ + {Flavor: "Strawberry", Quantity: 1}, + }, + Toppings: []string{"WhippedCream"}, + } + + err := applyFromTemplate("test/e2e/templates/icecreamorder-instance.tmpl", crdInstanceData, testNs) + Expect(err).NotTo(HaveOccurred(), "Failed to apply initial CRD instance") + + By("waiting for initial CRD instance file to appear in Git") + verifyInitialFile := func(g Gomega) { + pullLatestRepoState(g, crdLifecycleRepo.CheckoutDir) + + expectedFile := filepath.Join(crdLifecycleRepo.CheckoutDir, + "e2e/icecream-test", + fmt.Sprintf("%s/%s/%s.yaml", iceCreamInstanceDir(crdGroupCRDLifecycle), testNs, crdInstanceName)) + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred()) + g.Expect(string(content)).To(ContainSubstring("customerName: Bob")) + g.Expect(string(content)).To(ContainSubstring("flavor: Strawberry")) + } + Eventually(verifyInitialFile).Should(Succeed()) + + By("updating CRD instance with new values") + updatedCRDData := struct { + Name string + Namespace string + Labels map[string]string + Annotations map[string]string + CustomerName string + Container string + Scoops []struct { + Flavor string + Quantity int + } + Toppings []string + Group string + }{ + Name: crdInstanceName, + Namespace: testNs, + Labels: nil, + Annotations: nil, + Group: crdGroupCRDLifecycle, + CustomerName: "Bob", + Container: "WaffleBowl", + Scoops: []struct { + Flavor string + Quantity int + }{ + {Flavor: "RockyRoad", Quantity: 3}, + {Flavor: "MintChip", Quantity: 2}, + }, + Toppings: []string{"HotFudge", "Caramel", "Sprinkles"}, + } + + err = applyFromTemplate("test/e2e/templates/icecreamorder-instance.tmpl", updatedCRDData, testNs) + Expect(err).NotTo(HaveOccurred(), "Failed to update CRD instance") + + By("verifying updated CRD instance content in Git") + verifyUpdatedFile := func(g Gomega) { + pullLatestRepoState(g, crdLifecycleRepo.CheckoutDir) + + expectedFile := filepath.Join(crdLifecycleRepo.CheckoutDir, + "e2e/icecream-test", + fmt.Sprintf("%s/%s/%s.yaml", iceCreamInstanceDir(crdGroupCRDLifecycle), testNs, crdInstanceName)) + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred()) + g.Expect(string(content)).To(ContainSubstring("container: WaffleBowl"), + "Updated file should contain new container type") + g.Expect(string(content)).To(ContainSubstring("flavor: RockyRoad"), + "Updated file should contain new flavor") + g.Expect(string(content)).To(ContainSubstring("quantity: 3"), + "Updated file should contain new quantity") + } + Eventually(verifyUpdatedFile).Should(Succeed()) + + By("cleaning up IceCreamOrder instance") + _, _ = kubectlRunInNamespace(testNs, "delete", iceCreamCRDName(crdGroupCRDLifecycle), crdInstanceName) + + By("Note: GitTarget, WatchRule, GitProvider, and CRD kept for subsequent tests") + + By("✅ IceCreamOrder update E2E test passed") + fmt.Printf("✅ IceCreamOrder '%s' update successfully reflected in Git repo '%s'\n", + crdInstanceName, uniqueRepoName) + }) + + It("should delete Git file when IceCreamOrder is deleted via WatchRule", func() { + crdInstanceName := "charlies-order" + uniqueRepoName := crdLifecycleRepo.RepoName + + By("creating IceCreamOrder instance") + crdInstanceData := struct { + Name string + Namespace string + Labels map[string]string + Annotations map[string]string + CustomerName string + Container string + Scoops []struct { + Flavor string + Quantity int + } + Toppings []string + Group string + }{ + Name: crdInstanceName, + Namespace: testNs, + Labels: nil, + Annotations: nil, + Group: crdGroupCRDLifecycle, + CustomerName: "Charlie", + Container: "Cone", + Scoops: []struct { + Flavor string + Quantity int + }{ + {Flavor: "Chocolate", Quantity: 2}, + }, + Toppings: []string{"Sprinkles"}, + } + + err := applyFromTemplate("test/e2e/templates/icecreamorder-instance.tmpl", crdInstanceData, testNs) + Expect(err).NotTo(HaveOccurred(), "Failed to apply CR") + + By("waiting for CR file to appear in Git repository") + verifyFileCreated := func(g Gomega) { + pullLatestRepoState(g, crdLifecycleRepo.CheckoutDir) + + expectedFile := filepath.Join(crdLifecycleRepo.CheckoutDir, + "e2e/icecream-test", + fmt.Sprintf("%s/%s/%s.yaml", iceCreamInstanceDir(crdGroupCRDLifecycle), testNs, crdInstanceName)) + fileInfo, statErr := os.Stat(expectedFile) + g.Expect(statErr). + NotTo(HaveOccurred(), fmt.Sprintf("CRD instance file should exist at %s", expectedFile)) + g.Expect(fileInfo.Size()).To(BeNumerically(">", 0), "CRD instance file should not be empty") + } + // 60s tolerates controller-reflector re-establishment if the deployment + // happened to roll during the prior spec (default 30s was racing it). + Eventually(verifyFileCreated, 60*time.Second, 2*time.Second).Should(Succeed()) + + By("deleting the CR to trigger DELETE operation") + _, err = kubectlRunInNamespace(testNs, "delete", iceCreamCRDName(crdGroupCRDLifecycle), crdInstanceName) + Expect(err).NotTo(HaveOccurred(), "CRD instance deletion should succeed") + + By("verifying CRD instance file is deleted from Git repository") + verifyFileDeleted := func(g Gomega) { + pullLatestRepoState(g, crdLifecycleRepo.CheckoutDir) + + expectedFile := filepath.Join(crdLifecycleRepo.CheckoutDir, + "e2e/icecream-test", + fmt.Sprintf("%s/%s/%s.yaml", iceCreamInstanceDir(crdGroupCRDLifecycle), testNs, crdInstanceName)) + _, statErr := os.Stat(expectedFile) + g.Expect(statErr). + To(HaveOccurred(), fmt.Sprintf("CRD instance file should NOT exist at %s", expectedFile)) + g.Expect(os.IsNotExist(statErr)).To(BeTrue(), "Error should be 'file does not exist'") + + By("verifying git log shows DELETE commit") + gitLogCmd := exec.Command("git", "log", "--oneline", "-n", "5") + gitLogCmd.Dir = crdLifecycleRepo.CheckoutDir + logOutput, logErr := gitLogCmd.CombinedOutput() + g.Expect(logErr).NotTo(HaveOccurred(), "Should be able to read git log") + g.Expect(string(logOutput)).To(ContainSubstring("DELETE"), + "Git log should contain DELETE operation") + } + Eventually(verifyFileDeleted).Should(Succeed()) + + By("✅ IceCreamOrder deletion E2E test passed") + fmt.Printf("✅ IceCreamOrder '%s' deletion successfully removed file from Git repo '%s'\n", + crdInstanceName, uniqueRepoName) + }) + + It("should delete Git file when IceCreamOrder CRD is deleted via ClusterWatchRule", func() { + gitProviderName := "gitprovider-normal" + clusterWatchRuleName := "clusterwatchrule-crd-delete" + crdName := iceCreamCRDName(crdGroupCRDLifecycle) + + By("creating ClusterWatchRule with Cluster scope for CRDs") + destName := clusterWatchRuleName + "-dest" + createGitTarget(destName, testNs, gitProviderName, "e2e/crd-delete-test", "main") + verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") + + clusterWatchRuleData := struct { + Name string + DestinationName string + Namespace string + }{ + Name: clusterWatchRuleName, + DestinationName: destName, + Namespace: testNs, + } + + err := applyFromTemplate("test/e2e/templates/manager/clusterwatchrule-crd.tmpl", clusterWatchRuleData, "") + Expect(err).NotTo(HaveOccurred(), "Failed to apply ClusterWatchRule for CRDs") + + By("verifying ClusterWatchRule is ready") + + verifyResourceStatus("clusterwatchrule", clusterWatchRuleName, "", "True", "Ready", "") + + By("verifying CRD file exists in Git before deletion") + verifyFileExists := func(g Gomega) { + pullLatestRepoState(g, crdLifecycleRepo.CheckoutDir) + + expectedFile := filepath.Join(crdLifecycleRepo.CheckoutDir, + "e2e/crd-delete-test", + "apiextensions.k8s.io/v1/customresourcedefinitions/"+iceCreamCRDMirrorFile(crdGroupCRDLifecycle)) + _, statErr := os.Stat(expectedFile) + g.Expect(statErr).NotTo(HaveOccurred(), "CRD file should exist before deletion") + } + Eventually(verifyFileExists).Should(Succeed()) + + By("deleting the CRD to trigger DELETE operation") + _, deleteErr := kubectlRun("delete", "crd", crdName) + Expect(deleteErr).NotTo(HaveOccurred(), "CRD deletion should succeed") + By("verifying CRD file is deleted from Git repository") + verifyFileDeleted := func(g Gomega) { + pullLatestRepoState(g, crdLifecycleRepo.CheckoutDir) + + expectedFile := filepath.Join(crdLifecycleRepo.CheckoutDir, + "e2e/crd-delete-test", + "apiextensions.k8s.io/v1/customresourcedefinitions/"+iceCreamCRDMirrorFile(crdGroupCRDLifecycle)) + _, statErr := os.Stat(expectedFile) + g.Expect(statErr).To(HaveOccurred(), "CRD file should NOT exist after deletion") + g.Expect(os.IsNotExist(statErr)).To(BeTrue(), "Error should be 'file does not exist'") + + // Verify git log shows DELETE commit + gitLogCmd := exec.Command("git", "log", "--oneline", "-n", "5") + gitLogCmd.Dir = crdLifecycleRepo.CheckoutDir + logOutput, logErr := gitLogCmd.CombinedOutput() + g.Expect(logErr).NotTo(HaveOccurred(), "Should be able to read git log") + g.Expect(string(logOutput)).To(ContainSubstring("DELETE"), + "Git log should contain DELETE operation") + } + Eventually(verifyFileDeleted, "60s", "1s").Should(Succeed()) + + By("verifying the deleted CRD file does not reappear after terminating updates") + Consistently(func(g Gomega) { + pullLatestRepoState(g, crdLifecycleRepo.CheckoutDir) + + expectedFile := filepath.Join(crdLifecycleRepo.CheckoutDir, + "e2e/crd-delete-test", + "apiextensions.k8s.io/v1/customresourcedefinitions/"+iceCreamCRDMirrorFile(crdGroupCRDLifecycle)) + _, statErr := os.Stat(expectedFile) + g.Expect(statErr).To(HaveOccurred(), "CRD file must stay deleted after CRD termination updates") + g.Expect(os.IsNotExist(statErr)).To(BeTrue(), "CRD file must not reappear in Git") + }, "15s", "1s").Should(Succeed()) + + By("cleaning up test resources") + cleanupClusterWatchRule(clusterWatchRuleName) + cleanupGitTarget(destName, testNs) + + By("✅ CRD deletion via ClusterWatchRule E2E test passed") + }) +}) diff --git a/test/e2e/demo-todo.md b/test/e2e/demo-todo.md deleted file mode 100644 index 1df51b18f..000000000 --- a/test/e2e/demo-todo.md +++ /dev/null @@ -1,5 +0,0 @@ - - -* Get a 100 votes at the same time! -* move to a voter that is released by GH: https://github.com/sunib/voter -* https within cluster \ No newline at end of file diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index bca687baf..dc64f1ea2 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -47,7 +47,11 @@ func TestE2E(t *testing.T) { RunSpecs(t, "e2e suite") } -var _ = BeforeSuite(func() { +// SynchronizedBeforeSuite splits bootstrap into work that must happen exactly +// once (the Task prepare flow and the cluster-scoped CRD pre-cleanup, run on +// parallel process #1) and per-process configuration (the E2E_AGE_KEY_FILE +// fallback, which relies on os.Setenv and therefore must run in every process). +var _ = SynchronizedBeforeSuite(func() []byte { defer func() { if recovered := recover(); recovered != nil { markSuiteWidePreservation("BeforeSuite failed") @@ -62,14 +66,20 @@ var _ = BeforeSuite(func() { } By("preparing e2e cluster prerequisites via Task target") - ensureE2EPrepared() + prepareE2EClusterOnce() + return nil +}, func(_ []byte) { + configureE2EProcess() }) var _ = AfterEach(func() { dumpFailureDiagnostics() }) -func ensureE2EPrepared() { +// prepareE2EClusterOnce runs the expensive, cluster-mutating bootstrap exactly +// once (parallel process #1): the Task prepare flow plus the cluster-scoped CRD +// pre-cleanup. It is safe to run concurrently with nothing else. +func prepareE2EClusterOnce() { ctx := resolveE2EContext() setE2EKubectlContext(ctx) ns := resolveE2ENamespace() @@ -93,20 +103,57 @@ func ensureE2EPrepared() { Expect(err).NotTo(HaveOccurred(), "failed to set kubectl context for e2e run") _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) - By("ensuring IceCreamOrder CRD is removed before tests") - output, err = kubectlRun( - "delete", "crd", "icecreamorders.shop.example.com", - "--ignore-not-found=true", - ) - Expect(err).NotTo(HaveOccurred(), "failed to delete IceCreamOrder CRD before tests") - _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) + // Each CRD-installing e2e file now owns its own IceCreamOrder CRD group + // (see test/e2e/icecream.go). Remove all of them, plus the legacy shared + // group, so a warm/reused cluster starts clean. + By("ensuring IceCreamOrder CRDs are removed before tests") + for _, group := range []string{ + crdGroupCRDLifecycle, + crdGroupRestartSnapshot, + crdGroupBiDirectional, + "shop.example.com", // legacy pre-isolation group + } { + output, err = kubectlRun( + "delete", "crd", iceCreamCRDName(group), + "--ignore-not-found=true", + ) + Expect(err).NotTo(HaveOccurred(), + "failed to delete IceCreamOrder CRD %q before tests", iceCreamCRDName(group)) + _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) + } + + // The Gitea org is a singleton every repo fixture lives under. Create it + // once here — this runs on parallel process #1 only and blocks the other + // processes until it returns — so no two per-spec BeforeAlls ever race a + // concurrent POST /orgs. An in-process lock cannot solve this: Ginkgo + // --procs runs specs in separate OS processes, so the race is cross-process. + By("ensuring the shared Gitea organization exists before parallel specs run") + ensureSharedGiteaOrgOnce() +} + +// ensureSharedGiteaOrgOnce creates the shared test org exactly once, from the +// SynchronizedBeforeSuite process-#1 function. Per-spec bootstrap then only +// creates its own uniquely-named repo under an org that already exists. +func ensureSharedGiteaOrgOnce() { + gitea := giteaTestInstance() + Expect(waitForGiteaAPI(gitea.Client())).To(Succeed(), + "Gitea API must be reachable before creating the shared org") + + ctx, cancel := gitea.Context() + defer cancel() + _, err := gitea.Client().EnsureOrg(ctx, gitea.Org, "Test Organization", "E2E Test Organization") + Expect(err).NotTo(HaveOccurred(), "failed to ensure shared Gitea org %q", gitea.Org) +} - // The Task prepare flow writes the age key under the stamp directory. When running `go test` directly - // (without `task test-e2e`), ensure the suite uses that prepared key file. +// configureE2EProcess runs in every parallel process. The Task prepare flow +// writes the age key under the stamp directory; when running `go test` directly +// (without `task test-e2e`), point the suite at that prepared key file. This +// relies on os.Setenv, so it must run per-process rather than once. +func configureE2EProcess() { if strings.TrimSpace(os.Getenv("E2E_AGE_KEY_FILE")) == "" { wd, err := os.Getwd() Expect(err).NotTo(HaveOccurred(), "failed to get working directory for e2e run") - ageKeyPath := filepath.Join(wd, ".stamps", "cluster", ctx, "age-key.txt") + ageKeyPath := filepath.Join(wd, ".stamps", "cluster", resolveE2EContext(), "age-key.txt") Expect(os.Setenv("E2E_AGE_KEY_FILE", ageKeyPath)).To(Succeed()) } } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 248efcd79..141f03142 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -21,16 +21,11 @@ package e2e import ( "bytes" "context" - "encoding/base64" "encoding/json" "fmt" "os" - "os/exec" - "path" - "path/filepath" "sort" "strings" - "time" "filippo.io/age" . "github.com/onsi/ginkgo/v2" @@ -42,1698 +37,6 @@ import ( const defaultE2EAgeKeyPath = "/tmp/e2e-age-key.txt" -// managerRepo holds the file-local repo fixtures for the Manager describe block. -var managerRepo *RepoArtifacts - -var _ = Describe("Manager", Label("manager"), Ordered, func() { - var controllerPodName string // Name of first controller pod for logging - var testNs string - - // Before running the tests, set up per-run test fixtures. - BeforeAll(func() { - By("setting up Prometheus client for metrics testing") - setupPrometheusClient() - verifyPrometheusAvailable() - - By("creating test namespace") - testNs = testNamespaceFor("manager") - _, _ = kubectlRun("create", "namespace", testNs) // idempotent; ignore AlreadyExists - - By("setting up Gitea repo and credentials for manager tests") - managerRepo = SetupRepo( - resolveE2EContext(), - testNs, - fmt.Sprintf("e2e-manager-%d", GinkgoRandomSeed()), - ) - - By("applying git secrets to test namespace") - _, err := kubectlRunInNamespace(testNs, "apply", "-f", managerRepo.SecretsYAML) - Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") - applySOPSAgeKeyToNamespace(testNs) - }) - - // After all tests have been executed, clean up the test namespace - AfterAll(func() { - cleanupNamespace(testNs) - - By("test infrastructure still running for debugging") - fmt.Printf("\n") - fmt.Printf("═══════════════════════════════════════════════════════════\n") - fmt.Printf("📊 E2E Infrastructure kept running for debugging purposes:\n") - fmt.Printf("═══════════════════════════════════════════════════════════\n") - fmt.Printf(" Prometheus: http://localhost:19090\n") - fmt.Printf(" Gitea: http://localhost:13000\n") - fmt.Printf(" Flux UI: http://localhost:19080\n") - fmt.Printf("\n") - fmt.Printf("═══════════════════════════════════════════════════════════\n") - fmt.Printf("\n") - }) - - // Optimize timeouts for faster test execution - SetDefaultEventuallyTimeout(30 * time.Second) - SetDefaultEventuallyPollingInterval(time.Second) - - Context("Manager", func() { - - It("should run successfully", Label("smoke"), func() { - By("validating that the gitops-reverser pods are running as expected") - verifyControllerUp := func(g Gomega) { - // Get the names of the gitops-reverser pods - podOutput, err := kubectlRunInNamespace( - namespace, - "get", - "pods", - "-l", - "control-plane=gitops-reverser", - "-o", "go-template={{ range .items }}"+ - "{{ if not .metadata.deletionTimestamp }}"+ - "{{ .metadata.name }}"+ - "{{ \"\\n\" }}{{ end }}{{ end }}", - ) - g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve gitops-reverser pod information") - podNames := utils.GetNonEmptyLines(podOutput) - g.Expect(podNames).To(HaveLen(1), "expected exactly 1 controller pod running") - controllerPodName = podNames[0] // Use first pod for logging - g.Expect(controllerPodName).To(ContainSubstring("gitops-reverser")) - - // Validate all pods' status - for _, podName := range podNames { - output, err := kubectlRunInNamespace( - namespace, - "get", - "pods", - podName, - "-o", - "jsonpath={.status.phase}", - ) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(Equal("Running"), fmt.Sprintf("Incorrect status for pod %s", podName)) - } - } - Eventually(verifyControllerUp).Should(Succeed()) - }) - - It("should expose the controller service", Label("smoke"), func() { - By("verifying controller service exists") - _, err := kubectlRunInNamespace(namespace, "get", "svc", controllerServiceName) - Expect(err).NotTo(HaveOccurred(), "Controller service should exist") - - By("verifying controller service routes to the controller pod") - Eventually(func(g Gomega) { - output, endpointsErr := kubectlRunInNamespace( - namespace, - "get", - "endpoints", - controllerServiceName, - "-o", - "jsonpath={.subsets[*].addresses[*].targetRef.name}", - ) - g.Expect(endpointsErr).NotTo(HaveOccurred(), "Failed to get controller service endpoints") - - lines := utils.GetNonEmptyLines(output) - podSet := map[string]struct{}{} - for _, line := range lines { - if !strings.HasPrefix(line, "Warning:") && - !strings.Contains(line, "deprecated") && - strings.Contains(line, "gitops-reverser") { - podSet[line] = struct{}{} - } - } - var podNames []string - for podName := range podSet { - podNames = append(podNames, podName) - } - - g.Expect(podNames).To(HaveLen(1), "controller service should route to exactly 1 pod") - g.Expect(podNames[0]).To(Equal(controllerPodName), "controller service should route to controller pod") - }, 30*time.Second).Should(Succeed()) - }) - - It("should expose the audit service separately", func() { - By("verifying audit service exists") - _, err := kubectlRunInNamespace(namespace, "get", "svc", auditServiceName) - Expect(err).NotTo(HaveOccurred(), "Audit service should exist") - - By("verifying audit service routes to the controller pod") - Eventually(func(g Gomega) { - output, endpointsErr := kubectlRunInNamespace( - namespace, - "get", - "endpoints", - auditServiceName, - "-o", - "jsonpath={.subsets[*].addresses[*].targetRef.name}", - ) - g.Expect(endpointsErr).NotTo(HaveOccurred(), "Failed to get audit service endpoints") - - lines := utils.GetNonEmptyLines(output) - var podNames []string - for _, line := range lines { - if strings.HasPrefix(line, "Warning:") || strings.Contains(line, "deprecated") { - continue - } - podNames = append(podNames, line) - } - - g.Expect(podNames).To(HaveLen(1), "audit service should route to exactly 1 pod") - g.Expect(podNames[0]).To(Equal(controllerPodName), "audit service should route to controller pod") - }, 30*time.Second).Should(Succeed()) - }) - - It("should ensure the metrics endpoint is serving metrics", Label("smoke"), func() { - By("validating that the controller service is available for metrics") - _, err := kubectlRunInNamespace(namespace, "get", "service", controllerServiceName) - Expect(err).NotTo(HaveOccurred(), "Controller service should exist") - - By("waiting for the metrics endpoint to be ready") - verifyMetricsEndpointReady := func(g Gomega) { - output, err := kubectlRunInNamespace(namespace, "get", "endpoints", controllerServiceName) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") - } - Eventually(verifyMetricsEndpointReady).Should(Succeed()) - - By("verifying that the controller manager is serving the metrics server") - verifyMetricsServerStarted := func(g Gomega) { - output, err := kubectlRunInNamespace(namespace, "logs", controllerPodName) - g.Expect(err).NotTo(HaveOccurred()) - jsonMetricsLogLine := "\"logger\":\"controller-runtime.metrics\"," + - "\"msg\":\"Serving metrics server\"" - g.Expect(output).To( - SatisfyAny( - ContainSubstring("controller-runtime.metrics\tServing metrics server"), - ContainSubstring(jsonMetricsLogLine), - ), - "Metrics server not yet started", - ) - } - Eventually(verifyMetricsServerStarted).Should(Succeed()) - - By("waiting for Prometheus to scrape controller metrics") - waitForMetric("sum(up{job='gitops-reverser'})", - func(v float64) bool { return v == 1 }, - "metrics endpoint should be up") - - By("verifying basic process metrics are exposed") - waitForMetric("sum(process_cpu_seconds_total{job='gitops-reverser'})", - func(v float64) bool { return v > 0 }, - "process metrics should exist") - - By("verifying metrics from the controller pod") - podCount, err := queryPrometheus("sum(up{job='gitops-reverser'})") - Expect(err).NotTo(HaveOccurred()) - Expect(podCount).To(BeEquivalentTo(1), "Should scrape from 1 controller pod") - - fmt.Printf("✅ Metrics collection verified from %.0f pods\n", podCount) - fmt.Printf("📊 Inspect metrics: %s\n", getPrometheusURL()) - }) - - It("should receive audit webhook events from kube-apiserver", Label("smoke"), func() { - By("recording baseline audit event count") - baselineAuditEvents, err := queryPrometheus("sum(gitopsreverser_audit_events_received_total) or vector(0)") - Expect(err).NotTo(HaveOccurred()) - fmt.Printf("📊 Baseline audit events: %.0f\n", baselineAuditEvents) - baselineOfficialAuditEvents, err := queryPrometheus( - "sum(gitopsreverser_audit_events_received_total{source='official'}) or vector(0)", - ) - Expect(err).NotTo(HaveOccurred()) - fmt.Printf("📊 Baseline official audit events: %.0f\n", baselineOfficialAuditEvents) - - By("creating a ConfigMap to trigger audit events") - _, err = kubectlRunInNamespace( - namespace, - "create", - "configmap", - "audit-test-cm", - "--from-literal=test=audit", - ) - Expect(err).NotTo(HaveOccurred(), "ConfigMap creation should succeed") - _, err = kubectlRunInNamespace( - namespace, - "patch", - "configmap", - "audit-test-cm", - "--type=merge", - "--patch", - `{"data":{"test":"audit-updated"}}`, - ) - Expect(err).NotTo(HaveOccurred(), "ConfigMap update should succeed") - - By("waiting for audit event metric to increment") - waitForMetricWithTimeout("sum(gitopsreverser_audit_events_received_total) or vector(0)", - func(v float64) bool { return v > baselineAuditEvents }, - "audit events should increment", 2*time.Minute) - waitForMetricWithTimeout( - "sum(gitopsreverser_audit_events_received_total{source='official'}) or vector(0)", - func(v float64) bool { return v > baselineOfficialAuditEvents }, - "audit events should increment for source=official", 2*time.Minute, - ) - - By("verifying audit events were received") - currentAuditEvents, err := queryPrometheus("sum(gitopsreverser_audit_events_received_total) or vector(0)") - Expect(err).NotTo(HaveOccurred()) - Expect(currentAuditEvents).To(BeNumerically(">", baselineAuditEvents), - "Should have received audit events from kube-apiserver") - - By("verifying the EventList ingress metric appears for source=official") - waitForMetricWithTimeout( - "sum(gitopsreverser_audit_eventlists_total{source='official'}) or vector(0)", - func(v float64) bool { return v > 0 }, - "EventList ingress requests should be counted for source=official", 2*time.Minute, - ) - - By("verifying no removed cluster/gvr/action label remains on any audit series") - for _, removed := range []string{"cluster", "gvr", "action"} { - stale, queryErr := queryPrometheus(fmt.Sprintf( - "sum({__name__=~'gitopsreverser_audit_.+', %s=~'.+'}) or vector(0)", removed)) - Expect(queryErr).NotTo(HaveOccurred()) - Expect(stale).To(BeNumerically("==", 0), - fmt.Sprintf("no audit series should still carry the %q label", removed)) - } - - newEvents := currentAuditEvents - baselineAuditEvents - fmt.Printf("✅ Received %.0f new audit events from kube-apiserver\n", newEvents) - fmt.Printf("📊 Total audit events: %.0f\n", currentAuditEvents) - - By("cleaning up audit test resources") - _, _ = kubectlRunInNamespace( - namespace, - "delete", - "configmap", - "audit-test-cm", - "--ignore-not-found=true", - ) - }) - - It("should validate GitProvider with real Gitea repository", Label("smoke"), func() { - gitProviderName := "gitprovider-e2e-test" - - By("showing initial controller logs") - showControllerLogs("before creating GitProvider") - - createGitProviderWithURLInNamespace( - gitProviderName, - testNs, - managerRepo.GitSecretHTTP, - managerRepo.RepoURLHTTP, - ) - - By("showing controller logs after GitProvider creation") - showControllerLogs("after creating GitProvider") - - verifyResourceStatus( - "gitprovider", gitProviderName, testNs, - "True", "Ready", "Repository connectivity validated", - ) - - By("showing final controller logs") - showControllerLogs("after status verification") - - _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", gitProviderName, "--ignore-not-found=true") - }) - - It("should handle GitProvider with invalid credentials", func() { - gitProviderName := "gitprovider-invalid-test" - createGitProviderWithURLInNamespace( - gitProviderName, - testNs, - managerRepo.GitSecretInvalid, - managerRepo.RepoURLHTTP, - ) - verifyResourceStatus("gitprovider", gitProviderName, testNs, "False", "ConnectionFailed", "") - _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", gitProviderName, "--ignore-not-found=true") - }) - - It("should handle GitTarget with nonexistent branch pattern", func() { - gitProviderName := "gitprovider-branch-test" - - // GitProvider should be Ready=True (validates connectivity, not branch existence) - createGitProviderWithURLInNamespace( - gitProviderName, - testNs, - managerRepo.GitSecretHTTP, - managerRepo.RepoURLHTTP, - ) - verifyResourceStatus( - "gitprovider", gitProviderName, testNs, "True", "Ready", "Repository connectivity validated", - ) - - // GitTarget with branch not matching any pattern should fail - destName := "dest-invalid-branch" - createGitTarget(destName, testNs, gitProviderName, "test/invalid", "different-branch") - - By("verifying GitTarget fails branch validation") - verifyResourceStatus("gittarget", destName, testNs, "False", "ValidationFailed", "BranchNotAllowed") - - cleanupGitTarget(destName, testNs) - _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", gitProviderName, "--ignore-not-found=true") - }) - - It("should validate GitProvider with SSH authentication", func() { - gitProviderName := "gitprovider-ssh-test" - - By("🔐 Starting SSH authentication test") - showControllerLogs("before SSH test") - - By("📋 Checking SSH secret exists") - secretOutput, err := kubectlRunInNamespace(testNs, "get", "secret", managerRepo.GitSecretSSH, "-o", "yaml") - if err != nil { - fmt.Printf("❌ SSH secret not found: %v\n", err) - } else { - previewLen := minInt(300, len(secretOutput)) - fmt.Printf( - "✅ SSH secret exists - showing first %d chars:\n%s...\n", - previewLen, - secretOutput[:previewLen], - ) - } - - createGitProviderWithURLInNamespace( - gitProviderName, - testNs, - managerRepo.GitSecretSSH, - managerRepo.RepoURLSSH, - ) - - By("🔍 Controller logs after SSH GitProvider creation") - showControllerLogs("after SSH GitProvider creation") - - verifyResourceStatus( - "gitprovider", gitProviderName, testNs, "True", "Ready", "Repository connectivity validated", - ) - - By("✅ Final SSH test logs") - showControllerLogs("SSH test completion") - - _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", gitProviderName, "--ignore-not-found=true") - }) - - It("should handle a normal and healthy GitProvider", Label("smoke"), func() { - gitProviderName := "gitprovider-normal" - createGitProviderWithURLInNamespace( - gitProviderName, - testNs, - managerRepo.GitSecretHTTP, - managerRepo.RepoURLHTTP, - ) - verifyResourceStatus( - "gitprovider", gitProviderName, testNs, "True", "Ready", "Repository connectivity validated", - ) - }) - - It("should reconcile a WatchRule CR", func() { - gitProviderName := "gitprovider-normal" - watchRuleName := "watchrule-test" - - By("creating a WatchRule that references the working GitProvider") - destName := watchRuleName + "-dest" - createGitTarget(destName, testNs, gitProviderName, getBaseFolder(), "main") - - data := struct { - Name string - Namespace string - DestinationName string - }{ - Name: watchRuleName, - Namespace: testNs, - DestinationName: destName, - } - - err := applyFromTemplate("test/e2e/templates/watchrule.tmpl", data, testNs) - Expect(err).NotTo(HaveOccurred(), "Failed to apply WatchRule") - - By("verifying the WatchRule is reconciled") - verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") - - By("cleaning up test resources") - cleanupWatchRule(watchRuleName, testNs) - cleanupGitTarget(destName, testNs) - }) - - It("should commit encrypted Secret manifests when WatchRule includes secrets", Label("smoke"), func() { - gitProviderName := "gitprovider-normal" - watchRuleName := "watchrule-secret-encryption-test" - secretName := "test-secret-encryption" - - By("creating WatchRule that includes secrets") - destName := watchRuleName + "-dest" - createGitTarget(destName, testNs, gitProviderName, "e2e/secret-encryption-test", "main") - - data := struct { - Name string - Namespace string - DestinationName string - }{ - Name: watchRuleName, - Namespace: testNs, - DestinationName: destName, - } - - err := applyFromTemplate("test/e2e/templates/watchrule-secret.tmpl", data, testNs) - Expect(err).NotTo(HaveOccurred(), "Failed to apply WatchRule") - verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") - verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") - verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") - - By("creating Secret in watched namespace") - _, _ = kubectlRunInNamespace(testNs, "delete", "secret", secretName, "--ignore-not-found=true") - - _, err = kubectlRunInNamespace( - testNs, - "create", - "secret", - "generic", - secretName, - "--from-literal=password=do-not-commit", - ) - Expect(err).NotTo(HaveOccurred(), "Secret creation should succeed") - - By("patching Secret once to avoid informer start race and force an update event") - _, err = kubectlRunInNamespace( - testNs, - "patch", - "secret", - secretName, - "--type=merge", - "--patch", - `{"stringData":{"password":"never-commit-this"}}`, - ) - Expect(err).NotTo(HaveOccurred(), "Secret patch should succeed") - - By("verifying Secret file is committed and does not contain plaintext data") - verifyEncryptedSecretCommitted := func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - pullOutput, pullErr := pullCmd.CombinedOutput() - if pullErr != nil { - g.Expect(pullErr).NotTo(HaveOccurred(), - fmt.Sprintf("Should successfully pull latest changes. Output: %s", string(pullOutput))) - } - - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/secret-encryption-test", - fmt.Sprintf("v1/secrets/%s/%s.sops.yaml", testNs, secretName)) - content, readErr := os.ReadFile(expectedFile) - g.Expect(readErr).NotTo(HaveOccurred(), fmt.Sprintf("Secret file must exist at %s", expectedFile)) - g.Expect(string(content)).To(ContainSubstring("sops:")) - g.Expect(string(content)).NotTo(ContainSubstring("do-not-commit")) - - bootstrapSOPSFile := filepath.Join(managerRepo.CheckoutDir, "e2e/secret-encryption-test", ".sops.yaml") - bootstrapContent, bootstrapErr := os.ReadFile(bootstrapSOPSFile) - g.Expect(bootstrapErr).NotTo( - HaveOccurred(), - fmt.Sprintf(".sops.yaml must exist at %s", bootstrapSOPSFile), - ) - ageKey, ageKeyErr := readSOPSAgeKeyFromFile(getE2EAgeKeyPath()) - g.Expect(ageKeyErr).NotTo(HaveOccurred(), "Should read age private key") - recipient, recipientErr := deriveAgeRecipient(ageKey) - g.Expect(recipientErr).NotTo(HaveOccurred(), "Should derive age recipient") - g.Expect(string(bootstrapContent)).To(ContainSubstring(recipient)) - - decryptedOutput, decryptErr := decryptWithControllerSOPS(content, ageKey) - g.Expect(decryptErr).NotTo(HaveOccurred(), "Should decrypt committed secret via controller sops binary") - g.Expect(decryptedOutput).To(ContainSubstring("bmV2ZXItY29tbWl0LXRoaXM=")) - } - Eventually(verifyEncryptedSecretCommitted, "30s", "2s").Should(Succeed()) - - By("cleaning up test resources") - _, _ = kubectlRunInNamespace(testNs, "delete", "secret", secretName, "--ignore-not-found=true") - cleanupWatchRule(watchRuleName, testNs) - cleanupGitTarget(destName, testNs) - }) - - It("should generate missing SOPS age secret when age.recipients.generateWhenMissing is enabled", func() { - gitProviderName := "gitprovider-normal" - watchRuleName := "watchrule-secret-autogen-test" - secretName := "test-secret-autogen" - generatedSecretName := "sops-age-key-autogen" - - By("ensuring generated encryption secret does not exist before test") - _, _ = kubectlRunInNamespace( - testNs, - "delete", - "secret", - generatedSecretName, - "--ignore-not-found=true", - ) - - By("creating GitTarget with age recipient auto-generation enabled") - destName := watchRuleName + "-dest" - createGitTargetWithEncryptionOptions( - destName, - testNs, - gitProviderName, - "e2e/secret-autogen-test", - "main", - generatedSecretName, - true, - ) - - data := struct { - Name string - Namespace string - DestinationName string - }{ - Name: watchRuleName, - Namespace: testNs, - DestinationName: destName, - } - - err := applyFromTemplate("test/e2e/templates/watchrule-secret.tmpl", data, testNs) - Expect(err).NotTo(HaveOccurred(), "Failed to apply WatchRule") - verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") - - By("validating generated encryption secret has recipient and warning annotations") - var generatedAgeKey string - var generatedRecipient string - Eventually(func(g Gomega) { - output, getErr := kubectlRunInNamespace( - testNs, - "get", - "secret", - generatedSecretName, - "-o", - "json", - ) - g.Expect(getErr).NotTo(HaveOccurred()) - - var secretObj map[string]interface{} - unmarshalErr := json.Unmarshal([]byte(output), &secretObj) - g.Expect(unmarshalErr).NotTo(HaveOccurred()) - - annotations, _, annoErr := unstructured.NestedStringMap(secretObj, "metadata", "annotations") - g.Expect(annoErr).NotTo(HaveOccurred()) - g.Expect(annotations).To(HaveKey("configbutler.ai/age-recipient")) - g.Expect(annotations["configbutler.ai/age-recipient"]).To(HavePrefix("age1")) - g.Expect(annotations).To(HaveKeyWithValue("configbutler.ai/backup-warning", "REMOVE_AFTER_BACKUP")) - generatedRecipient = annotations["configbutler.ai/age-recipient"] - - secretData, found, keyErr := unstructured.NestedStringMap(secretObj, "data") - g.Expect(keyErr).NotTo(HaveOccurred()) - g.Expect(found).To(BeTrue()) - var sopsAgeKeyB64 string - for key, value := range secretData { - if strings.HasSuffix(key, ".agekey") { - sopsAgeKeyB64 = value - break - } - } - g.Expect(sopsAgeKeyB64).NotTo(BeEmpty()) - - keyBytes, decodeErr := base64.StdEncoding.DecodeString(sopsAgeKeyB64) - g.Expect(decodeErr).NotTo(HaveOccurred()) - generatedAgeKey = strings.TrimSpace(string(keyBytes)) - g.Expect(generatedAgeKey).To(HavePrefix("AGE-SECRET-KEY-")) - }, "30s", "2s").Should(Succeed()) - - By("waiting for auto-generated target bootstrap file to be present") - Eventually(func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - pullOutput, pullErr := pullCmd.CombinedOutput() - if pullErr != nil { - g.Expect(pullErr).NotTo(HaveOccurred(), - fmt.Sprintf("Should successfully pull latest changes. Output: %s", string(pullOutput))) - } - - bootstrapSOPSFile := filepath.Join(managerRepo.CheckoutDir, "e2e/secret-autogen-test", ".sops.yaml") - bootstrapContent, bootstrapErr := os.ReadFile(bootstrapSOPSFile) - g.Expect(bootstrapErr).NotTo(HaveOccurred(), - fmt.Sprintf(".sops.yaml must exist at %s", bootstrapSOPSFile)) - g.Expect(string(bootstrapContent)).To(ContainSubstring(generatedRecipient)) - }, "30s", "2s").Should(Succeed()) - - By("creating Secret in watched namespace") - _, _ = kubectlRunInNamespace(testNs, "delete", "secret", secretName, "--ignore-not-found=true") - _, err = kubectlRunInNamespace( - testNs, - "create", - "secret", - "generic", - secretName, - "--from-literal=password=do-not-commit", - ) - Expect(err).NotTo(HaveOccurred(), "Secret creation should succeed") - - By("patching Secret once to avoid informer start race and force an update event") - _, err = kubectlRunInNamespace( - testNs, - "patch", - "secret", - secretName, - "--type=merge", - "--patch", - `{"stringData":{"password":"autogen-never-commit-this"}}`, - ) - Expect(err).NotTo(HaveOccurred(), "Secret patch should succeed") - - By("verifying committed secret is encrypted and decryptable with generated key") - verifyEncryptedSecretCommitted := func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - pullOutput, pullErr := pullCmd.CombinedOutput() - if pullErr != nil { - g.Expect(pullErr).NotTo(HaveOccurred(), - fmt.Sprintf("Should successfully pull latest changes. Output: %s", string(pullOutput))) - } - - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/secret-autogen-test", - fmt.Sprintf("v1/secrets/%s/%s.sops.yaml", testNs, secretName)) - content, readErr := os.ReadFile(expectedFile) - g.Expect(readErr).NotTo(HaveOccurred(), fmt.Sprintf("Secret file must exist at %s", expectedFile)) - g.Expect(string(content)).To(ContainSubstring("sops:")) - g.Expect(string(content)).NotTo(ContainSubstring("autogen-never-commit-this")) - - bootstrapSOPSFile := filepath.Join(managerRepo.CheckoutDir, "e2e/secret-autogen-test", ".sops.yaml") - bootstrapContent, bootstrapErr := os.ReadFile(bootstrapSOPSFile) - g.Expect(bootstrapErr).NotTo( - HaveOccurred(), - fmt.Sprintf(".sops.yaml must exist at %s", bootstrapSOPSFile), - ) - g.Expect(string(bootstrapContent)).To(ContainSubstring(generatedRecipient)) - - decryptedOutput, decryptErr := decryptWithControllerSOPS(content, generatedAgeKey) - g.Expect(decryptErr).NotTo(HaveOccurred(), "Should decrypt committed secret via generated age key") - g.Expect(decryptedOutput).To(ContainSubstring("YXV0b2dlbi1uZXZlci1jb21taXQtdGhpcw==")) - } - Eventually(verifyEncryptedSecretCommitted, "30s", "2s").Should(Succeed()) - - By("cleaning up test resources") - _, _ = kubectlRunInNamespace(testNs, "delete", "secret", secretName, "--ignore-not-found=true") - _, _ = kubectlRunInNamespace( - testNs, - "delete", - "secret", - generatedSecretName, - "--ignore-not-found=true", - ) - cleanupWatchRule(watchRuleName, testNs) - cleanupGitTarget(destName, testNs) - }) - - It("should create Git commit when ConfigMap is added via WatchRule", Label("smoke"), func() { - gitProviderName := "gitprovider-normal" - watchRuleName := "watchrule-configmap-test" - configMapName := "test-configmap" - uniqueRepoName := managerRepo.RepoName - - By("creating WatchRule that monitors ConfigMaps") - destName := watchRuleName + "-dest" - createGitTarget(destName, testNs, gitProviderName, "e2e/configmap-test", "main") - - data := struct { - Name string - Namespace string - DestinationName string - }{ - Name: watchRuleName, - Namespace: testNs, - DestinationName: destName, - } - - err2 := applyFromTemplate("test/e2e/templates/manager/watchrule-configmap.tmpl", data, testNs) - Expect(err2).NotTo(HaveOccurred(), "Failed to apply WatchRule") - - By("verifying WatchRule is ready") - verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") - verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") - - // Let any in-flight reconciles from prior specs drain before triggering - // our event. Without this, a stale WatchRule still in informer cache can - // double-commit the same ConfigMap under a different commitPath and end - // up on HEAD, masking our commit. - time.Sleep(5 * time.Second) - - By("creating test ConfigMap to trigger Git commit") - configMapData := struct { - Name string - Namespace string - }{ - Name: configMapName, - Namespace: testNs, - } - - err3 := applyFromTemplate( - "test/e2e/templates/manager/configmap.tmpl", - configMapData, - testNs, - "--as=jane@acme.com", // Important: we validate later if the user is included in the git commit! - ) - Expect(err3).NotTo(HaveOccurred(), "Failed to apply ConfigMap") - - By("waiting for controller reconciliation of ConfigMap event") - verifyReconciliationLogs := func(g Gomega) { - output, err := controllerLogs(500) - g.Expect(err).NotTo(HaveOccurred()) - - // Check for git commit operation in logs - g.Expect(output).To(ContainSubstring("git commit"), - "Should see git commit operation in controller logs") - } - Eventually(verifyReconciliationLogs).Should(Succeed()) - - By("verifying ConfigMap YAML file exists in Gitea repository") - verifyGitCommit := func(g Gomega) { - // Use the pre-checked out repository directory - By("using pre-checked out repository for verification") - - // Pull latest changes from the remote repository - By("pulling latest changes from remote repository") - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - // Don't use utils.Run() here because it overwrites cmd.Dir with the project directory - pullOutput, pullErr := pullCmd.CombinedOutput() - if pullErr != nil { - g.Expect(pullErr).NotTo(HaveOccurred(), - fmt.Sprintf("Should successfully pull latest changes. Output: %s", string(pullOutput))) - } - - // Check for the expected ConfigMap file (new API-aligned path) - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/configmap-test", - fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, configMapName)) - fileInfo, statErr := os.Stat(expectedFile) - g.Expect(statErr).NotTo(HaveOccurred(), fmt.Sprintf("ConfigMap file should exist at %s", expectedFile)) - g.Expect(fileInfo.Size()).To(BeNumerically(">", 0), "ConfigMap file should not be empty") - - expectedRepoPath := path.Join( - "e2e/configmap-test", - fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, configMapName), - ) - assertLatestCommitForPathTouchesOnlyWithOptional( - g, - managerRepo.CheckoutDir, - expectedRepoPath, - []string{expectedRepoPath}, - []string{ - path.Join("e2e/configmap-test", "README.md"), - path.Join("e2e/configmap-test", ".sops.yaml"), - }, - ) - assertLatestCommitTouchesNoNamespaces( - g, - managerRepo.CheckoutDir, - "e2e/configmap-test/v1/configmaps", - []string{ - "gitops-reverser", - "flux-system", - "kube-system", - "tilt-playground", - }, - ) - - // Verify file content contains expected ConfigMap data - content, readErr := os.ReadFile(expectedFile) - g.Expect(readErr).NotTo(HaveOccurred()) - g.Expect(string(content)).To(ContainSubstring("test-key: test-value"), - "ConfigMap file should contain expected data") - - // Verify latest commit message contains operation, resource path - gitLogCmd := exec.Command("git", "log", "-1", "--pretty=%B") - gitLogCmd.Dir = managerRepo.CheckoutDir - commitMsg, commitErr := gitLogCmd.CombinedOutput() - - if commitErr != nil { - g.Expect(commitErr).NotTo(HaveOccurred(), - fmt.Sprintf("Should read latest commit message. Output: %s", string(commitMsg))) - } - - msg := string(commitMsg) - g.Expect(msg).To(ContainSubstring("[CREATE]"), - "Latest commit message should include operation [CREATE]") - g.Expect(msg).To(ContainSubstring(fmt.Sprintf("v1/configmaps/%s", configMapName)), - "Latest commit message should include resource path") - - gitLogCmd = exec.Command("git", "log", "-1", "--pretty=%an") - gitLogCmd.Dir = managerRepo.CheckoutDir - authorMsg, commitErr := gitLogCmd.CombinedOutput() - if commitErr != nil { - g.Expect(commitErr).NotTo(HaveOccurred(), - fmt.Sprintf("Should read latest commit author. Output: %s", string(authorMsg))) - } - - author := string(authorMsg) - g.Expect(author).To(ContainSubstring("jane@acme.com")) - } - Eventually(verifyGitCommit).Should(Succeed()) - - By("cleaning up test resources") - _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", configMapName, "--ignore-not-found=true") - cleanupWatchRule(watchRuleName, testNs) - cleanupGitTarget(destName, testNs) - - By("✅ ConfigMap to Git commit E2E test passed - verified actual file creation and commit") - fmt.Printf("✅ ConfigMap '%s' successfully triggered Git commit with YAML file in repo '%s'\n", - configMapName, uniqueRepoName) - }) - - // Regression net for the rule-add backfill gap: a ConfigMap that - // already exists in the cluster *before* a WatchRule is applied must - // land in git once the rule becomes Ready. Today this fails because - // ReconcileForRuleChange takes the early-return at manager.go:660 - // when the new rule's GVR is already covered by other rules (and on - // fresh-target installs the snapshot output can be dropped because no - // FolderReconciler is registered yet). The unit-level coverage lives - // in internal/watch/rule_change_snapshot_test.go; this spec locks the - // observable in at the user-visible layer so a future revamp of the - // snapshot trigger logic can't silently regress it. - // - It("should backfill pre-existing ConfigMap when WatchRule is added afterwards", Label("smoke"), func() { - gitProviderName := "gitprovider-normal" - watchRuleName := "watchrule-backfill-test" - configMapName := "preexisting-configmap" - uniqueRepoName := managerRepo.RepoName - - By("creating GitTarget but no WatchRule yet") - destName := watchRuleName + "-dest" - createGitTarget(destName, testNs, gitProviderName, "e2e/backfill-rule-add", "main") - verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") - - By("creating the ConfigMap BEFORE the rule that should select it") - configMapData := struct { - Name string - Namespace string - }{ - Name: configMapName, - Namespace: testNs, - } - err := applyFromTemplate( - "test/e2e/templates/manager/configmap.tmpl", - configMapData, - testNs, - "--as=jane@acme.com", - ) - Expect(err).NotTo(HaveOccurred(), "Failed to pre-create ConfigMap") - - // Confirm nothing committed yet — no rule exists, so the ConfigMap - // must not appear in the repo before we apply the WatchRule. - expectedRepoPath := path.Join( - "e2e/backfill-rule-add", - fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, configMapName), - ) - expectedFile := filepath.Join(managerRepo.CheckoutDir, expectedRepoPath) - - By("applying the WatchRule (rule arrives after the resource)") - watchRuleData := struct { - Name string - Namespace string - DestinationName string - }{ - Name: watchRuleName, - Namespace: testNs, - DestinationName: destName, - } - err = applyFromTemplate("test/e2e/templates/manager/watchrule-configmap.tmpl", watchRuleData, testNs) - Expect(err).NotTo(HaveOccurred(), "Failed to apply WatchRule") - verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") - - By("waiting for the pre-existing ConfigMap to be backfilled into git") - verifyBackfill := func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - pullOutput, pullErr := pullCmd.CombinedOutput() - if pullErr != nil { - g.Expect(pullErr).NotTo(HaveOccurred(), - fmt.Sprintf("Should successfully pull latest changes. Output: %s", string(pullOutput))) - } - - fileInfo, statErr := os.Stat(expectedFile) - g.Expect(statErr).NotTo(HaveOccurred(), - fmt.Sprintf("Pre-existing ConfigMap must be backfilled at %s after the rule lands", expectedFile)) - g.Expect(fileInfo.Size()).To(BeNumerically(">", 0), "Backfilled ConfigMap file should not be empty") - - content, readErr := os.ReadFile(expectedFile) - g.Expect(readErr).NotTo(HaveOccurred()) - g.Expect(string(content)).To(ContainSubstring("test-key: test-value"), - "Backfilled file should contain the ConfigMap data that existed before the rule") - } - Eventually(verifyBackfill). - WithTimeout(60 * time.Second). - WithPolling(2 * time.Second). - Should(Succeed()) - - By("cleaning up test resources") - _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", configMapName, "--ignore-not-found=true") - cleanupWatchRule(watchRuleName, testNs) - cleanupGitTarget(destName, testNs) - - By("✅ rule-add backfill E2E test passed") - fmt.Printf("✅ Pre-existing ConfigMap '%s' was backfilled to repo '%s' after the WatchRule landed\n", - configMapName, uniqueRepoName) - }) - - It("should delete Git file when ConfigMap is deleted via WatchRule", Label("smoke"), func() { - gitProviderName := "gitprovider-normal" - watchRuleName := "watchrule-delete-test" - configMapName := "test-configmap-to-delete" - uniqueRepoName := managerRepo.RepoName - - By("creating WatchRule that monitors ConfigMaps") - destName := watchRuleName + "-dest" - createGitTarget(destName, testNs, gitProviderName, "e2e/delete-test", "main") - data := struct { - Name string - Namespace string - DestinationName string - }{ - Name: watchRuleName, - Namespace: testNs, - DestinationName: destName, - } - - err2 := applyFromTemplate("test/e2e/templates/manager/watchrule-configmap.tmpl", data, testNs) - Expect(err2).NotTo(HaveOccurred(), "Failed to apply WatchRule") - - By("verifying WatchRule is ready") - verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") - verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") - - // See "should create Git commit when ConfigMap is added": prior specs' - // WatchRules can still be in the controller's event router and pile on - // extra commits at unrelated commitPaths, knocking HEAD off our commit. - time.Sleep(5 * time.Second) - - By("creating test ConfigMap to trigger Git commit") - configMapData := struct { - Name string - Namespace string - }{ - Name: configMapName, - Namespace: testNs, - } - - err3 := applyFromTemplate("test/e2e/templates/manager/configmap.tmpl", configMapData, testNs) - Expect(err3).NotTo(HaveOccurred(), "Failed to apply ConfigMap") - - By("waiting for ConfigMap file to appear in Git repository") - verifyFileCreated := func(g Gomega) { - // Pull latest changes from the remote repository - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - pullOutput, pullErr := pullCmd.CombinedOutput() - if pullErr != nil { - g.Expect(pullErr).NotTo(HaveOccurred(), - fmt.Sprintf("Should successfully pull latest changes. Output: %s", string(pullOutput))) - } - - // Check for the expected ConfigMap file (new API-aligned path) - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/delete-test", - fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, configMapName)) - fileInfo, statErr := os.Stat(expectedFile) - g.Expect(statErr).NotTo(HaveOccurred(), fmt.Sprintf("ConfigMap file should exist at %s", expectedFile)) - g.Expect(fileInfo.Size()).To(BeNumerically(">", 0), "ConfigMap file should not be empty") - } - Eventually(verifyFileCreated).Should(Succeed()) - - By("deleting the ConfigMap to trigger DELETE operation") - _, err := kubectlRunInNamespace(testNs, "delete", "configmap", configMapName) - Expect(err).NotTo(HaveOccurred(), "ConfigMap deletion should succeed") - - By("verifying ConfigMap file is deleted from Git repository") - verifyFileDeleted := func(g Gomega) { - // Pull latest changes from the remote repository - By("pulling latest changes after deletion") - pullLatestRepoState(g, managerRepo.CheckoutDir) - - // Check that the ConfigMap file no longer exists (new API-aligned path) - expectedRelativePath := path.Join( - "e2e/delete-test", - fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, configMapName), - ) - expectedFile := filepath.Join(managerRepo.CheckoutDir, expectedRelativePath) - _, statErr := os.Stat(expectedFile) - g.Expect(statErr).To(HaveOccurred(), fmt.Sprintf("ConfigMap file should NOT exist at %s", expectedFile)) - g.Expect(os.IsNotExist(statErr)).To(BeTrue(), "Error should be 'file does not exist'") - - assertLatestCommitForPathTouchesOnlyWithOptional( - g, - managerRepo.CheckoutDir, - expectedRelativePath, - []string{expectedRelativePath}, - nil, - ) - assertLatestCommitTouchesNoNamespaces( - g, - managerRepo.CheckoutDir, - "e2e/delete-test/v1/configmaps", - []string{ - "gitops-reverser", - "flux-system", - "kube-system", - "tilt-playground", - }, - ) - - // Verify git log shows DELETE commit - By("verifying git log shows DELETE operation") - gitLogCmd := exec.Command("git", "log", "--oneline", "-n", "5") - gitLogCmd.Dir = managerRepo.CheckoutDir - logOutput, logErr := gitLogCmd.CombinedOutput() - g.Expect(logErr).NotTo(HaveOccurred(), "Should be able to read git log") - g.Expect(string(logOutput)).To(ContainSubstring("DELETE"), - "Git log should contain DELETE operation") - } - Eventually(verifyFileDeleted).Should(Succeed()) - - By("cleaning up test resources") - cleanupWatchRule(watchRuleName, testNs) - cleanupGitTarget(destName, testNs) - - By("✅ ConfigMap deletion E2E test passed - verified file removal from Git") - fmt.Printf("✅ ConfigMap '%s' deletion successfully triggered Git commit removing file from repo '%s'\n", - configMapName, uniqueRepoName) - }) - - It("should create Git commit when IceCreamOrder CRD is installed via ClusterWatchRule", Label("smoke"), func() { - gitProviderName := "gitprovider-normal" - clusterWatchRuleName := "clusterwatchrule-crd-install" - crdName := "icecreamorders.shop.example.com" - - By("creating ClusterWatchRule with Cluster scope for CRDs") - destName := clusterWatchRuleName + "-dest" - createGitTarget(destName, testNs, gitProviderName, "e2e/crd-install-test", "main") - - clusterWatchRuleData := struct { - Name string - DestinationName string - Namespace string - }{ - Name: clusterWatchRuleName, - DestinationName: destName, - Namespace: testNs, - } - - err := applyFromTemplate("test/e2e/templates/manager/clusterwatchrule-crd.tmpl", clusterWatchRuleData, "") - Expect(err).NotTo(HaveOccurred(), "Failed to apply ClusterWatchRule for CRDs") - - By("verifying ClusterWatchRule is ready") - verifyResourceStatus("clusterwatchrule", clusterWatchRuleName, "", "True", "Ready", "") - verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") - - By("installing the IceCreamOrder CRD to trigger Git commit") - _, err = kubectlRun("apply", "-f", "test/e2e/templates/icecreamorder-crd.yaml") - Expect(err).NotTo(HaveOccurred(), "Failed to install CRD") - - By("waiting for CRD to be established") - verifyCRDEstablished := func(g Gomega) { - output, err := kubectlRun( - "get", - "crd", - crdName, - "-o", - "jsonpath={.status.conditions[?(@.type=='Established')].status}", - ) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(Equal("True")) - } - Eventually(verifyCRDEstablished).Should(Succeed()) - - By("verifying CRD YAML file exists in Git repository (NO namespace in path - cluster resource)") - verifyGitCommit := func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - pullOutput, pullErr := pullCmd.CombinedOutput() - if pullErr != nil { - g.Expect(pullErr).NotTo(HaveOccurred(), - fmt.Sprintf("Should successfully pull latest changes. Output: %s", string(pullOutput))) - } - - // CRDs are cluster-scoped, so path should NOT include namespace - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/crd-install-test", - "apiextensions.k8s.io/v1/customresourcedefinitions/icecreamorders.shop.example.com.yaml") - fileInfo, statErr := os.Stat(expectedFile) - g.Expect(statErr).NotTo(HaveOccurred(), fmt.Sprintf("CRD file should exist at %s", expectedFile)) - g.Expect(fileInfo.Size()).To(BeNumerically(">", 0), "CRD file should not be empty") - - // Verify file content - content, readErr := os.ReadFile(expectedFile) - g.Expect(readErr).NotTo(HaveOccurred()) - g.Expect(string(content)).To(ContainSubstring("kind: CustomResourceDefinition"), - "File should contain CRD kind") - g.Expect(string(content)).To(ContainSubstring("name: icecreamorders.shop.example.com"), - "File should contain CRD name") - } - Eventually(verifyGitCommit). - WithTimeout(60 * time.Second). - WithPolling(2 * time.Second). - Should(Succeed()) - - By("cleaning up test resources") - cleanupClusterWatchRule(clusterWatchRuleName) - cleanupGitTarget(destName, testNs) - // Keep CRD installed for subsequent tests - - By("✅ CRD installation via ClusterWatchRule E2E test passed") - }) - - It("should create Git commit when IceCreamOrder is added via WatchRule", func() { - gitProviderName := "gitprovider-normal" - watchRuleName := "watchrule-icecream-orders" - - By("installing the IceCreamOrder CRD first (needed for custom resource tests)") - _, err := kubectlRun("apply", "-f", "test/e2e/templates/icecreamorder-crd.yaml") - Expect(err).NotTo(HaveOccurred(), "Failed to install sample CRD") - - By("waiting for CRD to be established") - verifyCRDEstablished := func(g Gomega) { - output, err := kubectlRun( - "get", - "crd", - "icecreamorders.shop.example.com", - "-o", - "jsonpath={.status.conditions[?(@.type=='Established')].status}", - ) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(Equal("True")) - } - Eventually(verifyCRDEstablished, 30*time.Second, time.Second).Should(Succeed()) - - crdInstanceName := "alices-order" - uniqueRepoName := managerRepo.RepoName - - By("creating WatchRule that monitors IceCreamOrder resources") - destName := watchRuleName + "-dest" - createGitTarget(destName, testNs, gitProviderName, "e2e/icecream-test", "main") - - data := struct { - Name string - Namespace string - DestinationName string - }{ - Name: watchRuleName, - Namespace: testNs, - DestinationName: destName, - } - - err2 := applyFromTemplate("test/e2e/templates/watchrule-crd.tmpl", data, testNs) - Expect(err2).NotTo(HaveOccurred(), "Failed to apply WatchRule for CRDs") - - By("verifying WatchRule is ready") - verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") - verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") - - By("creating CR with labels and annotations to trigger Git commit") - crdInstanceData := struct { - Name string - Namespace string - Labels map[string]string - Annotations map[string]string - CustomerName string - Container string - Scoops []struct { - Flavor string - Quantity int - } - Toppings []string - }{ - Name: crdInstanceName, - Namespace: testNs, - Labels: map[string]string{ - "environment": "test", - "team": "engineering", - }, - Annotations: map[string]string{ - "description": "Alice's favorite ice cream", - "priority": "high", - "kubectl.kubernetes.io/last-applied-configuration": "should-be-filtered", - "deployment.kubernetes.io/revision": "should-also-be-filtered", - }, - CustomerName: "Alice", - Container: "Cone", - Scoops: []struct { - Flavor string - Quantity int - }{ - {Flavor: "Vanilla", Quantity: 2}, - {Flavor: "Chocolate", Quantity: 1}, - }, - Toppings: []string{"Sprinkles", "HotFudge"}, - } - - err3 := applyFromTemplate("test/e2e/templates/icecreamorder-instance.tmpl", crdInstanceData, testNs) - Expect(err3).NotTo(HaveOccurred(), "Failed to apply CRD instance") - - By("waiting for controller reconciliation of CRD instance event") - verifyReconciliationLogs := func(g Gomega) { - output, err := kubectlRunInNamespace( - namespace, - "logs", - "-l", - "control-plane=gitops-reverser", - "--tail=500", - "--prefix=true", - ) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring("git commit"), - "Should see git commit operation in logs") - } - Eventually(verifyReconciliationLogs, 45*time.Second, 2*time.Second).Should(Succeed()) - - By("verifying CRD instance YAML file exists in Gitea repository") - verifyGitCommit := func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - pullOutput, pullErr := pullCmd.CombinedOutput() - if pullErr != nil { - g.Expect(pullErr).NotTo(HaveOccurred(), - fmt.Sprintf("Should successfully pull latest changes. Output: %s", string(pullOutput))) - } - - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/icecream-test", - fmt.Sprintf("shop.example.com/v1/icecreamorders/%s/%s.yaml", testNs, crdInstanceName)) - fileInfo, statErr := os.Stat(expectedFile) - g.Expect(statErr). - NotTo(HaveOccurred(), fmt.Sprintf("CRD instance file should exist at %s", expectedFile)) - g.Expect(fileInfo.Size()).To(BeNumerically(">", 0), "CRD instance file should not be empty") - - content, readErr := os.ReadFile(expectedFile) - g.Expect(readErr).NotTo(HaveOccurred()) - contentStr := string(content) - g.Expect(contentStr).To(ContainSubstring("kind: IceCreamOrder"), - "CRD instance file should contain IceCreamOrder kind") - g.Expect(contentStr).To(ContainSubstring("customerName: Alice"), - "CRD instance file should contain customer name") - g.Expect(contentStr).To(ContainSubstring("container: Cone"), - "CRD instance file should contain container type") - g.Expect(contentStr).To(ContainSubstring("flavor: Vanilla"), - "CRD instance file should contain ice cream flavors") - - // Verify labels are present - g.Expect(contentStr).To(ContainSubstring("environment: test"), - "CRD instance file should contain environment label") - g.Expect(contentStr).To(ContainSubstring("team: engineering"), - "CRD instance file should contain team label") - - // Verify user annotations are present - g.Expect(contentStr).To(ContainSubstring("description: Alice's favorite ice cream"), - "CRD instance file should contain description annotation") - g.Expect(contentStr).To(ContainSubstring("priority: high"), - "CRD instance file should contain priority annotation") - - // Verify filtered annotations are NOT present - g.Expect(contentStr).NotTo(ContainSubstring("kubectl.kubernetes.io/last-applied-configuration"), - "CRD instance file should NOT contain kubectl annotation") - g.Expect(contentStr).NotTo(ContainSubstring("deployment.kubernetes.io/revision"), - "CRD instance file should NOT contain deployment annotation") - - // Verify status field is NOT present in Git - g.Expect(contentStr).NotTo(ContainSubstring("status:"), - "CRD instance file should NOT contain status field") - } - Eventually(verifyGitCommit). - WithTimeout(60 * time.Second). - WithPolling(2 * time.Second). - Should(Succeed()) - - By("verifying the IceCreamOrder still exists before status update") - verifyCRDInstanceExists := func(g Gomega) { - _, err := kubectlRunInNamespace(testNs, "get", "icecreamorder", crdInstanceName) - g.Expect(err).NotTo(HaveOccurred(), "IceCreamOrder should exist before status patch") - } - Eventually(verifyCRDInstanceExists, 15*time.Second, time.Second).Should(Succeed()) - - By("applying status update to the IceCreamOrder CR") - statusPatch := `{"status":{"phase":"Preparing","cost":12.5,"message":"Queued for pickup"}}` - statusOutput, statusErr := kubectlRunInNamespace( - testNs, - "patch", - "icecreamorder", - crdInstanceName, - "--type=merge", - "--subresource=status", - "-p", - statusPatch, - ) - Expect(statusErr).NotTo(HaveOccurred(), "Status subresource patch should succeed") - By(fmt.Sprintf("status patched successfully: %s", statusOutput)) - - By("getting current git commit hash") - gitRevCmd := exec.Command("git", "rev-parse", "HEAD") - gitRevCmd.Dir = managerRepo.CheckoutDir - beforeStatusCommit, _ := gitRevCmd.Output() - - By("waiting to ensure no new commit is created from status update") - time.Sleep(10 * time.Second) - - By("verifying no new commit was created and status is not in Git") - verifyStatusNotCommitted := func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - _, _ = pullCmd.CombinedOutput() - - // Check that commit hash hasn't changed - gitRevCmd := exec.Command("git", "rev-parse", "HEAD") - gitRevCmd.Dir = managerRepo.CheckoutDir - afterStatusCommit, err := gitRevCmd.Output() - g.Expect(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("Commit before status: %s", string(beforeStatusCommit))) - By(fmt.Sprintf("Commit after status: %s", string(afterStatusCommit))) - - // Read the file again to ensure status is still not present - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/icecream-test", - fmt.Sprintf("shop.example.com/v1/icecreamorders/%s/%s.yaml", testNs, crdInstanceName)) - content, readErr := os.ReadFile(expectedFile) - g.Expect(readErr).NotTo(HaveOccurred()) - g.Expect(string(content)).NotTo(ContainSubstring("status:"), - "CRD instance file should still NOT contain status field after status update") - g.Expect(string(content)).NotTo(ContainSubstring("phase:"), - "CRD instance file should NOT contain status phase content") - g.Expect(string(content)).NotTo(ContainSubstring("Queued for pickup"), - "CRD instance file should NOT contain status content") - } - Eventually(verifyStatusNotCommitted).Should(Succeed()) - - By("✅ Status update verified - no Git commit created and status not in file") - - By("cleaning up IceCreamOrder instance") - _, _ = kubectlRunInNamespace(testNs, "delete", "icecreamorder", crdInstanceName) - - By("Note: GitTarget, WatchRule, GitProvider, and CRD kept for subsequent tests") - - By("✅ IceCreamOrder to Git commit E2E test passed") - fmt.Printf("✅ IceCreamOrder '%s' successfully triggered Git commit in repo '%s'\n", - crdInstanceName, uniqueRepoName) - }) - - It("should update Git file when IceCreamOrder is modified via WatchRule", func() { - crdInstanceName := "bobs-order" - uniqueRepoName := managerRepo.RepoName - - By("creating initial IceCreamOrder instance") - crdInstanceData := struct { - Name string - Namespace string - Labels map[string]string - Annotations map[string]string - CustomerName string - Container string - Scoops []struct { - Flavor string - Quantity int - } - Toppings []string - }{ - Name: crdInstanceName, - Namespace: testNs, - Labels: nil, - Annotations: nil, - CustomerName: "Bob", - Container: "Cup", - Scoops: []struct { - Flavor string - Quantity int - }{ - {Flavor: "Strawberry", Quantity: 1}, - }, - Toppings: []string{"WhippedCream"}, - } - - err := applyFromTemplate("test/e2e/templates/icecreamorder-instance.tmpl", crdInstanceData, testNs) - Expect(err).NotTo(HaveOccurred(), "Failed to apply initial CRD instance") - - By("waiting for initial CRD instance file to appear in Git") - verifyInitialFile := func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - _, _ = pullCmd.CombinedOutput() - - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/icecream-test", - fmt.Sprintf("shop.example.com/v1/icecreamorders/%s/%s.yaml", testNs, crdInstanceName)) - content, readErr := os.ReadFile(expectedFile) - g.Expect(readErr).NotTo(HaveOccurred()) - g.Expect(string(content)).To(ContainSubstring("customerName: Bob")) - g.Expect(string(content)).To(ContainSubstring("flavor: Strawberry")) - } - Eventually(verifyInitialFile).Should(Succeed()) - - By("updating CRD instance with new values") - updatedCRDData := struct { - Name string - Namespace string - Labels map[string]string - Annotations map[string]string - CustomerName string - Container string - Scoops []struct { - Flavor string - Quantity int - } - Toppings []string - }{ - Name: crdInstanceName, - Namespace: testNs, - Labels: nil, - Annotations: nil, - CustomerName: "Bob", - Container: "WaffleBowl", - Scoops: []struct { - Flavor string - Quantity int - }{ - {Flavor: "RockyRoad", Quantity: 3}, - {Flavor: "MintChip", Quantity: 2}, - }, - Toppings: []string{"HotFudge", "Caramel", "Sprinkles"}, - } - - err = applyFromTemplate("test/e2e/templates/icecreamorder-instance.tmpl", updatedCRDData, testNs) - Expect(err).NotTo(HaveOccurred(), "Failed to update CRD instance") - - By("verifying updated CRD instance content in Git") - verifyUpdatedFile := func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - _, _ = pullCmd.CombinedOutput() - - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/icecream-test", - fmt.Sprintf("shop.example.com/v1/icecreamorders/%s/%s.yaml", testNs, crdInstanceName)) - content, readErr := os.ReadFile(expectedFile) - g.Expect(readErr).NotTo(HaveOccurred()) - g.Expect(string(content)).To(ContainSubstring("container: WaffleBowl"), - "Updated file should contain new container type") - g.Expect(string(content)).To(ContainSubstring("flavor: RockyRoad"), - "Updated file should contain new flavor") - g.Expect(string(content)).To(ContainSubstring("quantity: 3"), - "Updated file should contain new quantity") - } - Eventually(verifyUpdatedFile).Should(Succeed()) - - By("cleaning up IceCreamOrder instance") - _, _ = kubectlRunInNamespace(testNs, "delete", "icecreamorder", crdInstanceName) - - By("Note: GitTarget, WatchRule, GitProvider, and CRD kept for subsequent tests") - - By("✅ IceCreamOrder update E2E test passed") - fmt.Printf("✅ IceCreamOrder '%s' update successfully reflected in Git repo '%s'\n", - crdInstanceName, uniqueRepoName) - }) - - It("should delete Git file when IceCreamOrder is deleted via WatchRule", func() { - crdInstanceName := "charlies-order" - uniqueRepoName := managerRepo.RepoName - - By("creating IceCreamOrder instance") - crdInstanceData := struct { - Name string - Namespace string - Labels map[string]string - Annotations map[string]string - CustomerName string - Container string - Scoops []struct { - Flavor string - Quantity int - } - Toppings []string - }{ - Name: crdInstanceName, - Namespace: testNs, - Labels: nil, - Annotations: nil, - CustomerName: "Charlie", - Container: "Cone", - Scoops: []struct { - Flavor string - Quantity int - }{ - {Flavor: "Chocolate", Quantity: 2}, - }, - Toppings: []string{"Sprinkles"}, - } - - err := applyFromTemplate("test/e2e/templates/icecreamorder-instance.tmpl", crdInstanceData, testNs) - Expect(err).NotTo(HaveOccurred(), "Failed to apply CR") - - By("waiting for CR file to appear in Git repository") - verifyFileCreated := func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - _, _ = pullCmd.CombinedOutput() - - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/icecream-test", - fmt.Sprintf("shop.example.com/v1/icecreamorders/%s/%s.yaml", testNs, crdInstanceName)) - fileInfo, statErr := os.Stat(expectedFile) - g.Expect(statErr). - NotTo(HaveOccurred(), fmt.Sprintf("CRD instance file should exist at %s", expectedFile)) - g.Expect(fileInfo.Size()).To(BeNumerically(">", 0), "CRD instance file should not be empty") - } - Eventually(verifyFileCreated).Should(Succeed()) - - By("deleting the CR to trigger DELETE operation") - _, err = kubectlRunInNamespace(testNs, "delete", "icecreamorder", crdInstanceName) - Expect(err).NotTo(HaveOccurred(), "CRD instance deletion should succeed") - - By("verifying CRD instance file is deleted from Git repository") - verifyFileDeleted := func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - _, _ = pullCmd.CombinedOutput() - - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/icecream-test", - fmt.Sprintf("shop.example.com/v1/icecreamorders/%s/%s.yaml", testNs, crdInstanceName)) - _, statErr := os.Stat(expectedFile) - g.Expect(statErr). - To(HaveOccurred(), fmt.Sprintf("CRD instance file should NOT exist at %s", expectedFile)) - g.Expect(os.IsNotExist(statErr)).To(BeTrue(), "Error should be 'file does not exist'") - - By("verifying git log shows DELETE commit") - gitLogCmd := exec.Command("git", "log", "--oneline", "-n", "5") - gitLogCmd.Dir = managerRepo.CheckoutDir - logOutput, logErr := gitLogCmd.CombinedOutput() - g.Expect(logErr).NotTo(HaveOccurred(), "Should be able to read git log") - g.Expect(string(logOutput)).To(ContainSubstring("DELETE"), - "Git log should contain DELETE operation") - } - Eventually(verifyFileDeleted).Should(Succeed()) - - By("✅ IceCreamOrder deletion E2E test passed") - fmt.Printf("✅ IceCreamOrder '%s' deletion successfully removed file from Git repo '%s'\n", - crdInstanceName, uniqueRepoName) - }) - - It("should delete Git file when IceCreamOrder CRD is deleted via ClusterWatchRule", func() { - gitProviderName := "gitprovider-normal" - clusterWatchRuleName := "clusterwatchrule-crd-delete" - crdName := "icecreamorders.shop.example.com" - - By("creating ClusterWatchRule with Cluster scope for CRDs") - destName := clusterWatchRuleName + "-dest" - createGitTarget(destName, testNs, gitProviderName, "e2e/crd-delete-test", "main") - verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") - - clusterWatchRuleData := struct { - Name string - DestinationName string - Namespace string - }{ - Name: clusterWatchRuleName, - DestinationName: destName, - Namespace: testNs, - } - - err := applyFromTemplate("test/e2e/templates/manager/clusterwatchrule-crd.tmpl", clusterWatchRuleData, "") - Expect(err).NotTo(HaveOccurred(), "Failed to apply ClusterWatchRule for CRDs") - - By("verifying ClusterWatchRule is ready") - - verifyResourceStatus("clusterwatchrule", clusterWatchRuleName, "", "True", "Ready", "") - - By("verifying CRD file exists in Git before deletion") - verifyFileExists := func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - _, _ = pullCmd.CombinedOutput() - - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/crd-delete-test", - "apiextensions.k8s.io/v1/customresourcedefinitions/icecreamorders.shop.example.com.yaml") - _, statErr := os.Stat(expectedFile) - g.Expect(statErr).NotTo(HaveOccurred(), "CRD file should exist before deletion") - } - Eventually(verifyFileExists).Should(Succeed()) - - By("deleting the CRD to trigger DELETE operation") - _, deleteErr := kubectlRun("delete", "crd", crdName) - Expect(deleteErr).NotTo(HaveOccurred(), "CRD deletion should succeed") - By("verifying CRD file is deleted from Git repository") - verifyFileDeleted := func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - _, _ = pullCmd.CombinedOutput() - - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/crd-delete-test", - "apiextensions.k8s.io/v1/customresourcedefinitions/icecreamorders.shop.example.com.yaml") - _, statErr := os.Stat(expectedFile) - g.Expect(statErr).To(HaveOccurred(), "CRD file should NOT exist after deletion") - g.Expect(os.IsNotExist(statErr)).To(BeTrue(), "Error should be 'file does not exist'") - - // Verify git log shows DELETE commit - gitLogCmd := exec.Command("git", "log", "--oneline", "-n", "5") - gitLogCmd.Dir = managerRepo.CheckoutDir - logOutput, logErr := gitLogCmd.CombinedOutput() - g.Expect(logErr).NotTo(HaveOccurred(), "Should be able to read git log") - g.Expect(string(logOutput)).To(ContainSubstring("DELETE"), - "Git log should contain DELETE operation") - } - Eventually(verifyFileDeleted, "60s", "1s").Should(Succeed()) - - By("verifying the deleted CRD file does not reappear after terminating updates") - Consistently(func(g Gomega) { - pullCmd := exec.Command("git", "pull") - pullCmd.Dir = managerRepo.CheckoutDir - _, _ = pullCmd.CombinedOutput() - - expectedFile := filepath.Join(managerRepo.CheckoutDir, - "e2e/crd-delete-test", - "apiextensions.k8s.io/v1/customresourcedefinitions/icecreamorders.shop.example.com.yaml") - _, statErr := os.Stat(expectedFile) - g.Expect(statErr).To(HaveOccurred(), "CRD file must stay deleted after CRD termination updates") - g.Expect(os.IsNotExist(statErr)).To(BeTrue(), "CRD file must not reappear in Git") - }, "15s", "1s").Should(Succeed()) - - By("cleaning up test resources") - cleanupClusterWatchRule(clusterWatchRuleName) - cleanupGitTarget(destName, testNs) - - By("✅ CRD deletion via ClusterWatchRule E2E test passed") - }) - - AfterAll(func() { - By("cleaning up shared test resources") - - // Clean up WatchRule from IceCreamOrder tests - cleanupWatchRule("watchrule-icecream-orders", testNs) - - // Clean up GitTarget from IceCreamOrder tests - cleanupGitTarget("watchrule-icecream-orders-dest", testNs) - - // Clean up GitProvider from IceCreamOrder tests - cleanupNamespacedResource(testNs, "gitprovider", "gitprovider-normal") - - // Clean up IceCreamOrder CRD - _, _ = kubectlRun( - "delete", - "crd", - "icecreamorders.shop.example.com", - "--ignore-not-found=true", - ) - }) - - // +kubebuilder:scaffold:e2e-webhooks-checks - }) -}) - func readSOPSAgeKeyFromFile(path string) (string, error) { content, err := os.ReadFile(path) if err != nil { @@ -1914,7 +217,11 @@ func verifyResourceStatus(resourceType, name, ns, expectedStatus, expectedReason g.Expect(readyMessage).To(ContainSubstring(expectedMessageContains)) } } - Eventually(verifyStatus).Should(Succeed()) + // 90s (vs the 30s default) absorbs slow-CI and parallel load: in particular, + // after a fresh CRD install the controller's discovery cache can lag tens of + // seconds before it serves the new GVR and a dependent WatchRule can be + // planned and reach Ready. The happy path still returns as soon as Ready. + Eventually(verifyStatus, "90s", "2s").Should(Succeed()) } // showControllerLogs displays the current controller logs to help with debugging during test execution. @@ -1953,11 +260,3 @@ func showControllerLogs(context string) { fmt.Printf("%s\n", output) fmt.Printf("----------------------------------------\n") } - -// minInt returns the minimum of two integers. -func minInt(a, b int) int { - if a < b { - return a - } - return b -} diff --git a/test/e2e/gitprovider_validation_e2e_test.go b/test/e2e/gitprovider_validation_e2e_test.go new file mode 100644 index 000000000..e9bcdc114 --- /dev/null +++ b/test/e2e/gitprovider_validation_e2e_test.go @@ -0,0 +1,162 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Manager GitProvider Validation", Label("manager"), Ordered, func() { + var ( + testNs string + validationRepo *RepoArtifacts + ) + + BeforeAll(func() { + By("creating GitProvider validation test namespace") + testNs = testNamespaceFor("manager-gitprovider") + _, _ = kubectlRun("create", "namespace", testNs) // idempotent; ignore AlreadyExists + + By("setting up Gitea repo and credentials for GitProvider validation tests") + validationRepo = SetupRepo( + resolveE2EContext(), + testNs, + fmt.Sprintf("e2e-manager-gitprovider-%d", GinkgoRandomSeed()), + ) + + By("applying git secrets to test namespace") + _, err := kubectlRunInNamespace(testNs, "apply", "-f", validationRepo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") + applySOPSAgeKeyToNamespace(testNs) + }) + + AfterAll(func() { + cleanupNamespace(testNs) + }) + + SetDefaultEventuallyTimeout(30 * time.Second) + SetDefaultEventuallyPollingInterval(time.Second) + + It("should validate GitProvider with real Gitea repository", Label("smoke"), func() { + gitProviderName := "gitprovider-e2e-test" + + By("showing initial controller logs") + showControllerLogs("before creating GitProvider") + + createGitProviderWithURLInNamespace( + gitProviderName, + testNs, + validationRepo.GitSecretHTTP, + validationRepo.RepoURLHTTP, + ) + + By("showing controller logs after GitProvider creation") + showControllerLogs("after creating GitProvider") + + verifyResourceStatus( + "gitprovider", gitProviderName, testNs, + "True", "Ready", "Repository connectivity validated", + ) + + By("showing final controller logs") + showControllerLogs("after status verification") + + _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", gitProviderName, "--ignore-not-found=true") + }) + + It("should handle GitProvider with invalid credentials", func() { + gitProviderName := "gitprovider-invalid-test" + createGitProviderWithURLInNamespace( + gitProviderName, + testNs, + validationRepo.GitSecretInvalid, + validationRepo.RepoURLHTTP, + ) + verifyResourceStatus("gitprovider", gitProviderName, testNs, "False", "ConnectionFailed", "") + _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", gitProviderName, "--ignore-not-found=true") + }) + + It("should handle GitTarget with nonexistent branch pattern", func() { + gitProviderName := "gitprovider-branch-test" + + // GitProvider should be Ready=True (validates connectivity, not branch existence) + createGitProviderWithURLInNamespace( + gitProviderName, + testNs, + validationRepo.GitSecretHTTP, + validationRepo.RepoURLHTTP, + ) + verifyResourceStatus( + "gitprovider", gitProviderName, testNs, "True", "Ready", "Repository connectivity validated", + ) + + // GitTarget with branch not matching any pattern should fail + destName := "dest-invalid-branch" + createGitTarget(destName, testNs, gitProviderName, "test/invalid", "different-branch") + + By("verifying GitTarget fails branch validation") + verifyResourceStatus("gittarget", destName, testNs, "False", "ValidationFailed", "BranchNotAllowed") + + cleanupGitTarget(destName, testNs) + _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", gitProviderName, "--ignore-not-found=true") + }) + + It("should validate GitProvider with SSH authentication", func() { + gitProviderName := "gitprovider-ssh-test" + + By("🔐 Starting SSH authentication test") + showControllerLogs("before SSH test") + + By("📋 Checking SSH secret exists") + secretOutput, err := kubectlRunInNamespace(testNs, "get", "secret", validationRepo.GitSecretSSH, "-o", "yaml") + if err != nil { + fmt.Printf("❌ SSH secret not found: %v\n", err) + } else { + previewLen := min(300, len(secretOutput)) + fmt.Printf( + "✅ SSH secret exists - showing first %d chars:\n%s...\n", + previewLen, + secretOutput[:previewLen], + ) + } + + createGitProviderWithURLInNamespace( + gitProviderName, + testNs, + validationRepo.GitSecretSSH, + validationRepo.RepoURLSSH, + ) + + By("🔍 Controller logs after SSH GitProvider creation") + showControllerLogs("after SSH GitProvider creation") + + verifyResourceStatus( + "gitprovider", gitProviderName, testNs, "True", "Ready", "Repository connectivity validated", + ) + + By("✅ Final SSH test logs") + showControllerLogs("SSH test completion") + + _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", gitProviderName, "--ignore-not-found=true") + }) +}) diff --git a/test/e2e/gittarget_isolation_e2e_test.go b/test/e2e/gittarget_isolation_e2e_test.go new file mode 100644 index 000000000..49a2959e7 --- /dev/null +++ b/test/e2e/gittarget_isolation_e2e_test.go @@ -0,0 +1,191 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "path" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// This spec is the e2e regression for GitTarget isolation on rule changes (see +// docs/design/gittarget-isolation-on-rule-change.md). The original symptom was a +// parallel-run flake: one GitTarget's ConfigMap event commit ("[CREATE] ...") +// was replaced by a snapshot commit ("reconcile: sync N resources") because an +// unrelated spec changed a *different* target's rules at the same time, dragging +// every target into rule-change snapshot mode. +// +// It is deliberately NOT Serial — the whole point is that isolation must hold +// under parallel execution. It is Label("smoke") so it runs in the same suite +// where the flake originally appeared. +var _ = Describe("Manager GitTarget Isolation", Label("manager"), Label("smoke"), Ordered, func() { + const ( + providerName = "gitprovider-iso" + targetA = "iso-target-a" + targetB = "iso-target-b" + ruleA = "iso-rule-a" + ruleB = "iso-rule-b" + pathA = "e2e/iso-a" + pathB = "e2e/iso-b" + ) + + var ( + testNs string + isoRepo *RepoArtifacts + ) + + BeforeAll(func() { + By("creating GitTarget isolation test namespace") + testNs = testNamespaceFor("manager-isolation") + _, _ = kubectlRun("create", "namespace", testNs) // idempotent; ignore AlreadyExists + + By("setting up Gitea repo and credentials for isolation tests") + isoRepo = SetupRepo( + resolveE2EContext(), + testNs, + fmt.Sprintf("e2e-manager-isolation-%d", GinkgoRandomSeed()), + ) + + By("applying git secrets to test namespace") + _, err := kubectlRunInNamespace(testNs, "apply", "-f", isoRepo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") + applySOPSAgeKeyToNamespace(testNs) + + By("creating shared GitProvider for isolation specs") + createGitProviderWithURLInNamespace( + providerName, + testNs, + isoRepo.GitSecretHTTP, + isoRepo.RepoURLHTTP, + ) + verifyResourceStatus( + "gitprovider", providerName, testNs, + "True", "Ready", "Repository connectivity validated", + ) + + By("creating two independent GitTargets writing to separate paths in the same repo") + createGitTarget(targetA, testNs, providerName, pathA, "main") + createGitTarget(targetB, testNs, providerName, pathB, "main") + verifyResourceStatus("gittarget", targetA, testNs, "True", "Ready", "") + verifyResourceStatus("gittarget", targetB, testNs, "True", "Ready", "") + + By("target A and target B both watch ConfigMaps (baseline steady state)") + applyIsolationWatchRule(ruleA, testNs, targetA, `"configmaps"`) + applyIsolationWatchRule(ruleB, testNs, targetB, `"configmaps"`) + verifyResourceStatus("watchrule", ruleA, testNs, "True", "Ready", "") + verifyResourceStatus("watchrule", ruleB, testNs, "True", "Ready", "") + }) + + AfterAll(func() { + cleanupNamespace(testNs) + }) + + SetDefaultEventuallyTimeout(30 * time.Second) + SetDefaultEventuallyPollingInterval(time.Second) + + It("keeps target A's commits as events while target B's rules churn", Label("smoke"), func() { + // Let any in-flight reconciles from setup drain before we begin, so the + // baseline snapshot commits are settled and only our event commits land + // next on target A's path. + time.Sleep(5 * time.Second) + + // Each iteration changes target B's effective watch plan (toggling an + // extra GVR on/off, which also churns the global informer set) and then + // creates a brand-new ConfigMap that target A must commit. If isolation + // holds, target A always produces an event commit; pre-fix, target B's + // churn would force target A into a "reconcile: sync" snapshot. + for i := range 3 { + cmName := fmt.Sprintf("iso-cm-%d", i) + + By(fmt.Sprintf("churning target B's rule set (iteration %d)", i)) + if i%2 == 0 { + applyIsolationWatchRule(ruleB, testNs, targetB, `"configmaps", "services"`) + } else { + applyIsolationWatchRule(ruleB, testNs, targetB, `"configmaps"`) + } + + By(fmt.Sprintf("creating ConfigMap %q for target A", cmName)) + applyIsolationConfigMap(cmName, testNs) + + By("asserting target A commits it as an event commit, not a snapshot") + relPath := path.Join(pathA, fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, cmName)) + assertEventCommit := func(g Gomega) { + pullLatestRepoState(g, isoRepo.CheckoutDir) + + msg := lastCommitMessageForPath(g, isoRepo.CheckoutDir, relPath) + g.Expect(msg).To(ContainSubstring("[CREATE]"), + "target A's commit for %s must be a [CREATE] event commit", cmName) + g.Expect(msg).To(ContainSubstring(fmt.Sprintf("v1/configmaps/%s", cmName)), + "target A's commit message must name the resource path") + g.Expect(msg).NotTo(ContainSubstring("reconcile: sync"), + "target A must not enter snapshot mode because of target B's unrelated rule change "+ + "(GitTarget isolation — see docs/design/gittarget-isolation-on-rule-change.md)") + } + Eventually(assertEventCommit).Should(Succeed()) + } + }) +}) + +// applyIsolationWatchRule applies a WatchRule selecting the given resources list +// (a raw YAML array body, e.g. `"configmaps", "services"`) for the target. +func applyIsolationWatchRule(name, namespace, target, resources string) { + data := struct { + Name string + Namespace string + DestinationName string + Resources string + }{ + Name: name, + Namespace: namespace, + DestinationName: target, + Resources: resources, + } + Expect(applyFromTemplate("test/e2e/templates/manager/watchrule-resources.tmpl", data, namespace)). + To(Succeed(), "failed to apply isolation WatchRule %q", name) +} + +// applyIsolationConfigMap creates a watched ConfigMap, attributed to a fixed +// user so the commit is a genuine event commit. +func applyIsolationConfigMap(name, namespace string) { + data := struct { + Name string + Namespace string + }{ + Name: name, + Namespace: namespace, + } + Expect(applyFromTemplate( + "test/e2e/templates/manager/configmap.tmpl", + data, + namespace, + "--as=jane@acme.com", + )).To(Succeed(), "failed to apply isolation ConfigMap %q", name) +} + +// lastCommitMessageForPath returns the body of the most recent commit that +// touched the given repo-relative path. Scoping by path keeps each GitTarget's +// history unambiguous even when several targets share one repo. +func lastCommitMessageForPath(g Gomega, checkoutDir, relPath string) string { + out, err := gitRun(checkoutDir, "log", "-1", "--pretty=%B", "--", relPath) + g.Expect(err).NotTo(HaveOccurred(), "git log failed: %s", out) + return out +} diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index d657488d9..7505f0898 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -336,6 +336,38 @@ func gitRun(dir string, args ...string) (string, error) { return string(out), err } +// expectServiceRoutesToPod asserts that the given Service's endpoints resolve +// to exactly the expected controller pod. +func expectServiceRoutesToPod(g Gomega, serviceName, expectedPod string) { + GinkgoHelper() + + output, err := kubectlRunInNamespace( + namespace, + "get", + "endpoints", + serviceName, + "-o", + "jsonpath={.subsets[*].addresses[*].targetRef.name}", + ) + g.Expect(err).NotTo(HaveOccurred(), "Failed to get %s endpoints", serviceName) + + podSet := map[string]struct{}{} + for _, line := range utils.GetNonEmptyLines(output) { + if strings.HasPrefix(line, "Warning:") || strings.Contains(line, "deprecated") { + continue + } + podSet[line] = struct{}{} + } + + podNames := make([]string, 0, len(podSet)) + for podName := range podSet { + podNames = append(podNames, podName) + } + + g.Expect(podNames).To(HaveLen(1), "%s should route to exactly 1 pod", serviceName) + g.Expect(podNames[0]).To(Equal(expectedPod), "%s should route to the controller pod", serviceName) +} + func dumpFailureDiagnostics() { if !CurrentSpecReport().Failed() { return diff --git a/test/e2e/icecream.go b/test/e2e/icecream.go new file mode 100644 index 000000000..f4ccd1c49 --- /dev/null +++ b/test/e2e/icecream.go @@ -0,0 +1,61 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +// Per-file IceCreamOrder CRD groups. Every e2e file that installs the +// IceCreamOrder CRD owns its own API group so the three files no longer share +// cluster-scoped state and can run on parallel Ginkgo processes (Phase 2.5 of +// docs/design/e2e-speedup-plan.md). The kind, plural and singular stay the same +// across all three; only the group differs. Because the plural ("icecreamorders") +// is shared, kubectl resource references MUST be qualified as +// "icecreamorders." (see iceCreamCRDName) to avoid ambiguity when more +// than one of these CRDs is installed concurrently. +const ( + crdGroupCRDLifecycle = "crd-lifecycle.e2e.example.com" + crdGroupRestartSnapshot = "restart-snapshot.e2e.example.com" + crdGroupBiDirectional = "bi-directional.e2e.example.com" +) + +// iceCreamCRDName returns the fully-qualified CRD name for the given group. This +// doubles as the unambiguous kubectl resource selector (e.g. +// "icecreamorders.crd-lifecycle.e2e.example.com"). +func iceCreamCRDName(group string) string { + return "icecreamorders." + group +} + +// iceCreamCRDMirrorFile returns the filename gitops-reverser writes for the CRD +// under apiextensions.k8s.io/v1/customresourcedefinitions/. +func iceCreamCRDMirrorFile(group string) string { + return iceCreamCRDName(group) + ".yaml" +} + +// iceCreamInstanceDir returns the mirror path prefix for IceCreamOrder instances +// of the given group: "/v1/icecreamorders". +func iceCreamInstanceDir(group string) string { + return group + "/v1/icecreamorders" +} + +// applyIceCreamCRD renders and applies the IceCreamOrder CRD for the given group. +func applyIceCreamCRD(group string) error { + return applyFromTemplate( + "test/e2e/templates/icecreamorder-crd.tmpl", + struct{ Group string }{Group: group}, + "", + ) +} diff --git a/test/e2e/image_refresh_test.go b/test/e2e/image_refresh_test.go index ac309fe17..026dda39b 100644 --- a/test/e2e/image_refresh_test.go +++ b/test/e2e/image_refresh_test.go @@ -40,7 +40,10 @@ import ( // or via: // // task test-image-refresh -var _ = Describe("image refresh dependency chain", Label("image-refresh"), Ordered, func() { +// +// Serial: redeploys / changes the controller image, perturbing the shared +// controller for any concurrent spec. See docs/design/e2e-serial-registry.md. +var _ = Describe("image refresh dependency chain", Label("image-refresh"), Serial, Ordered, func() { var ( projectDir string deployedStamp string diff --git a/test/e2e/repo_assertions_test.go b/test/e2e/repo_assertions_test.go index 1fbf81dae..ec53cfeba 100644 --- a/test/e2e/repo_assertions_test.go +++ b/test/e2e/repo_assertions_test.go @@ -50,17 +50,6 @@ func pullLatestRepoState(g Gomega, checkoutDir string) { } } -func assertLatestCommitTouchesOnly(g Gomega, checkoutDir string, expectedPaths []string) { - GinkgoHelper() - - actualPaths := commitTouchedPaths(g, checkoutDir, "HEAD") - expected := append([]string(nil), expectedPaths...) - sort.Strings(expected) - - g.Expect(actualPaths).To(Equal(expected), - fmt.Sprintf("Latest commit should touch only expected paths. Actual: %v", actualPaths)) -} - func assertLatestCommitForPathTouchesOnlyWithOptional( g Gomega, checkoutDir string, diff --git a/test/e2e/repo_setup.go b/test/e2e/repo_setup.go index 2e31fabe5..032c8f9b2 100644 --- a/test/e2e/repo_setup.go +++ b/test/e2e/repo_setup.go @@ -70,10 +70,10 @@ func bootstrapRepoArtifacts(namespace, repoName string) (*RepoArtifacts, error) ctx, cancel := gitea.Context() defer cancel() - if _, err := gitea.Client().EnsureOrg(ctx, gitea.Org, "Test Organization", "E2E Test Organization"); err != nil { - return nil, fmt.Errorf("ensure Gitea org %q: %w", gitea.Org, err) - } - + // The shared org is created once in SynchronizedBeforeSuite (process #1) + // before any spec runs, so it always exists here. Creating it per-spec used + // to race a concurrent POST /orgs across the parallel Ginkgo processes. Each + // repo name is unique, so the repo create below cannot collide the same way. if _, err := gitea.Client().EnsureOrgRepo( ctx, gitea.Org, diff --git a/test/e2e/restart_snapshot_e2e_test.go b/test/e2e/restart_snapshot_e2e_test.go index be54ede87..b34129eb1 100644 --- a/test/e2e/restart_snapshot_e2e_test.go +++ b/test/e2e/restart_snapshot_e2e_test.go @@ -45,7 +45,9 @@ import ( // IMPORTANT: the wildcard rule is what makes this test meaningful. A test that // watches with a concrete `apiVersions: ["v1"]` rule passes even with the bug // present — which is exactly why the existing e2e suite never caught this. -var _ = Describe("Restart Snapshot Safety", Label("restart-snapshot", "smoke"), Ordered, func() { +// Serial: rolls the controller deployment, which disrupts any spec running +// concurrently on another process. See docs/design/e2e-serial-registry.md. +var _ = Describe("Restart Snapshot Safety", Label("restart-snapshot", "smoke"), Serial, Ordered, func() { var ( testNs string restartRepo *RepoArtifacts @@ -64,6 +66,10 @@ var _ = Describe("Restart Snapshot Safety", Label("restart-snapshot", "smoke"), orderNames := []string{"restart-order-alpha", "restart-order-bravo", "restart-order-charlie"} BeforeAll(func() { + By("setting up the Prometheus client for drain-signal metrics") + setupPrometheusClient() + verifyPrometheusAvailable() + By("creating the restart-snapshot test namespace") testNs = testNamespaceFor("restart-snapshot") _, _ = kubectlRun("create", "namespace", testNs) // idempotent; ignore AlreadyExists @@ -92,11 +98,11 @@ var _ = Describe("Restart Snapshot Safety", Label("restart-snapshot", "smoke"), It("keeps the git mirror intact when the controller restarts", func() { By("installing the IceCreamOrder CRD") - _, err := kubectlRun("apply", "-f", "test/e2e/templates/icecreamorder-crd.yaml") + err := applyIceCreamCRD(crdGroupRestartSnapshot) Expect(err).NotTo(HaveOccurred(), "failed to install IceCreamOrder CRD") Eventually(func(g Gomega) { output, getErr := kubectlRun( - "get", "crd", "icecreamorders.shop.example.com", + "get", "crd", iceCreamCRDName(crdGroupRestartSnapshot), "-o", "jsonpath={.status.conditions[?(@.type=='Established')].status}", ) g.Expect(getErr).NotTo(HaveOccurred()) @@ -111,10 +117,12 @@ var _ = Describe("Restart Snapshot Safety", Label("restart-snapshot", "smoke"), Name string DestinationName string Namespace string + Group string }{ Name: clusterWatchRuleName, DestinationName: gitTargetName, Namespace: testNs, + Group: crdGroupRestartSnapshot, } Expect(applyFromTemplate( "test/e2e/templates/restart/clusterwatchrule-wildcard.tmpl", cwrData, "", @@ -125,13 +133,13 @@ var _ = Describe("Restart Snapshot Safety", Label("restart-snapshot", "smoke"), By("creating quiet IceCreamOrder resources to build up the git mirror") for _, name := range orderNames { - createIceCreamOrder(testNs, name) + createIceCreamOrder(crdGroupRestartSnapshot, testNs, name) } expectedFiles := make([]string, 0, len(orderNames)) for _, name := range orderNames { expectedFiles = append(expectedFiles, filepath.Join( - gitTargetPath, "shop.example.com", "v1", "icecreamorders", testNs, name+".yaml", + gitTargetPath, iceCreamInstanceDir(crdGroupRestartSnapshot), testNs, name+".yaml", )) } @@ -150,10 +158,57 @@ var _ = Describe("Restart Snapshot Safety", Label("restart-snapshot", "smoke"), By("restarting the controller deployment (plain rollout restart)") restartControllerDeployment() - // The destructive snapshot commit lands within seconds of the new pod - // starting. A quiet order, once wiped, never comes back — so once the - // mirror is intact for a sustained window after the restart settles, it - // will stay intact. + // Anchor the reconcile gate to the NEW pod. restartControllerDeployment + // blocks until the rollout completes, so exactly one non-terminating + // controller pod remains — the freshly started one. controllerPodNames + // already filters out pods with a deletionTimestamp. + var newControllerPod string + Eventually(func(g Gomega) { + pods, podErr := controllerPodNames() + g.Expect(podErr).NotTo(HaveOccurred()) + g.Expect(pods).To(HaveLen(1), "expected exactly one non-terminating controller pod after rollout") + newControllerPod = pods[0] + }, 30*time.Second, 2*time.Second).Should(Succeed()) + By(fmt.Sprintf("new controller pod after restart: %s", newControllerPod)) + + // Phase 3 drain signals replace a 75 s blind wait. The reconcile counter + // is scoped to the new pod via its `pod` target label: a counter resets to + // 0 on a fresh pod, so `{pod=""} > 0` proves the new pod completed its + // own post-restart snapshot reconcile. A sum() over a pre-restart baseline + // cannot prove this — once Prometheus marks the old pod's series stale, the + // new pod's first increment can bring the cross-pod sum back to the old + // total rather than above it, so `> baseline` could never pass. + // branch_worker_queue_depth returning to 0 then confirms that submission + // has been committed and pushed — the exact moment any destructive commit + // would have landed — instead of guessing with a fixed sleep. + By("waiting for the new pod to complete its post-restart reconcile") + waitForMetricWithTimeout( + fmt.Sprintf( + `sum(gitopsreverser_target_reconcile_completed_total`+ + `{gittarget_namespace=%q,gittarget_name=%q,pod=%q}) or vector(0)`, + testNs, gitTargetName, newControllerPod, + ), + func(v float64) bool { return v > 0 }, + "GitTarget reconcile completed on the new controller pod", + 90*time.Second, + ) + + By("waiting for the branch worker queue to drain") + waitForMetricWithTimeout( + fmt.Sprintf( + `sum(gitopsreverser_branch_worker_queue_depth`+ + `{provider_namespace=%q,provider_name=%q,branch="main",pod=%q}) or vector(0)`, + testNs, providerName, newControllerPod, + ), + func(v float64) bool { return v == 0 }, + "branch worker queue drained after restart", + 90*time.Second, + ) + + // A short stability window guards against a delayed second reconcile and + // against any Prometheus cross-restart sample staleness in the gates + // above: a quiet order, once wiped, never comes back, so a mirror that + // stays intact for this window after the drain will stay intact. By("verifying the git mirror is NOT wiped by the restart") Consistently(func(g Gomega) { pullLatestRepoState(g, restartRepo.CheckoutDir) @@ -162,7 +217,7 @@ var _ = Describe("Restart Snapshot Safety", Label("restart-snapshot", "smoke"), g.Expect(statErr).NotTo(HaveOccurred(), "file %q disappeared after the controller restart — startup snapshot wiped the mirror", relPath) } - }, 75*time.Second, 5*time.Second).Should(Succeed()) + }, 15*time.Second, 5*time.Second).Should(Succeed()) By("confirming no commit since the restart deleted tracked files") deletions, logErr := gitRun( @@ -176,10 +231,11 @@ var _ = Describe("Restart Snapshot Safety", Label("restart-snapshot", "smoke"), }) }) -// createIceCreamOrder applies a minimal IceCreamOrder custom resource. -func createIceCreamOrder(ns, name string) { +// createIceCreamOrder applies a minimal IceCreamOrder custom resource for the +// given CRD group. +func createIceCreamOrder(group, ns, name string) { By(fmt.Sprintf("creating IceCreamOrder '%s/%s'", ns, name)) - manifest := fmt.Sprintf(`apiVersion: shop.example.com/v1 + manifest := fmt.Sprintf(`apiVersion: %s/v1 kind: IceCreamOrder metadata: name: %s @@ -190,7 +246,7 @@ spec: scoops: - flavor: Vanilla quantity: 1 -`, name, ns, name) +`, group, name, ns, name) _, err := kubectlRunWithStdin(ns, manifest, "apply", "-f", "-") Expect(err).NotTo(HaveOccurred(), "failed to apply IceCreamOrder %s/%s", ns, name) } diff --git a/test/e2e/setup/flux/releases/ingress.yaml b/test/e2e/setup/demo-only/flux/ingress.yaml similarity index 100% rename from test/e2e/setup/flux/releases/ingress.yaml rename to test/e2e/setup/demo-only/flux/ingress.yaml diff --git a/test/e2e/setup/flux/releases/kro.yaml b/test/e2e/setup/demo-only/flux/kro.yaml similarity index 100% rename from test/e2e/setup/flux/releases/kro.yaml rename to test/e2e/setup/demo-only/flux/kro.yaml diff --git a/test/e2e/setup/demo-only/flux/kustomization.yaml b/test/e2e/setup/demo-only/flux/kustomization.yaml new file mode 100644 index 000000000..da52112be --- /dev/null +++ b/test/e2e/setup/demo-only/flux/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Traefik and kro are only needed by the demo-only flows (podinfo IngressRoutes +# and kro ResourceGraphDefinitions). They are applied and waited on by the +# _demo-ready task, kept out of the core bring-up (test/e2e/setup/flux) so a +# slow/failed install never blocks the measured e2e suites. Both charts default +# to rendering a ServiceMonitor; keeping them here also means the core path no +# longer races the prometheus-operator CRD install. +resources: + - ingress.yaml + - kro.yaml diff --git a/test/e2e/setup/flux/kustomization.yaml b/test/e2e/setup/flux/kustomization.yaml index 72d222841..4ee832b86 100644 --- a/test/e2e/setup/flux/kustomization.yaml +++ b/test/e2e/setup/flux/kustomization.yaml @@ -12,12 +12,10 @@ resources: - repositories/jetstack.yaml - repositories/gitea-charts.yaml - repositories/valkey.yaml - - releases/ingress.yaml - releases/prometheus-operator.yaml - releases/cert-manager.yaml - releases/gitea.yaml - releases/valkey.yaml - - releases/kro.yaml - releases/reflector.yaml generatorOptions: diff --git a/test/e2e/setup/flux/releases/aggregated-api.yaml b/test/e2e/setup/flux/releases/aggregated-api.yaml index c0ad69406..6494add62 100644 --- a/test/e2e/setup/flux/releases/aggregated-api.yaml +++ b/test/e2e/setup/flux/releases/aggregated-api.yaml @@ -10,7 +10,7 @@ spec: mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip operation: copy ref: - semver: 0.4.0 + semver: 0.6.0 --- apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease @@ -20,6 +20,16 @@ metadata: spec: interval: 60m timeout: 10m + # This release renders cert-manager Issuer/Certificate resources (tls.mode: + # cert-manager below), so it must not install until cert-manager's CRDs and + # webhook are ready. Without this, kubectl apply -k fires both releases in + # parallel and aggregated-api's helm install races cert-manager: if the CRDs + # aren't established yet the install fails, exhausts its 3 remediation + # retries, and the release stays not-Ready until the next 60m interval — + # which times out wait-flux-services.sh on helmreleases/aggregated-api. + dependsOn: + - name: cert-manager + namespace: cert-manager chartRef: kind: OCIRepository name: apiservice-audit-proxy diff --git a/test/e2e/setup/flux/releases/prometheus-operator.yaml b/test/e2e/setup/flux/releases/prometheus-operator.yaml index 880c732df..dd393f382 100644 --- a/test/e2e/setup/flux/releases/prometheus-operator.yaml +++ b/test/e2e/setup/flux/releases/prometheus-operator.yaml @@ -5,6 +5,13 @@ metadata: namespace: flux-system spec: interval: 30m + # The monitoring.coreos.com CRDs have very large schemas and are slow to reach + # Established; with wait:true the first reconcile can time out finding them + # 'InProgress'. Without retryInterval the next attempt is a full interval (30m) + # away — longer than the bring-up budget — so a transient establishment lag + # becomes a hard failure (wait-flux-services.sh times out on this Kustomization). + # Retry quickly so the CRDs settle within the bring-up window instead. + retryInterval: 20s timeout: 5m path: ./ prune: true diff --git a/test/e2e/signing_e2e_test.go b/test/e2e/signing_e2e_test.go index 73cc95060..8827f71b8 100644 --- a/test/e2e/signing_e2e_test.go +++ b/test/e2e/signing_e2e_test.go @@ -184,7 +184,7 @@ var _ = Describe("Commit Signing", Label("signing"), Ordered, func() { g.Expect(rawObj).To(ContainSubstring("-----BEGIN SSH SIGNATURE-----"), "commit should carry an SSH signature") commitHash = h - }, "60s", "3s").Should(Succeed()) + }, "90s", "3s").Should(Succeed()) By("verifying the commit locally with ssh-keygen and git verify-commit") assertLocalSSHVerification(signingRepo.CheckoutDir, commitHash, signingPublicKey, committerEmail) @@ -306,7 +306,7 @@ var _ = Describe("Commit Signing", Label("signing"), Ordered, func() { g.Expect(rawErr).NotTo(HaveOccurred()) g.Expect(rawObj).To(ContainSubstring("-----BEGIN SSH SIGNATURE-----")) commitHash = h - }, "60s", "3s").Should(Succeed()) + }, "90s", "3s").Should(Succeed()) By("verifying the BYOK commit locally") assertLocalSSHVerification(signingRepo.CheckoutDir, commitHash, publicKeyStr, committerEmail) @@ -387,7 +387,10 @@ var _ = Describe("Commit Signing", Label("signing"), Ordered, func() { g.Expect(parts[1]).To(Equal(customEmail), "committer email should match configured value") g.Expect(parts[2]).To(HavePrefix("e2e:"), "commit subject should use custom template prefix") g.Expect(parts[2]).To(ContainSubstring("configmaps"), "commit subject should include resource type") - }, "60s", "3s").Should(Succeed()) + // 90s: signing does the slowest per-event work (commit + SSH sign + + // push); under Ginkgo parallelism the shared controller can push this + // past 60s on a contended run. + }, "90s", "3s").Should(Succeed()) }) // ── Test 4: batch/atomic commit uses batch message template ───────────── diff --git a/test/e2e/templates/icecreamorder-crd.yaml b/test/e2e/templates/icecreamorder-crd.tmpl similarity index 98% rename from test/e2e/templates/icecreamorder-crd.yaml rename to test/e2e/templates/icecreamorder-crd.tmpl index 4d87345e6..1f4b2ad86 100644 --- a/test/e2e/templates/icecreamorder-crd.yaml +++ b/test/e2e/templates/icecreamorder-crd.tmpl @@ -2,10 +2,10 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: # name must match the spec fields below, and be in the form: . - name: icecreamorders.shop.example.com + name: icecreamorders.{{ .Group }} spec: # group name to use for REST API: /apis// - group: shop.example.com + group: {{ .Group }} # list of versions supported by this CRD versions: - name: v1 diff --git a/test/e2e/templates/icecreamorder-instance.tmpl b/test/e2e/templates/icecreamorder-instance.tmpl index 2cb5789d7..9e65af3b6 100644 --- a/test/e2e/templates/icecreamorder-instance.tmpl +++ b/test/e2e/templates/icecreamorder-instance.tmpl @@ -1,4 +1,4 @@ -apiVersion: shop.example.com/v1 +apiVersion: {{ .Group }}/v1 kind: IceCreamOrder metadata: name: {{ .Name }} diff --git a/test/e2e/templates/manager/watchrule-resources.tmpl b/test/e2e/templates/manager/watchrule-resources.tmpl new file mode 100644 index 000000000..6715a2170 --- /dev/null +++ b/test/e2e/templates/manager/watchrule-resources.tmpl @@ -0,0 +1,11 @@ +apiVersion: configbutler.ai/v1alpha1 +kind: WatchRule +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +spec: + targetRef: + kind: GitTarget + name: {{.DestinationName}} + rules: + - resources: [{{.Resources}}] diff --git a/test/e2e/templates/restart/clusterwatchrule-wildcard.tmpl b/test/e2e/templates/restart/clusterwatchrule-wildcard.tmpl index aec9ab522..41b12e82f 100644 --- a/test/e2e/templates/restart/clusterwatchrule-wildcard.tmpl +++ b/test/e2e/templates/restart/clusterwatchrule-wildcard.tmpl @@ -13,6 +13,6 @@ spec: # restart snapshots an empty cluster and deletes the whole tracked tree. - scope: Namespaced operations: [CREATE, UPDATE, DELETE] - apiGroups: ["shop.example.com"] + apiGroups: ["{{ .Group }}"] apiVersions: ["*"] resources: ["icecreamorders"] diff --git a/test/e2e/templates/watchrule-crd.tmpl b/test/e2e/templates/watchrule-crd.tmpl index 5d4a85e2e..939c8e264 100644 --- a/test/e2e/templates/watchrule-crd.tmpl +++ b/test/e2e/templates/watchrule-crd.tmpl @@ -7,6 +7,6 @@ spec: targetRef: name: {{ .DestinationName }} rules: - - apiGroups: ["shop.example.com"] + - apiGroups: ["{{ .Group }}"] apiVersions: ["v1"] resources: ["icecreamorders"] diff --git a/test/e2e/tools/ginkgo-allure/main.go b/test/e2e/tools/ginkgo-allure/main.go new file mode 100644 index 000000000..f8815136b --- /dev/null +++ b/test/e2e/tools/ginkgo-allure/main.go @@ -0,0 +1,329 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// ginkgo-allure converts Ginkgo JSON reports into Allure result files. +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/onsi/ginkgo/v2/types" +) + +const ( + exitUsage = 2 + usageInstruct = "usage: ginkgo-allure --output-dir ..." +) + +func main() { + os.Exit(mainRun(os.Args[1:], os.Stdout, os.Stderr)) +} + +func mainRun(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("ginkgo-allure", flag.ContinueOnError) + fs.SetOutput(stderr) + outputDir := fs.String("output-dir", "allure-results", "directory for generated Allure result files") + if err := fs.Parse(args); err != nil { + return exitUsage + } + if *outputDir == "" || fs.NArg() == 0 { + fmt.Fprintln(stderr, usageInstruct) + return exitUsage + } + count, err := run(*outputDir, fs.Args()) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + fmt.Fprintf(stdout, "wrote %d Allure result files to %s\n", count, *outputDir) + return 0 +} + +type allureResult struct { + UUID string `json:"uuid"` + HistoryID string `json:"historyId"` + TestCaseID string `json:"testCaseId"` + FullName string `json:"fullName"` + Name string `json:"name"` + Status string `json:"status"` + StatusDetails *allureStatusDetails `json:"statusDetails,omitempty"` + Stage string `json:"stage"` + Start int64 `json:"start,omitempty"` + Stop int64 `json:"stop,omitempty"` + Labels []allureLabel `json:"labels,omitempty"` + Parameters []allureParameter `json:"parameters,omitempty"` + Attachments []allureAttachment `json:"attachments,omitempty"` +} + +type allureStatusDetails struct { + Message string `json:"message,omitempty"` + Trace string `json:"trace,omitempty"` +} + +type allureLabel struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type allureParameter struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type allureAttachment struct { + Name string `json:"name"` + Source string `json:"source"` + Type string `json:"type"` +} + +func run(outputDir string, paths []string) (int, error) { + if err := os.MkdirAll(outputDir, 0o750); err != nil { + return 0, fmt.Errorf("create output dir %s: %w", outputDir, err) + } + + count := 0 + for _, path := range paths { + reports, err := readReports(path) + if err != nil { + return count, err + } + reportCount, err := convertReports(outputDir, path, reports) + if err != nil { + return count, err + } + count += reportCount + } + return count, nil +} + +func convertReports(outputDir, path string, reports []types.Report) (int, error) { + count := 0 + for reportIndex, report := range reports { + if report.SuiteConfig.DryRun { + continue + } + for specIndex, spec := range report.SpecReports { + if spec.LeafNodeType != types.NodeTypeIt || specSkippedByFilter(report, spec) { + continue + } + result := convertSpec(path, reportIndex, specIndex, report, spec) + if err := writeResult(outputDir, result, spec.CombinedOutput()); err != nil { + return count, err + } + count++ + } + } + return count, nil +} + +func readReports(path string) ([]types.Report, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("open %s: %w", path, err) + } + var reports []types.Report + if err := json.Unmarshal(data, &reports); err != nil { + return nil, fmt.Errorf("parse %s: %w", path, err) + } + return reports, nil +} + +func convertSpec(path string, reportIndex, specIndex int, report types.Report, spec types.SpecReport) allureResult { + fullName := spec.FullText() + if fullName == "" { + fullName = spec.LeafNodeText + } + if fullName == "" { + fullName = fmt.Sprintf("%s#%d", filepath.Base(path), specIndex) + } + + start, stop := specTimes(report, spec) + result := allureResult{ + UUID: uuidFromParts(path, reportIndex, specIndex, fullName), + HistoryID: hashString("history", fullName), + TestCaseID: hashString("test-case", fullName), + FullName: fullName, + Name: resultName(spec), + Status: allureStatus(spec.State), + Stage: "finished", + Start: unixMillis(start), + Stop: unixMillis(stop), + Labels: labels(report, spec), + Parameters: []allureParameter{ + {Name: "ginkgo.parallel_process", Value: strconv.Itoa(spec.ParallelProcess)}, + {Name: "ginkgo.random_seed", Value: strconv.FormatInt(report.SuiteConfig.RandomSeed, 10)}, + }, + } + if details := statusDetails(spec); details != nil { + result.StatusDetails = details + } + return result +} + +func writeResult(outputDir string, result allureResult, combinedOutput string) error { + if strings.TrimSpace(combinedOutput) != "" { + source := result.UUID + "-attachment.txt" + if err := os.WriteFile(filepath.Join(outputDir, source), []byte(combinedOutput), 0o600); err != nil { + return fmt.Errorf("write attachment %s: %w", source, err) + } + result.Attachments = append(result.Attachments, allureAttachment{ + Name: "Ginkgo output", + Source: source, + Type: "text/plain", + }) + } + + data, err := json.MarshalIndent(result, "", " ") + if err != nil { + return fmt.Errorf("marshal result %s: %w", result.UUID, err) + } + name := result.UUID + "-result.json" + if err := os.WriteFile(filepath.Join(outputDir, name), append(data, '\n'), 0o600); err != nil { + return fmt.Errorf("write result %s: %w", name, err) + } + return nil +} + +func resultName(spec types.SpecReport) string { + if spec.LeafNodeText != "" { + return spec.LeafNodeText + } + return spec.FullText() +} + +func labels(report types.Report, spec types.SpecReport) []allureLabel { + out := []allureLabel{ + {Name: "language", Value: "go"}, + {Name: "framework", Value: "ginkgo"}, + } + if spec.ParallelProcess > 0 { + out = append(out, allureLabel{Name: "thread", Value: fmt.Sprintf("ginkgo-process-%d", spec.ParallelProcess)}) + } + if report.SuiteDescription != "" { + out = append(out, allureLabel{Name: "suite", Value: report.SuiteDescription}) + } + if report.SuitePath != "" { + out = append(out, allureLabel{Name: "package", Value: filepath.Base(report.SuitePath)}) + } + if len(spec.ContainerHierarchyTexts) > 0 { + out = append(out, allureLabel{Name: "parentSuite", Value: spec.ContainerHierarchyTexts[0]}) + } + if len(spec.ContainerHierarchyTexts) > 1 { + out = append(out, allureLabel{Name: "subSuite", Value: strings.Join(spec.ContainerHierarchyTexts[1:], " ")}) + } + for _, label := range spec.Labels() { + out = append(out, allureLabel{Name: "tag", Value: label}) + } + if spec.IsSerial { + out = append(out, allureLabel{Name: "tag", Value: "Serial"}) + } + return out +} + +func allureStatus(state types.SpecState) string { + switch { + case state == types.SpecStatePassed: + return "passed" + case state == types.SpecStateSkipped || state == types.SpecStatePending: + return "skipped" + case state == types.SpecStateFailed: + return "failed" + case state.Is(types.SpecStateAborted | types.SpecStatePanicked | types.SpecStateInterrupted | types.SpecStateTimedout): + return "broken" + default: + return "unknown" + } +} + +func specSkippedByFilter(report types.Report, spec types.SpecReport) bool { + return report.SuiteConfig.LabelFilter != "" && + spec.State == types.SpecStateSkipped && + spec.EndTime.IsZero() && + spec.RunTime == 0 && + spec.Failure.IsZero() +} + +func statusDetails(spec types.SpecReport) *allureStatusDetails { + if spec.Failure.IsZero() { + return nil + } + trace := spec.Failure.Location.FullStackTrace + if trace == "" && spec.Failure.Location != (types.CodeLocation{}) { + trace = spec.Failure.Location.String() + } + if spec.Failure.ForwardedPanic != "" { + if trace != "" { + trace += "\n" + } + trace += spec.Failure.ForwardedPanic + } + return &allureStatusDetails{ + Message: spec.Failure.Message, + Trace: trace, + } +} + +func specTimes(report types.Report, spec types.SpecReport) (time.Time, time.Time) { + start := spec.StartTime + if start.IsZero() { + start = report.StartTime + } + stop := spec.EndTime + if stop.IsZero() && !start.IsZero() { + stop = start.Add(spec.RunTime) + } + return start, stop +} + +func unixMillis(t time.Time) int64 { + if t.IsZero() { + return 0 + } + return t.UnixNano() / int64(time.Millisecond) +} + +func uuidFromParts(parts ...any) string { + hash := hashParts(parts...) + hexed := hex.EncodeToString(hash[:16]) + return fmt.Sprintf("%s-%s-%s-%s-%s", hexed[0:8], hexed[8:12], hexed[12:16], hexed[16:20], hexed[20:32]) +} + +func hashString(parts ...any) string { + hash := hashParts(parts...) + return hex.EncodeToString(hash[:16]) +} + +func hashParts(parts ...any) [32]byte { + h := sha256.New() + for _, part := range parts { + _, _ = fmt.Fprintf(h, "%v\x00", part) + } + var out [32]byte + copy(out[:], h.Sum(nil)) + return out +} diff --git a/test/e2e/tools/ginkgo-allure/main_test.go b/test/e2e/tools/ginkgo-allure/main_test.go new file mode 100644 index 000000000..c7cf8c34b --- /dev/null +++ b/test/e2e/tools/ginkgo-allure/main_test.go @@ -0,0 +1,437 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/onsi/ginkgo/v2/types" +) + +func fixtureReport() []types.Report { + start := time.Date(2026, 5, 31, 12, 0, 0, 0, time.UTC) + return []types.Report{{ + SuitePath: "/workspaces/gitops-reverser/test/e2e", + SuiteDescription: "E2E Suite", + StartTime: start, + SuiteConfig: types.SuiteConfig{ + RandomSeed: 12345, + }, + SpecReports: []types.SpecReport{ + { + LeafNodeType: types.NodeTypeBeforeSuite, + LeafNodeText: "BeforeSuite", + }, + { + ContainerHierarchyTexts: []string{"Manager", "Smoke"}, + ContainerHierarchyLabels: [][]string{{"manager"}}, + LeafNodeType: types.NodeTypeIt, + LeafNodeText: "passes", + LeafNodeLabels: []string{"smoke"}, + State: types.SpecStatePassed, + StartTime: start, + EndTime: start.Add(2 * time.Second), + RunTime: 2 * time.Second, + ParallelProcess: 2, + CapturedGinkgoWriterOutput: "useful output\n", + }, + { + ContainerHierarchyTexts: []string{"Manager"}, + LeafNodeType: types.NodeTypeIt, + LeafNodeText: "fails", + State: types.SpecStateFailed, + StartTime: start.Add(3 * time.Second), + RunTime: time.Second, + Failure: types.Failure{ + Message: "expected success", + Location: types.CodeLocation{ + FileName: "test/e2e/example_test.go", + LineNumber: 42, + FullStackTrace: "stack trace", + }, + }, + }, + { + ContainerHierarchyTexts: []string{"Manager"}, + LeafNodeType: types.NodeTypeIt, + LeafNodeText: "times out", + State: types.SpecStateTimedout, + }, + { + ContainerHierarchyTexts: []string{"Manager"}, + LeafNodeType: types.NodeTypeIt, + LeafNodeText: "skips", + State: types.SpecStateSkipped, + }, + }, + }} +} + +func TestRun_ConvertsGinkgoJSONToAllureResults(t *testing.T) { + reportPath := writeFixtureReport(t, fixtureReport()) + outputDir := filepath.Join(t.TempDir(), "allure-results") + + count, err := run(outputDir, []string{reportPath}) + if err != nil { + t.Fatalf("run: %v", err) + } + if count != 4 { + t.Fatalf("count: got %d, want 4", count) + } + + results := readAllureResults(t, outputDir) + if len(results) != 4 { + t.Fatalf("results: got %d, want 4", len(results)) + } + + assertPassingResult(t, outputDir, findResult(t, results, "passes")) + assertFailedResult(t, findResult(t, results, "fails")) + + timedOut := findResult(t, results, "times out") + if timedOut.Status != "broken" { + t.Errorf("timed out status: got %q", timedOut.Status) + } + + skipped := findResult(t, results, "skips") + if skipped.Status != "skipped" { + t.Errorf("skipped status: got %q", skipped.Status) + } +} + +func assertPassingResult(t *testing.T, outputDir string, result allureResult) { + t.Helper() + if result.Status != "passed" { + t.Errorf("passed status: got %q", result.Status) + } + if result.FullName != "Manager Smoke passes" { + t.Errorf("fullName: got %q", result.FullName) + } + assertHasLabel(t, result, "framework", "ginkgo") + assertHasLabel(t, result, "language", "go") + assertHasLabel(t, result, "thread", "ginkgo-process-2") + assertHasLabel(t, result, "suite", "E2E Suite") + assertHasLabel(t, result, "package", "e2e") + assertHasLabel(t, result, "parentSuite", "Manager") + assertHasLabel(t, result, "subSuite", "Smoke") + assertHasLabel(t, result, "tag", "manager") + assertHasLabel(t, result, "tag", "smoke") + assertHasParameter(t, result, "ginkgo.parallel_process", "2") + assertHasParameter(t, result, "ginkgo.random_seed", "12345") + if result.Start == 0 || result.Stop == 0 || result.Stop <= result.Start { + t.Errorf("bad timing: start=%d stop=%d", result.Start, result.Stop) + } + if len(result.Attachments) != 1 { + t.Fatalf("attachments: got %d, want 1", len(result.Attachments)) + } + attachment, err := os.ReadFile(filepath.Join(outputDir, result.Attachments[0].Source)) + if err != nil { + t.Fatalf("read attachment: %v", err) + } + if string(attachment) != "useful output\n" { + t.Errorf("attachment: got %q", string(attachment)) + } +} + +func assertFailedResult(t *testing.T, result allureResult) { + t.Helper() + if result.Status != "failed" { + t.Errorf("failed status: got %q", result.Status) + } + if result.StatusDetails == nil || result.StatusDetails.Message != "expected success" { + t.Fatalf("status details missing failure message: %#v", result.StatusDetails) + } + if result.StatusDetails.Trace != "stack trace" { + t.Errorf("trace: got %q", result.StatusDetails.Trace) + } +} + +func TestMainRun_ExitCodes(t *testing.T) { + good := writeFixtureReport(t, fixtureReport()) + cases := []struct { + name string + args []string + wantCode int + }{ + {name: "no args", args: nil, wantCode: exitUsage}, + {name: "empty output dir", args: []string{"--output-dir", "", good}, wantCode: exitUsage}, + {name: "bad flag", args: []string{"-not-a-flag"}, wantCode: exitUsage}, + {name: "missing report", args: []string{"--output-dir", t.TempDir(), "/no/such/file"}, wantCode: 1}, + {name: "success", args: []string{"--output-dir", t.TempDir(), good}, wantCode: 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + got := mainRun(tc.args, &stdout, &stderr) + if got != tc.wantCode { + t.Errorf("mainRun: code=%d, want %d (stderr=%s)", got, tc.wantCode, stderr.String()) + } + }) + } +} + +func TestRun_BadJSON(t *testing.T) { + path := filepath.Join(t.TempDir(), "bad.json") + if err := os.WriteFile(path, []byte("not json"), 0o600); err != nil { + t.Fatalf("write bad fixture: %v", err) + } + _, err := run(t.TempDir(), []string{path}) + if err == nil { + t.Fatal("expected invalid JSON error") + } +} + +func TestRun_IgnoresDryRunReports(t *testing.T) { + reports := fixtureReport() + reports[0].SuiteConfig.DryRun = true + reportPath := writeFixtureReport(t, reports) + outputDir := filepath.Join(t.TempDir(), "allure-results") + + count, err := run(outputDir, []string{reportPath}) + if err != nil { + t.Fatalf("run: %v", err) + } + if count != 0 { + t.Fatalf("count: got %d, want 0", count) + } + if results := readAllureResults(t, outputDir); len(results) != 0 { + t.Fatalf("results: got %d, want 0", len(results)) + } +} + +func TestRun_IgnoresSpecsSkippedByLabelFilter(t *testing.T) { + reports := fixtureReport() + reports[0].SuiteConfig.LabelFilter = "smoke" + reports[0].SpecReports = []types.SpecReport{ + { + ContainerHierarchyTexts: []string{"Manager"}, + LeafNodeType: types.NodeTypeIt, + LeafNodeText: "runs", + State: types.SpecStatePassed, + StartTime: reports[0].StartTime, + EndTime: reports[0].StartTime.Add(time.Second), + RunTime: time.Second, + ParallelProcess: 1, + }, + { + ContainerHierarchyTexts: []string{"Manager"}, + LeafNodeType: types.NodeTypeIt, + LeafNodeText: "filtered out", + State: types.SpecStateSkipped, + StartTime: reports[0].StartTime.Add(time.Second), + ParallelProcess: 1, + }, + { + ContainerHierarchyTexts: []string{"Manager"}, + LeafNodeType: types.NodeTypeIt, + LeafNodeText: "skips intentionally", + State: types.SpecStateSkipped, + StartTime: reports[0].StartTime.Add(2 * time.Second), + Failure: types.Failure{ + Message: "operator skipped", + }, + ParallelProcess: 1, + }, + } + reportPath := writeFixtureReport(t, reports) + outputDir := filepath.Join(t.TempDir(), "allure-results") + + count, err := run(outputDir, []string{reportPath}) + if err != nil { + t.Fatalf("run: %v", err) + } + if count != 2 { + t.Fatalf("count: got %d, want 2", count) + } + results := readAllureResults(t, outputDir) + if len(results) != 2 { + t.Fatalf("results: got %d, want 2", len(results)) + } + findResult(t, results, "runs") + findResult(t, results, "skips intentionally") + for _, result := range results { + if result.Name == "filtered out" { + t.Fatal("filtered-out spec was converted") + } + } +} + +func TestRun_OutputDirCreateError(t *testing.T) { + reportPath := writeFixtureReport(t, fixtureReport()) + outputDir := filepath.Join(t.TempDir(), "not-a-dir") + if err := os.WriteFile(outputDir, []byte("file"), 0o600); err != nil { + t.Fatalf("write output placeholder: %v", err) + } + _, err := run(outputDir, []string{reportPath}) + if err == nil { + t.Fatal("expected output dir creation error") + } +} + +func TestConvertSpec_FallbacksAndFailureDetails(t *testing.T) { + spec := types.SpecReport{ + LeafNodeType: types.NodeTypeIt, + State: types.SpecStatePanicked, + Failure: types.Failure{ + Message: "boom", + Location: types.CodeLocation{ + FileName: "test/e2e/example_test.go", + LineNumber: 42, + }, + ForwardedPanic: "panic payload", + }, + } + + result := convertSpec("/tmp/ginkgo-report-empty.json", 0, 7, types.Report{}, spec) + + if result.FullName != "ginkgo-report-empty.json#7" { + t.Errorf("fullName: got %q", result.FullName) + } + if result.Name != "" { + t.Errorf("name: got %q, want empty fallback", result.Name) + } + if result.Status != "broken" { + t.Errorf("status: got %q", result.Status) + } + if result.Start != 0 || result.Stop != 0 { + t.Errorf("timing: start=%d stop=%d, want zero values", result.Start, result.Stop) + } + if result.StatusDetails == nil { + t.Fatal("missing status details") + } + if result.StatusDetails.Message != "boom" { + t.Errorf("message: got %q", result.StatusDetails.Message) + } + if result.StatusDetails.Trace != "test/e2e/example_test.go:42\npanic payload" { + t.Errorf("trace: got %q", result.StatusDetails.Trace) + } +} + +func TestWriteResult_NoAttachment(t *testing.T) { + outputDir := t.TempDir() + result := allureResult{ + UUID: "no-attachment", + Name: "passes", + Status: "passed", + Stage: "finished", + } + if err := writeResult(outputDir, result, " \n\t"); err != nil { + t.Fatalf("writeResult: %v", err) + } + + results := readAllureResults(t, outputDir) + if len(results) != 1 { + t.Fatalf("results: got %d, want 1", len(results)) + } + if len(results[0].Attachments) != 0 { + t.Errorf("attachments: got %#v, want none", results[0].Attachments) + } +} + +func TestStatusMappings(t *testing.T) { + cases := map[types.SpecState]string{ + types.SpecStatePassed: "passed", + types.SpecStateSkipped: "skipped", + types.SpecStatePending: "skipped", + types.SpecStateFailed: "failed", + types.SpecStatePanicked: "broken", + types.SpecStateAborted: "broken", + types.SpecStateInterrupted: "broken", + types.SpecStateTimedout: "broken", + types.SpecStateInvalid: "unknown", + } + for state, want := range cases { + if got := allureStatus(state); got != want { + t.Errorf("allureStatus(%s) = %q, want %q", state, got, want) + } + } +} + +func writeFixtureReport(t *testing.T, reports []types.Report) string { + t.Helper() + data, err := json.Marshal(reports) + if err != nil { + t.Fatalf("marshal report: %v", err) + } + path := filepath.Join(t.TempDir(), "ginkgo-report.json") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write report: %v", err) + } + return path +} + +func readAllureResults(t *testing.T, outputDir string) []allureResult { + t.Helper() + entries, err := os.ReadDir(outputDir) + if err != nil { + t.Fatalf("read output dir: %v", err) + } + results := make([]allureResult, 0, len(entries)) + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), "-result.json") { + continue + } + data, err := os.ReadFile(filepath.Join(outputDir, entry.Name())) + if err != nil { + t.Fatalf("read result: %v", err) + } + var result allureResult + if err := json.Unmarshal(data, &result); err != nil { + t.Fatalf("parse result: %v", err) + } + results = append(results, result) + } + return results +} + +func findResult(t *testing.T, results []allureResult, name string) allureResult { + t.Helper() + for _, result := range results { + if result.Name == name { + return result + } + } + t.Fatalf("missing result %q in %#v", name, results) + return allureResult{} +} + +func assertHasLabel(t *testing.T, result allureResult, name, value string) { + t.Helper() + for _, label := range result.Labels { + if label.Name == name && label.Value == value { + return + } + } + t.Errorf("missing label %s=%s in %#v", name, value, result.Labels) +} + +func assertHasParameter(t *testing.T, result allureResult, name, value string) { + t.Helper() + for _, parameter := range result.Parameters { + if parameter.Name == name && parameter.Value == value { + return + } + } + t.Errorf("missing parameter %s=%s in %#v", name, value, result.Parameters) +} diff --git a/test/e2e/tools/spec-timings/main.go b/test/e2e/tools/spec-timings/main.go new file mode 100644 index 000000000..7b0f839ac --- /dev/null +++ b/test/e2e/tools/spec-timings/main.go @@ -0,0 +1,139 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// spec-timings reads a Ginkgo JSON report and prints a duration-sorted +// table of specs with each spec's share of the total wallclock. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "os" + "sort" + "strings" + "time" + + "github.com/onsi/ginkgo/v2/types" +) + +const ( + exitUsage = 2 + percentScalar = 100.0 + usageInstruct = "usage: spec-timings " +) + +func main() { + os.Exit(mainRun(os.Args[1:], os.Stdout, os.Stderr)) +} + +func mainRun(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("spec-timings", flag.ContinueOnError) + fs.SetOutput(stderr) + if err := fs.Parse(args); err != nil { + return exitUsage + } + if fs.NArg() != 1 { + fmt.Fprintln(stderr, usageInstruct) + return exitUsage + } + if err := run(fs.Arg(0), stdout); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + return 0 +} + +type specRow struct { + name string + duration time.Duration +} + +func run(path string, out io.Writer) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("open %s: %w", path, err) + } + defer f.Close() + + rows, total, err := parseReport(f) + if err != nil { + return fmt.Errorf("parse %s: %w", path, err) + } + writeTable(out, rows, total) + return nil +} + +func parseReport(r io.Reader) ([]specRow, time.Duration, error) { + data, err := io.ReadAll(r) + if err != nil { + return nil, 0, err + } + var reports []types.Report + if err := json.Unmarshal(data, &reports); err != nil { + return nil, 0, err + } + + var rows []specRow + var total time.Duration + for _, rep := range reports { + for _, sr := range rep.SpecReports { + if sr.LeafNodeType != types.NodeTypeIt { + continue + } + if sr.State == types.SpecStateSkipped || sr.State == types.SpecStatePending { + continue + } + parts := append([]string{}, sr.ContainerHierarchyTexts...) + parts = append(parts, sr.LeafNodeText) + rows = append(rows, specRow{ + name: strings.Join(dedupeConsecutive(parts), " "), + duration: sr.RunTime, + }) + total += sr.RunTime + } + } + sort.SliceStable(rows, func(i, j int) bool { return rows[i].duration > rows[j].duration }) + return rows, total, nil +} + +func writeTable(out io.Writer, rows []specRow, total time.Duration) { + fmt.Fprintln(out, " Duration % Spec") + for _, row := range rows { + pct := 0.0 + if total > 0 { + pct = percentScalar * float64(row.duration) / float64(total) + } + fmt.Fprintf(out, "%8.1f s %4.1f %s\n", row.duration.Seconds(), pct, row.name) + } + fmt.Fprintf(out, "%8.1f s total\n", total.Seconds()) +} + +// dedupeConsecutive collapses adjacent equal entries — nested Describes +// with the same text otherwise print as "Manager Manager should ...". +func dedupeConsecutive(in []string) []string { + out := make([]string, 0, len(in)) + for _, s := range in { + if len(out) > 0 && out[len(out)-1] == s { + continue + } + out = append(out, s) + } + return out +} diff --git a/test/e2e/tools/spec-timings/main_test.go b/test/e2e/tools/spec-timings/main_test.go new file mode 100644 index 000000000..82db4b949 --- /dev/null +++ b/test/e2e/tools/spec-timings/main_test.go @@ -0,0 +1,250 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/onsi/ginkgo/v2/types" +) + +func fixtureReports() []types.Report { + return []types.Report{{ + SpecReports: []types.SpecReport{ + { + LeafNodeType: types.NodeTypeBeforeSuite, + LeafNodeText: "BeforeSuite", + RunTime: time.Second, + }, + { + LeafNodeType: types.NodeTypeIt, + ContainerHierarchyTexts: []string{"Manager", "Manager"}, + LeafNodeText: "fast spec", + State: types.SpecStatePassed, + RunTime: 300 * time.Millisecond, + }, + { + LeafNodeType: types.NodeTypeIt, + ContainerHierarchyTexts: []string{"Restart Snapshot Safety"}, + LeafNodeText: "long spec", + State: types.SpecStateFailed, + RunTime: 90 * time.Second, + }, + { + LeafNodeType: types.NodeTypeIt, + ContainerHierarchyTexts: []string{"Audit Redis Queue"}, + LeafNodeText: "middle spec", + State: types.SpecStatePassed, + RunTime: 10 * time.Second, + }, + { + LeafNodeType: types.NodeTypeIt, + ContainerHierarchyTexts: []string{"Should Be Skipped"}, + LeafNodeText: "skipped", + State: types.SpecStateSkipped, + RunTime: 0, + }, + { + LeafNodeType: types.NodeTypeIt, + ContainerHierarchyTexts: []string{"Should Be Pending"}, + LeafNodeText: "pending", + State: types.SpecStatePending, + RunTime: 0, + }, + }, + }} +} + +func TestParseReport_FiltersAndSortsAndDedupes(t *testing.T) { + data, err := json.Marshal(fixtureReports()) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + + rows, total, err := parseReport(bytes.NewReader(data)) + if err != nil { + t.Fatalf("parseReport: %v", err) + } + + wantNames := []string{ + "Restart Snapshot Safety long spec", + "Audit Redis Queue middle spec", + "Manager fast spec", + } + if len(rows) != len(wantNames) { + t.Fatalf("rows: want %d, got %d (%v)", len(wantNames), len(rows), rows) + } + for i, want := range wantNames { + if rows[i].name != want { + t.Errorf("row[%d]: want %q, got %q", i, want, rows[i].name) + } + } + + wantTotal := 90*time.Second + 10*time.Second + 300*time.Millisecond + if total != wantTotal { + t.Errorf("total: want %v, got %v", wantTotal, total) + } +} + +func TestWriteTable_FormatAndPercentages(t *testing.T) { + rows := []specRow{ + {name: "Restart Snapshot Safety long spec", duration: 90 * time.Second}, + {name: "Audit Redis Queue middle spec", duration: 10 * time.Second}, + {name: "Manager fast spec", duration: 300 * time.Millisecond}, + } + total := 90*time.Second + 10*time.Second + 300*time.Millisecond + + var buf bytes.Buffer + writeTable(&buf, rows, total) + + got := buf.String() + wantLines := []string{ + " Duration % Spec", + " 90.0 s 89.7 Restart Snapshot Safety long spec", + " 10.0 s 10.0 Audit Redis Queue middle spec", + " 0.3 s 0.3 Manager fast spec", + " 100.3 s total", + } + for _, line := range wantLines { + if !strings.Contains(got, line+"\n") { + t.Errorf("output missing line %q\nfull output:\n%s", line, got) + } + } +} + +func TestWriteTable_ZeroTotalAvoidsDivision(t *testing.T) { + var buf bytes.Buffer + writeTable(&buf, nil, 0) + got := buf.String() + if !strings.Contains(got, " Duration % Spec") { + t.Errorf("missing header: %s", got) + } + if !strings.Contains(got, " 0.0 s total") { + t.Errorf("missing total row: %s", got) + } +} + +func TestDedupeConsecutive(t *testing.T) { + cases := []struct { + in, want []string + }{ + {nil, []string{}}, + {[]string{"a"}, []string{"a"}}, + {[]string{"a", "a"}, []string{"a"}}, + {[]string{"a", "a", "b", "b", "a"}, []string{"a", "b", "a"}}, + {[]string{"Manager", "Manager", "spec"}, []string{"Manager", "spec"}}, + } + for _, tc := range cases { + got := dedupeConsecutive(tc.in) + if !equalStrings(got, tc.want) { + t.Errorf("dedupeConsecutive(%v) = %v, want %v", tc.in, got, tc.want) + } + } +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestRun_EndToEndWithFixtureFile(t *testing.T) { + data, err := json.Marshal(fixtureReports()) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + path := filepath.Join(t.TempDir(), "report.json") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + + var buf bytes.Buffer + if err := run(path, &buf); err != nil { + t.Fatalf("run: %v", err) + } + got := buf.String() + if !strings.Contains(got, "Restart Snapshot Safety long spec") { + t.Errorf("missing top spec in output: %s", got) + } + if !strings.Contains(got, "100.3 s total") { + t.Errorf("missing total in output: %s", got) + } +} + +func TestRun_MissingFile(t *testing.T) { + err := run(filepath.Join(t.TempDir(), "does-not-exist.json"), &bytes.Buffer{}) + if err == nil { + t.Fatal("expected error for missing file") + } +} + +func TestRun_BadJSON(t *testing.T) { + path := filepath.Join(t.TempDir(), "bad.json") + if err := os.WriteFile(path, []byte("not json"), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + err := run(path, &bytes.Buffer{}) + if err == nil { + t.Fatal("expected error for invalid JSON") + } +} + +func TestMainRun_ExitCodes(t *testing.T) { + good := filepath.Join(t.TempDir(), "report.json") + data, err := json.Marshal(fixtureReports()) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + if err := os.WriteFile(good, data, 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + + cases := []struct { + name string + args []string + wantCode int + }{ + {name: "no args", args: nil, wantCode: exitUsage}, + {name: "too many args", args: []string{"a", "b"}, wantCode: exitUsage}, + {name: "bad flag", args: []string{"-not-a-flag"}, wantCode: exitUsage}, + {name: "missing file", args: []string{"/no/such/file"}, wantCode: 1}, + {name: "success", args: []string{good}, wantCode: 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + got := mainRun(tc.args, &stdout, &stderr) + if got != tc.wantCode { + t.Errorf("mainRun: code=%d, want %d (stderr=%s)", got, tc.wantCode, stderr.String()) + } + }) + } +} diff --git a/test/e2e/tools/ts/main.go b/test/e2e/tools/ts/main.go new file mode 100644 index 000000000..81583b03d --- /dev/null +++ b/test/e2e/tools/ts/main.go @@ -0,0 +1,45 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// ts prefixes each stdin line with the elapsed seconds since startup +// and the current wallclock, then writes it to stdout. Useful when +// decorating long-running e2e logs. +package main + +import ( + "bufio" + "fmt" + "os" + "time" +) + +const maxLineBytes = 1 << 20 + +func main() { + start := time.Now() + scanner := bufio.NewScanner(os.Stdin) + scanner.Buffer(make([]byte, maxLineBytes), maxLineBytes) + for scanner.Scan() { + now := time.Now() + fmt.Printf("%7.3fs | %s | %s\n", now.Sub(start).Seconds(), now.Format("15:04:05"), scanner.Text()) + } + if err := scanner.Err(); err != nil { + fmt.Fprintln(os.Stderr, "ts:", err) + os.Exit(1) + } +} diff --git a/test/e2e/watchrule_configmap_secret_e2e_test.go b/test/e2e/watchrule_configmap_secret_e2e_test.go new file mode 100644 index 000000000..920d730d0 --- /dev/null +++ b/test/e2e/watchrule_configmap_secret_e2e_test.go @@ -0,0 +1,717 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +var _ = Describe("Manager WatchRule ConfigMap and Secret", Label("manager"), Ordered, func() { + var ( + testNs string + watchRuleRepo *RepoArtifacts + ) + + BeforeAll(func() { + By("creating WatchRule ConfigMap/Secret test namespace") + testNs = testNamespaceFor("manager-watchrule") + _, _ = kubectlRun("create", "namespace", testNs) // idempotent; ignore AlreadyExists + + By("setting up Gitea repo and credentials for WatchRule tests") + watchRuleRepo = SetupRepo( + resolveE2EContext(), + testNs, + fmt.Sprintf("e2e-manager-watchrule-%d", GinkgoRandomSeed()), + ) + + By("applying git secrets to test namespace") + _, err := kubectlRunInNamespace(testNs, "apply", "-f", watchRuleRepo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") + applySOPSAgeKeyToNamespace(testNs) + + By("creating shared gitprovider-normal for WatchRule specs") + createGitProviderWithURLInNamespace( + "gitprovider-normal", + testNs, + watchRuleRepo.GitSecretHTTP, + watchRuleRepo.RepoURLHTTP, + ) + verifyResourceStatus( + "gitprovider", "gitprovider-normal", testNs, + "True", "Ready", "Repository connectivity validated", + ) + }) + + AfterAll(func() { + cleanupNamespace(testNs) + }) + + SetDefaultEventuallyTimeout(30 * time.Second) + SetDefaultEventuallyPollingInterval(time.Second) + + It("should handle a normal and healthy GitProvider", Label("smoke"), func() { + // gitprovider-normal is created and verified in BeforeAll; this spec + // re-asserts it stays Ready without re-creating it. + verifyResourceStatus( + "gitprovider", "gitprovider-normal", testNs, "True", "Ready", "Repository connectivity validated", + ) + }) + + It("should reconcile a WatchRule CR", func() { + gitProviderName := "gitprovider-normal" + watchRuleName := "watchrule-test" + + By("creating a WatchRule that references the working GitProvider") + destName := watchRuleName + "-dest" + createGitTarget(destName, testNs, gitProviderName, getBaseFolder(), "main") + + data := struct { + Name string + Namespace string + DestinationName string + }{ + Name: watchRuleName, + Namespace: testNs, + DestinationName: destName, + } + + err := applyFromTemplate("test/e2e/templates/watchrule.tmpl", data, testNs) + Expect(err).NotTo(HaveOccurred(), "Failed to apply WatchRule") + + By("verifying the WatchRule is reconciled") + verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") + + By("cleaning up test resources") + cleanupWatchRule(watchRuleName, testNs) + cleanupGitTarget(destName, testNs) + }) + + It("should commit encrypted Secret manifests when WatchRule includes secrets", Label("smoke"), func() { + gitProviderName := "gitprovider-normal" + watchRuleName := "watchrule-secret-encryption-test" + secretName := "test-secret-encryption" + + By("creating WatchRule that includes secrets") + destName := watchRuleName + "-dest" + createGitTarget(destName, testNs, gitProviderName, "e2e/secret-encryption-test", "main") + + data := struct { + Name string + Namespace string + DestinationName string + }{ + Name: watchRuleName, + Namespace: testNs, + DestinationName: destName, + } + + err := applyFromTemplate("test/e2e/templates/watchrule-secret.tmpl", data, testNs) + Expect(err).NotTo(HaveOccurred(), "Failed to apply WatchRule") + verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") + verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") + verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") + + By("creating Secret in watched namespace") + _, _ = kubectlRunInNamespace(testNs, "delete", "secret", secretName, "--ignore-not-found=true") + + _, err = kubectlRunInNamespace( + testNs, + "create", + "secret", + "generic", + secretName, + "--from-literal=password=do-not-commit", + ) + Expect(err).NotTo(HaveOccurred(), "Secret creation should succeed") + + By("patching Secret once to avoid informer start race and force an update event") + _, err = kubectlRunInNamespace( + testNs, + "patch", + "secret", + secretName, + "--type=merge", + "--patch", + `{"stringData":{"password":"never-commit-this"}}`, + ) + Expect(err).NotTo(HaveOccurred(), "Secret patch should succeed") + + By("verifying Secret file is committed and does not contain plaintext data") + verifyEncryptedSecretCommitted := func(g Gomega) { + pullLatestRepoState(g, watchRuleRepo.CheckoutDir) + + expectedFile := filepath.Join(watchRuleRepo.CheckoutDir, + "e2e/secret-encryption-test", + fmt.Sprintf("v1/secrets/%s/%s.sops.yaml", testNs, secretName)) + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred(), fmt.Sprintf("Secret file must exist at %s", expectedFile)) + g.Expect(string(content)).To(ContainSubstring("sops:")) + g.Expect(string(content)).NotTo(ContainSubstring("do-not-commit")) + + bootstrapSOPSFile := filepath.Join(watchRuleRepo.CheckoutDir, "e2e/secret-encryption-test", ".sops.yaml") + bootstrapContent, bootstrapErr := os.ReadFile(bootstrapSOPSFile) + g.Expect(bootstrapErr).NotTo( + HaveOccurred(), + fmt.Sprintf(".sops.yaml must exist at %s", bootstrapSOPSFile), + ) + ageKey, ageKeyErr := readSOPSAgeKeyFromFile(getE2EAgeKeyPath()) + g.Expect(ageKeyErr).NotTo(HaveOccurred(), "Should read age private key") + recipient, recipientErr := deriveAgeRecipient(ageKey) + g.Expect(recipientErr).NotTo(HaveOccurred(), "Should derive age recipient") + g.Expect(string(bootstrapContent)).To(ContainSubstring(recipient)) + + decryptedOutput, decryptErr := decryptWithControllerSOPS(content, ageKey) + g.Expect(decryptErr).NotTo(HaveOccurred(), "Should decrypt committed secret via controller sops binary") + g.Expect(decryptedOutput).To(ContainSubstring("bmV2ZXItY29tbWl0LXRoaXM=")) + } + Eventually(verifyEncryptedSecretCommitted, "30s", "2s").Should(Succeed()) + + By("cleaning up test resources") + _, _ = kubectlRunInNamespace(testNs, "delete", "secret", secretName, "--ignore-not-found=true") + cleanupWatchRule(watchRuleName, testNs) + cleanupGitTarget(destName, testNs) + }) + + It("should generate missing SOPS age secret when age.recipients.generateWhenMissing is enabled", func() { + gitProviderName := "gitprovider-normal" + watchRuleName := "watchrule-secret-autogen-test" + secretName := "test-secret-autogen" + generatedSecretName := "sops-age-key-autogen" + + By("ensuring generated encryption secret does not exist before test") + _, _ = kubectlRunInNamespace( + testNs, + "delete", + "secret", + generatedSecretName, + "--ignore-not-found=true", + ) + + By("creating GitTarget with age recipient auto-generation enabled") + destName := watchRuleName + "-dest" + createGitTargetWithEncryptionOptions( + destName, + testNs, + gitProviderName, + "e2e/secret-autogen-test", + "main", + generatedSecretName, + true, + ) + + data := struct { + Name string + Namespace string + DestinationName string + }{ + Name: watchRuleName, + Namespace: testNs, + DestinationName: destName, + } + + err := applyFromTemplate("test/e2e/templates/watchrule-secret.tmpl", data, testNs) + Expect(err).NotTo(HaveOccurred(), "Failed to apply WatchRule") + verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") + + By("validating generated encryption secret has recipient and warning annotations") + var generatedAgeKey string + var generatedRecipient string + Eventually(func(g Gomega) { + output, getErr := kubectlRunInNamespace( + testNs, + "get", + "secret", + generatedSecretName, + "-o", + "json", + ) + g.Expect(getErr).NotTo(HaveOccurred()) + + var secretObj map[string]interface{} + unmarshalErr := json.Unmarshal([]byte(output), &secretObj) + g.Expect(unmarshalErr).NotTo(HaveOccurred()) + + annotations, _, annoErr := unstructured.NestedStringMap(secretObj, "metadata", "annotations") + g.Expect(annoErr).NotTo(HaveOccurred()) + g.Expect(annotations).To(HaveKey("configbutler.ai/age-recipient")) + g.Expect(annotations["configbutler.ai/age-recipient"]).To(HavePrefix("age1")) + g.Expect(annotations).To(HaveKeyWithValue("configbutler.ai/backup-warning", "REMOVE_AFTER_BACKUP")) + generatedRecipient = annotations["configbutler.ai/age-recipient"] + + secretData, found, keyErr := unstructured.NestedStringMap(secretObj, "data") + g.Expect(keyErr).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + var sopsAgeKeyB64 string + for key, value := range secretData { + if strings.HasSuffix(key, ".agekey") { + sopsAgeKeyB64 = value + break + } + } + g.Expect(sopsAgeKeyB64).NotTo(BeEmpty()) + + keyBytes, decodeErr := base64.StdEncoding.DecodeString(sopsAgeKeyB64) + g.Expect(decodeErr).NotTo(HaveOccurred()) + generatedAgeKey = strings.TrimSpace(string(keyBytes)) + g.Expect(generatedAgeKey).To(HavePrefix("AGE-SECRET-KEY-")) + }, "30s", "2s").Should(Succeed()) + + By("waiting for auto-generated target bootstrap file to be present") + Eventually(func(g Gomega) { + pullLatestRepoState(g, watchRuleRepo.CheckoutDir) + + bootstrapSOPSFile := filepath.Join(watchRuleRepo.CheckoutDir, "e2e/secret-autogen-test", ".sops.yaml") + bootstrapContent, bootstrapErr := os.ReadFile(bootstrapSOPSFile) + g.Expect(bootstrapErr).NotTo(HaveOccurred(), + fmt.Sprintf(".sops.yaml must exist at %s", bootstrapSOPSFile)) + g.Expect(string(bootstrapContent)).To(ContainSubstring(generatedRecipient)) + }, "30s", "2s").Should(Succeed()) + + By("creating Secret in watched namespace") + _, _ = kubectlRunInNamespace(testNs, "delete", "secret", secretName, "--ignore-not-found=true") + _, err = kubectlRunInNamespace( + testNs, + "create", + "secret", + "generic", + secretName, + "--from-literal=password=do-not-commit", + ) + Expect(err).NotTo(HaveOccurred(), "Secret creation should succeed") + + By("patching Secret once to avoid informer start race and force an update event") + _, err = kubectlRunInNamespace( + testNs, + "patch", + "secret", + secretName, + "--type=merge", + "--patch", + `{"stringData":{"password":"autogen-never-commit-this"}}`, + ) + Expect(err).NotTo(HaveOccurred(), "Secret patch should succeed") + + By("verifying committed secret is encrypted and decryptable with generated key") + verifyEncryptedSecretCommitted := func(g Gomega) { + pullLatestRepoState(g, watchRuleRepo.CheckoutDir) + + expectedFile := filepath.Join(watchRuleRepo.CheckoutDir, + "e2e/secret-autogen-test", + fmt.Sprintf("v1/secrets/%s/%s.sops.yaml", testNs, secretName)) + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred(), fmt.Sprintf("Secret file must exist at %s", expectedFile)) + g.Expect(string(content)).To(ContainSubstring("sops:")) + g.Expect(string(content)).NotTo(ContainSubstring("autogen-never-commit-this")) + + bootstrapSOPSFile := filepath.Join(watchRuleRepo.CheckoutDir, "e2e/secret-autogen-test", ".sops.yaml") + bootstrapContent, bootstrapErr := os.ReadFile(bootstrapSOPSFile) + g.Expect(bootstrapErr).NotTo( + HaveOccurred(), + fmt.Sprintf(".sops.yaml must exist at %s", bootstrapSOPSFile), + ) + g.Expect(string(bootstrapContent)).To(ContainSubstring(generatedRecipient)) + + decryptedOutput, decryptErr := decryptWithControllerSOPS(content, generatedAgeKey) + g.Expect(decryptErr).NotTo(HaveOccurred(), "Should decrypt committed secret via generated age key") + g.Expect(decryptedOutput).To(ContainSubstring("YXV0b2dlbi1uZXZlci1jb21taXQtdGhpcw==")) + } + Eventually(verifyEncryptedSecretCommitted, "30s", "2s").Should(Succeed()) + + By("cleaning up test resources") + _, _ = kubectlRunInNamespace(testNs, "delete", "secret", secretName, "--ignore-not-found=true") + _, _ = kubectlRunInNamespace( + testNs, + "delete", + "secret", + generatedSecretName, + "--ignore-not-found=true", + ) + cleanupWatchRule(watchRuleName, testNs) + cleanupGitTarget(destName, testNs) + }) + + It("should create Git commit when ConfigMap is added via WatchRule", Label("smoke"), func() { + gitProviderName := "gitprovider-normal" + watchRuleName := "watchrule-configmap-test" + configMapName := "test-configmap" + uniqueRepoName := watchRuleRepo.RepoName + + By("creating WatchRule that monitors ConfigMaps") + destName := watchRuleName + "-dest" + createGitTarget(destName, testNs, gitProviderName, "e2e/configmap-test", "main") + + data := struct { + Name string + Namespace string + DestinationName string + }{ + Name: watchRuleName, + Namespace: testNs, + DestinationName: destName, + } + + err2 := applyFromTemplate("test/e2e/templates/manager/watchrule-configmap.tmpl", data, testNs) + Expect(err2).NotTo(HaveOccurred(), "Failed to apply WatchRule") + + By("verifying WatchRule is ready") + verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") + verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") + + // Let any in-flight reconciles from prior specs drain before triggering + // our event. Without this, a stale WatchRule still in informer cache can + // double-commit the same ConfigMap under a different commitPath and end + // up on HEAD, masking our commit. + time.Sleep(5 * time.Second) + + By("creating test ConfigMap to trigger Git commit") + configMapData := struct { + Name string + Namespace string + }{ + Name: configMapName, + Namespace: testNs, + } + + err3 := applyFromTemplate( + "test/e2e/templates/manager/configmap.tmpl", + configMapData, + testNs, + "--as=jane@acme.com", // Important: we validate later if the user is included in the git commit! + ) + Expect(err3).NotTo(HaveOccurred(), "Failed to apply ConfigMap") + + By("waiting for controller reconciliation of ConfigMap event") + verifyReconciliationLogs := func(g Gomega) { + output, err := controllerLogs(500) + g.Expect(err).NotTo(HaveOccurred()) + + // Check for git commit operation in logs + g.Expect(output).To(ContainSubstring("git commit"), + "Should see git commit operation in controller logs") + } + Eventually(verifyReconciliationLogs).Should(Succeed()) + + By("verifying ConfigMap YAML file exists in Gitea repository") + verifyGitCommit := func(g Gomega) { + // Use the pre-checked out repository directory + By("using pre-checked out repository for verification") + + By("pulling latest changes from remote repository") + pullLatestRepoState(g, watchRuleRepo.CheckoutDir) + + // Check for the expected ConfigMap file (new API-aligned path) + expectedFile := filepath.Join(watchRuleRepo.CheckoutDir, + "e2e/configmap-test", + fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, configMapName)) + fileInfo, statErr := os.Stat(expectedFile) + g.Expect(statErr).NotTo(HaveOccurred(), fmt.Sprintf("ConfigMap file should exist at %s", expectedFile)) + g.Expect(fileInfo.Size()).To(BeNumerically(">", 0), "ConfigMap file should not be empty") + + expectedRepoPath := path.Join( + "e2e/configmap-test", + fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, configMapName), + ) + assertLatestCommitForPathTouchesOnlyWithOptional( + g, + watchRuleRepo.CheckoutDir, + expectedRepoPath, + []string{expectedRepoPath}, + []string{ + path.Join("e2e/configmap-test", "README.md"), + path.Join("e2e/configmap-test", ".sops.yaml"), + }, + ) + assertLatestCommitTouchesNoNamespaces( + g, + watchRuleRepo.CheckoutDir, + "e2e/configmap-test/v1/configmaps", + []string{ + "gitops-reverser", + "flux-system", + "kube-system", + "tilt-playground", + }, + ) + + // Verify file content contains expected ConfigMap data + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred()) + g.Expect(string(content)).To(ContainSubstring("test-key: test-value"), + "ConfigMap file should contain expected data") + + // Verify latest commit message contains operation, resource path + gitLogCmd := exec.Command("git", "log", "-1", "--pretty=%B") + gitLogCmd.Dir = watchRuleRepo.CheckoutDir + commitMsg, commitErr := gitLogCmd.CombinedOutput() + + if commitErr != nil { + g.Expect(commitErr).NotTo(HaveOccurred(), + fmt.Sprintf("Should read latest commit message. Output: %s", string(commitMsg))) + } + + msg := string(commitMsg) + g.Expect(msg).To(ContainSubstring("[CREATE]"), + "Latest commit message should include operation [CREATE]") + g.Expect(msg).To(ContainSubstring(fmt.Sprintf("v1/configmaps/%s", configMapName)), + "Latest commit message should include resource path") + + gitLogCmd = exec.Command("git", "log", "-1", "--pretty=%an") + gitLogCmd.Dir = watchRuleRepo.CheckoutDir + authorMsg, commitErr := gitLogCmd.CombinedOutput() + if commitErr != nil { + g.Expect(commitErr).NotTo(HaveOccurred(), + fmt.Sprintf("Should read latest commit author. Output: %s", string(authorMsg))) + } + + author := string(authorMsg) + g.Expect(author).To(ContainSubstring("jane@acme.com")) + } + Eventually(verifyGitCommit).Should(Succeed()) + + By("cleaning up test resources") + _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", configMapName, "--ignore-not-found=true") + cleanupWatchRule(watchRuleName, testNs) + cleanupGitTarget(destName, testNs) + + By("✅ ConfigMap to Git commit E2E test passed - verified actual file creation and commit") + fmt.Printf("✅ ConfigMap '%s' successfully triggered Git commit with YAML file in repo '%s'\n", + configMapName, uniqueRepoName) + }) + + // Regression net for the rule-add backfill gap: a ConfigMap that + // already exists in the cluster *before* a WatchRule is applied must + // land in git once the rule becomes Ready. Today this fails because + // ReconcileForRuleChange takes the early-return at manager.go:660 + // when the new rule's GVR is already covered by other rules (and on + // fresh-target installs the snapshot output can be dropped because no + // FolderReconciler is registered yet). The unit-level coverage lives + // in internal/watch/rule_change_snapshot_test.go; this spec locks the + // observable in at the user-visible layer so a future revamp of the + // snapshot trigger logic can't silently regress it. + // + It("should backfill pre-existing ConfigMap when WatchRule is added afterwards", Label("smoke"), func() { + gitProviderName := "gitprovider-normal" + watchRuleName := "watchrule-backfill-test" + configMapName := "preexisting-configmap" + uniqueRepoName := watchRuleRepo.RepoName + + By("creating GitTarget but no WatchRule yet") + destName := watchRuleName + "-dest" + createGitTarget(destName, testNs, gitProviderName, "e2e/backfill-rule-add", "main") + verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") + + By("creating the ConfigMap BEFORE the rule that should select it") + configMapData := struct { + Name string + Namespace string + }{ + Name: configMapName, + Namespace: testNs, + } + err := applyFromTemplate( + "test/e2e/templates/manager/configmap.tmpl", + configMapData, + testNs, + "--as=jane@acme.com", + ) + Expect(err).NotTo(HaveOccurred(), "Failed to pre-create ConfigMap") + + // Confirm nothing committed yet — no rule exists, so the ConfigMap + // must not appear in the repo before we apply the WatchRule. + expectedRepoPath := path.Join( + "e2e/backfill-rule-add", + fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, configMapName), + ) + expectedFile := filepath.Join(watchRuleRepo.CheckoutDir, expectedRepoPath) + + By("applying the WatchRule (rule arrives after the resource)") + watchRuleData := struct { + Name string + Namespace string + DestinationName string + }{ + Name: watchRuleName, + Namespace: testNs, + DestinationName: destName, + } + err = applyFromTemplate("test/e2e/templates/manager/watchrule-configmap.tmpl", watchRuleData, testNs) + Expect(err).NotTo(HaveOccurred(), "Failed to apply WatchRule") + verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") + + By("waiting for the pre-existing ConfigMap to be backfilled into git") + verifyBackfill := func(g Gomega) { + pullLatestRepoState(g, watchRuleRepo.CheckoutDir) + + fileInfo, statErr := os.Stat(expectedFile) + g.Expect(statErr).NotTo(HaveOccurred(), + fmt.Sprintf("Pre-existing ConfigMap must be backfilled at %s after the rule lands", expectedFile)) + g.Expect(fileInfo.Size()).To(BeNumerically(">", 0), "Backfilled ConfigMap file should not be empty") + + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred()) + g.Expect(string(content)).To(ContainSubstring("test-key: test-value"), + "Backfilled file should contain the ConfigMap data that existed before the rule") + } + Eventually(verifyBackfill). + WithTimeout(60 * time.Second). + WithPolling(2 * time.Second). + Should(Succeed()) + + By("cleaning up test resources") + _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", configMapName, "--ignore-not-found=true") + cleanupWatchRule(watchRuleName, testNs) + cleanupGitTarget(destName, testNs) + + By("✅ rule-add backfill E2E test passed") + fmt.Printf("✅ Pre-existing ConfigMap '%s' was backfilled to repo '%s' after the WatchRule landed\n", + configMapName, uniqueRepoName) + }) + + It("should delete Git file when ConfigMap is deleted via WatchRule", Label("smoke"), func() { + gitProviderName := "gitprovider-normal" + watchRuleName := "watchrule-delete-test" + configMapName := "test-configmap-to-delete" + uniqueRepoName := watchRuleRepo.RepoName + + By("creating WatchRule that monitors ConfigMaps") + destName := watchRuleName + "-dest" + createGitTarget(destName, testNs, gitProviderName, "e2e/delete-test", "main") + data := struct { + Name string + Namespace string + DestinationName string + }{ + Name: watchRuleName, + Namespace: testNs, + DestinationName: destName, + } + + err2 := applyFromTemplate("test/e2e/templates/manager/watchrule-configmap.tmpl", data, testNs) + Expect(err2).NotTo(HaveOccurred(), "Failed to apply WatchRule") + + By("verifying WatchRule is ready") + verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") + verifyResourceStatus("gittarget", destName, testNs, "True", "Ready", "") + + // See "should create Git commit when ConfigMap is added": prior specs' + // WatchRules can still be in the controller's event router and pile on + // extra commits at unrelated commitPaths, knocking HEAD off our commit. + time.Sleep(5 * time.Second) + + By("creating test ConfigMap to trigger Git commit") + configMapData := struct { + Name string + Namespace string + }{ + Name: configMapName, + Namespace: testNs, + } + + err3 := applyFromTemplate("test/e2e/templates/manager/configmap.tmpl", configMapData, testNs) + Expect(err3).NotTo(HaveOccurred(), "Failed to apply ConfigMap") + + By("waiting for ConfigMap file to appear in Git repository") + verifyFileCreated := func(g Gomega) { + // Pull latest changes from the remote repository + pullLatestRepoState(g, watchRuleRepo.CheckoutDir) + + // Check for the expected ConfigMap file (new API-aligned path) + expectedFile := filepath.Join(watchRuleRepo.CheckoutDir, + "e2e/delete-test", + fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, configMapName)) + fileInfo, statErr := os.Stat(expectedFile) + g.Expect(statErr).NotTo(HaveOccurred(), fmt.Sprintf("ConfigMap file should exist at %s", expectedFile)) + g.Expect(fileInfo.Size()).To(BeNumerically(">", 0), "ConfigMap file should not be empty") + } + Eventually(verifyFileCreated).Should(Succeed()) + + By("deleting the ConfigMap to trigger DELETE operation") + _, err := kubectlRunInNamespace(testNs, "delete", "configmap", configMapName) + Expect(err).NotTo(HaveOccurred(), "ConfigMap deletion should succeed") + + By("verifying ConfigMap file is deleted from Git repository") + verifyFileDeleted := func(g Gomega) { + // Pull latest changes from the remote repository + By("pulling latest changes after deletion") + pullLatestRepoState(g, watchRuleRepo.CheckoutDir) + + // Check that the ConfigMap file no longer exists (new API-aligned path) + expectedRelativePath := path.Join( + "e2e/delete-test", + fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, configMapName), + ) + expectedFile := filepath.Join(watchRuleRepo.CheckoutDir, expectedRelativePath) + _, statErr := os.Stat(expectedFile) + g.Expect(statErr).To(HaveOccurred(), fmt.Sprintf("ConfigMap file should NOT exist at %s", expectedFile)) + g.Expect(os.IsNotExist(statErr)).To(BeTrue(), "Error should be 'file does not exist'") + + assertLatestCommitForPathTouchesOnlyWithOptional( + g, + watchRuleRepo.CheckoutDir, + expectedRelativePath, + []string{expectedRelativePath}, + nil, + ) + assertLatestCommitTouchesNoNamespaces( + g, + watchRuleRepo.CheckoutDir, + "e2e/delete-test/v1/configmaps", + []string{ + "gitops-reverser", + "flux-system", + "kube-system", + "tilt-playground", + }, + ) + + // Verify git log shows DELETE commit + By("verifying git log shows DELETE operation") + gitLogCmd := exec.Command("git", "log", "--oneline", "-n", "5") + gitLogCmd.Dir = watchRuleRepo.CheckoutDir + logOutput, logErr := gitLogCmd.CombinedOutput() + g.Expect(logErr).NotTo(HaveOccurred(), "Should be able to read git log") + g.Expect(string(logOutput)).To(ContainSubstring("DELETE"), + "Git log should contain DELETE operation") + } + // 60s (vs the 30s default) tolerates a busier shared controller under + // Ginkgo parallelism: the DELETE commit can lag when other processes are + // driving reconciles concurrently. + Eventually(verifyFileDeleted, 60*time.Second, 2*time.Second).Should(Succeed()) + + By("cleaning up test resources") + cleanupWatchRule(watchRuleName, testNs) + cleanupGitTarget(destName, testNs) + + By("✅ ConfigMap deletion E2E test passed - verified file removal from Git") + fmt.Printf("✅ ConfigMap '%s' deletion successfully triggered Git commit removing file from repo '%s'\n", + configMapName, uniqueRepoName) + }) +})