diff --git a/.coverage-baseline b/.coverage-baseline index c67e78c6..c2add760 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -79.5 +79.9 diff --git a/README.md b/README.md index 967d7545..392e9820 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,10 @@ several source clusters, repositories, branches, and folders. - **Mirror only the objects you label.** A rule's `objectSelector` lets the API server choose which objects of a type are mirrored; an object that loses its label leaves the mirror like a deleted one. See [selecting objects by label](docs/configuration.md#selecting-objects-by-label-rulesobjectselector). +- **Each object reaches a target once.** A target watches each object through one collection, so an + edit is never delivered twice and a status-only update never splits a commit. A rule that would + overlap another on the same target is refused, naming both. See + [overlapping collections](docs/configuration.md#overlapping-collections). - **In-place edits keep the shape of your file.** Updates preserve key order, comments, and untouched documents in multi-document files. - **Inspect a repository before you point at it.** `manifest-analyzer --mode scan-repo` classifies diff --git a/charts/gitops-reverser/README.md b/charts/gitops-reverser/README.md index 740c33f6..e61bf2f0 100644 --- a/charts/gitops-reverser/README.md +++ b/charts/gitops-reverser/README.md @@ -176,8 +176,8 @@ nodeSelector: | `servers.audit.tls.secretNameOverride` | Override Secret name for audit TLS cert/key | `-audit-server-cert` | | `controllerManager.additionalSensitiveResources` | Extra Secret-shaped resource types encrypted as `resource` or `group/resource` | `[]` | | `controllerManager.insecureAllowGitHTTP` | Permit credentials with `http://` GitProvider URLs for trusted in-cluster development Git servers | `false` | -| `controllerManager.branchWorkerQueueDepth` | Event queue depth per branch worker. A full queue drops the write (`git_queue_drops_total`) instead of stalling the watch path, so the live attributed commit is lost even though convergence heals the mirror. Costs queue depth × payload of pod memory, which `branchBufferMaxSize` does **not** cover | `1000` | -| `controllerManager.branchBufferMaxSize` | Maximum in-memory event buffer per branch worker. Bounds the open commit window plus the writes retained for replay until a push succeeds | `8Mi` | +| `controllerManager.branchWorkerQueueDepth` | Event queue depth per branch worker. A full queue refuses the write (`git_queue_refusals_total`) instead of stalling the watch path; the watch delivers it again from its cursor, and only a cursor that expires first loses that write's own commit (and, under `prune.mode: OnEvent`, a delete's removal). Costs queue depth × payload of pod memory, which `branchBufferMaxSize` does **not** cover | `1000` | +| `controllerManager.branchBufferMaxSize` | Maximum in-memory event buffer per branch worker. Bounds the open commit window plus the writes retained for replay until a push succeeds. While the remote cannot be reached, retained writes that reach it close the worker to new writes, saves and resyncs until they are published; the watch delivers a refused event again | `8Mi` | | `controllerManager.commitRequestTTL` | How long a `CommitRequest` is kept once it finishes. Written onto the request as the `configbutler.ai/delete-after` annotation; remove or edit that to keep one. `"0"` writes none | `48h` | | `git.refreshInterval` | How often an idle `GitTarget` re-proves where its branch is on the Git remote. `"0"` disables it | `10m` | | `auditService.type` | Service type for the dedicated audit Service | `NodePort` | diff --git a/charts/gitops-reverser/values.schema.json b/charts/gitops-reverser/values.schema.json index 2339de43..1ee6a758 100644 --- a/charts/gitops-reverser/values.schema.json +++ b/charts/gitops-reverser/values.schema.json @@ -107,7 +107,7 @@ "branchWorkerQueueDepth": { "type": "integer", "minimum": 1, - "description": "Event queue depth per branch worker. A full queue drops the write (git_queue_drops_total) rather than stalling the watch path, losing that live attributed commit. Costs queue depth x payload of pod memory, which branchBufferMaxSize does not cover." + "description": "Event queue depth per branch worker. A full queue refuses the write (git_queue_refusals_total) rather than stalling the watch path; the watch delivers it again from its cursor. Costs queue depth x payload of pod memory, which branchBufferMaxSize does not cover." }, "branchBufferMaxSize": { "type": "string", diff --git a/charts/gitops-reverser/values.yaml b/charts/gitops-reverser/values.yaml index 2b610587..e0b84493 100644 --- a/charts/gitops-reverser/values.yaml +++ b/charts/gitops-reverser/values.yaml @@ -68,23 +68,27 @@ controllerManager: # writing to one (GitProvider, branch), so pointing another GitTarget at a branch # divides the headroom of a worker already in use. # - # A full queue DROPS the write and counts git_queue_drops_total, rather than stalling - # the watch path behind a slow remote. Convergence heals the mirror afterwards, but the - # live attributed commit for a dropped write never happens — so raise this if - # git_queue_drops_total is ever non-zero. Size it so a bounded burst cannot overrun it: + # A full queue REFUSES the write and counts git_queue_refusals_total, rather than stalling + # the watch path behind a slow remote. The watch keeps its cursor and delivers the write + # again, but its stream stalls meanwhile, and a cursor that expires first loses that + # write's own commit (and, under prune.mode OnEvent, a delete's removal) — so raise this if git_queue_refusals_total is non-zero on a healthy + # branch. Size it so a bounded burst cannot overrun it: # roughly (concurrent writers) x (GitTargets sharing the branch worker). Note that an # administrative burst (an unpaced `kubectl delete` of many objects) is far spikier than # the steady-state traffic the depth gauge shows you. # - # Costs pod memory ON TOP of branchBufferMaxSize, which does NOT cover this queue: that - # cap is accounted only after an item leaves it. Budget queue depth x serialized size + # Costs pod memory ON TOP of branchBufferMaxSize on a healthy branch, where that cap does + # NOT cover this queue (during a Git outage it does). Budget queue depth x serialized size # x ~6 (a queued object is held as an unstructured map), per SATURATED worker — about # 5Mi at this default for ~800-byte resources, but the multiplier is on the payload, so # size it against your own if you mirror megabyte-sized ConfigMaps or Secrets. branchWorkerQueueDepth: 1000 # Maximum in-memory event buffer per branch worker, as a Kubernetes resource quantity. # Bounds the open commit window plus the writes retained for replay until a push - # succeeds; tripping it finalizes the window early, ignoring the commit cadence. + # succeeds; tripping it finalizes the window early, ignoring the commit cadence. While the + # remote cannot be reached, it also bounds everything the worker accepts, queued work + # included: work that would exceed it pauses the worker's intake of new writes, saves and + # resyncs until a push lands, and the watch delivers what was refused once intake reopens. branchBufferMaxSize: 8Mi # How long a CommitRequest is kept once it finishes, as a duration. Every save leaves one object # behind, so without this they accumulate for as long as the namespace lives. The controller diff --git a/cmd/main.go b/cmd/main.go index eeec9bd7..94de00ea 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -633,9 +633,10 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { git.DefaultBranchWorkerQueueDepth, fmt.Sprintf("Event queue depth per branch worker (default %d). ", git.DefaultBranchWorkerQueueDepth)+ "A branch worker is shared by every "+ - "GitTarget writing to one (GitProvider, branch), and a full queue DROPS the write "+ - "(git_queue_drops_total) rather than stalling the watch path behind a slow remote, "+ - "losing that live attributed commit even though convergence later heals the mirror. "+ + "GitTarget writing to one (GitProvider, branch), and a full queue REFUSES the write "+ + "(git_queue_refusals_total) rather than stalling the watch path behind a slow remote; "+ + "the watch delivers it again from its cursor, and a cursor that expires first loses "+ + "that write's own commit (and, under prune.mode OnEvent, a delete's removal). "+ "Size it so a bounded burst cannot overrun it: roughly (concurrent writers) x "+ "(GitTargets sharing the branch worker). Costs queue depth x payload of pod memory "+ "ON TOP of --branch-buffer-max-size, which does not cover this queue.") diff --git a/docs/INDEX.md b/docs/INDEX.md index 0ee9466a..f9c48148 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -128,8 +128,10 @@ so you never have to read a proposal to find out it already shipped. | Doc | Open question | |---|---| -| [`branch-worker-event-model.md`](design/branch-worker-event-model.md) | **proposal; its first fix is built in #412.** The quiet-branch `WaitingForPush` failure (a failed push now retries on its own backoff) and the Git operation deadlines still to do. Worker ownership, FIFO inputs, save and deadline semantics, and the transition boundary for later durability. Journal storage and rollout belong to the HA plan. | -| [`gittarget-state-of-affairs.md`](design/gittarget-state-of-affairs.md) | **snapshot, 2026-10-02.** What is built for `GitTarget` and the branch worker, which page owns which decision, the known gaps ranked by user impact (verified in code), and the recommended order. Read it first to find the right page on this topic. | +| [`branch-worker-event-model.md`](design/branch-worker-event-model.md) | **proposal; its in-process fixes are built in #412 and #413.** Publication retry, Git operation deadlines, intake pause and resume, and publication status are done. Worker ownership, FIFO inputs, save and deadline semantics, and the transition boundary for later durability, which is next. Journal storage and rollout belong to the HA plan. | +| [`gittarget-state-of-affairs.md`](design/gittarget-state-of-affairs.md) | **snapshot, 2026-10-04.** What is built for `GitTarget` and the branch worker, which page owns which decision, the review findings and how each was closed, and what is left elsewhere. Read it first to find the right page on this topic. | +| [`gittarget-branch-worker-pending-writes.md`](design/gittarget-branch-worker-pending-writes.md) | **built in #413.** Ordered pending writes, one materializer and one retry schedule; a safe admission boundary for the watch; Git call deadlines; intake pause and resume; publication status. Holds the recovery contract for an outage, replay limits, and the acceptance scenarios. | +| [`event-pipeline-overview.md`](design/event-pipeline-overview.md) | **snapshot, 2026-10-04.** Diagrams of the pipeline as built: watch streams and their replay/resume rules, the three meanings of "replay", the branch worker's inputs, the pending writes' lifecycle, retry and intake, a save end to end, what is an event today, and an honest list of what is still missing. | | [`gittarget-parent-branch.md`](design/gittarget-parent-branch.md) | **merged in #407.** `spec.parentBranch`: a target stays on standby until a write needs a branch, then creates its write branch from the parent's tip as checked at publication. The contract as built, the defect it started from, prior art and naming. Open: parent per `GitProvider.status.branches` entry. | | [`gittarget-parent-hardening.md`](design/gittarget-parent-hardening.md) | **done in #407.** Parent changes on a live worker, an empty repository gaining a ref, an unresolved default branch, recovery that cannot strand work, and the refused upload, without adding a pull to the fast path; plus the two recovery fixes from the review at `93120caf`. Records the merge boundary and the next separate PRs. | | [`gittarget-parent-empty-repository.md`](design/gittarget-parent-empty-repository.md) | **deferred.** An empty repository's default branch: the server reports it over protocol v2 but go-git discards it (upstream fix first); then a separate policy decision on which branch the first commit may create. Assuming `main` was considered and rejected. | @@ -142,7 +144,7 @@ so you never have to read a proposal to find out it already shipped. | [`target-watch-plan.md`](design/target-watch-plan.md) | **partly built.** The diff is built and applied; removal semantics are not. The companion to [`watch-manager-ownership.md`](design/watch-manager-ownership.md): the ownership page says WHO applies a plan, this one says WHAT a plan is and what changing it may touch. A resource collection — group, resource, namespace, deliberately no served version — is the one identity the watch stream, the render-fidelity scope and the mark-and-sweep boundary all agree on, because a key that does not round-trip to the scope it sweeps under is the class of error that deletes user data. The plan is diffed into `keep`/`start`/`restart`/`stop` and applied per collection, so adding one WatchRule stops re-initializing every unrelated collection into a queue shared with other tenants; a `restart` is a served-version change, which is why the version is spec DATA rather than identity. Readiness and the fidelity revision are per scope, so a KEPT collection holds the result its own initialization produced rather than being asked to prove itself again over an unrelated edit. `stop` never touches files — removal is a Git-side sweep under the target's existing `spec.prune.mode`, not a watch-layer delete. "Cut at the producer" is the accepted consequence: nothing fences the queue, so a deselected collection may leave a short tail of writes, bounded by the queue and converged afterwards. Still open: the `stop` classification wants a settled `TypeRemoved` from `typeset` (see TODO), and removal on INTENT is undecided. | | [`collection-terminology-rename.md`](design/collection-terminology-rename.md) | **step 1, shipped.** Replaced cells with resource collections and removed per-rule `operations` from both rule kinds. Rules observe every object event; GitTarget prune mode controls removal. Records the rename census (760 Go identifiers, 265 documentation occurrences), the operation-filter inventory, API/Helm migration, compatibility boundaries, and test cases. | | [`watches-labels-simplification.md`](design/watches-labels-simplification.md) | **step 2, implemented.** After the collection rename and operation-filter removal, native per-rule `objectSelector` (#146). Kubernetes is authoritative: `Always` compares selected identities with all managed Git identities in the type/namespace scope, without comparing Git labels. Full support includes supported Kustomize layouts from the first release, plus filtered-removal attribution and ordered producer replacement. | -| [`label-selection-follow-ups.md`](design/label-selection-follow-ups.md) | **design, open.** What the label-selection review rounds left open, with the evidence for each: real deletions in a selected collection wait the full 3s attribution grace (the watch cannot tell them from a label exit, measured), the resync fence that stops overlapping collections from coalescing (kept on purpose), same-second selector-conflict ties, and one store rebuild per object coming back. Also lists what was fixed on the branch | +| [`label-selection-follow-ups.md`](design/label-selection-follow-ups.md) | **design, open.** What the label-selection review rounds left open, with the evidence for each: real deletions in a selected collection wait the full 3s attribution grace (the watch cannot tell them from a label exit, measured), the resync fence that stops overlapping collections from coalescing (kept on purpose), same-second overlap ties, and one store rebuild per object coming back. Also lists what was fixed on the branch | | [`attribution-removal-wait-options.md`](design/attribution-removal-wait-options.md) | a removal now waits for evidence about the DELETION rather than accepting the object's last write, which stopped it naming whoever last edited the object as the author of a deletion they did not perform. Enumerates the eight situations a resolution can be in and shows the cost is concentrated in exactly one: a removal for which no delete fact will ever arrive (a graceful pod delete, a status-only removal, a type the audit policy skips) spends the whole grace to return the answer it had at t=0, measured at ~3.1s against ~70ms when evidence is present. Prices five options against that, and recommends a per-route watermark — stop waiting once the fact stream has demonstrably moved past this event — over a second timeout flag whose right value lives in the API server's config rather than ours. Open: the decision, and how common the case is outside the e2e suite | | [`commitrequest-save-wait-options.md`](design/commitrequest-save-wait-options.md) | **design, decided** — a five-phase plan, reported from a live cluster: a `CommitRequest` can expire before the write it exists to publish arrives, because the close delay defaulted to `0` while the write is still held head-of-line waiting for an audit fact, so the save resolves `Ready=True`/`NoWindowInGrace` and the edit commits seconds later under the target's `liveTemplate`. The decision is to repair the default (`spec.closeDelay`, now a duration string defaulting to `"2s"`, **shipped as phase 0**) and then let a request **name the write it is waiting for**, reusing the `(uid, resourceVersion)` identity the attribution join already keys on, including its settled `ExactCapable` rule. Records why the fact index cannot BE the wait (it sits upstream of the watch event, so a fact-keyed wait fires before the window opens) and what to take from it anyway — the register-then-check waiter discipline, and a best-effort retrospective oracle bounded by a TUNABLE TTL. Twelve situations, and the page is deliberate about the two it cannot fix cheaply: row 11 (a saturated queue is rejected in the WATCH path and never reaches the worker, so no worker-side wait can diagnose it) is an accepted, named gap rather than a promise, and row 10 is fixed only by a hook in the fidelity-drop branch that marks a wait terminally UNSATISFIABLE — satisfying before that gate would arm a finalize for a write that never commits. Carries a **preconditions chapter** for saves on today's build (audit policy first, then delay sizing, fidelity warm-up, save-button debounce, and assert on the commit rather than on `Ready`). `CommitterActivity` is deferred, not recommended: by its own arrival-not-existence rule it cannot satisfy the already-arrived case. Open: situation 12 (one actor, two saves, one window — blocks phase 2), whether the fact-index lookup earns its coupling, and how an unsatisfied wait is graded | | [`commit-timing-surface.md`](design/commit-timing-surface.md) | **built.** One configuration surface for commit windows and saves, an in-place `v1alpha3` break with one tested upgrade procedure: only writes open a window; a `window` block (`idleTimeout`, `maxDuration`, target default `1m`) on both kinds, which a `CommitRequest` replaces whole for the window it attaches to; `attach: CurrentOrNext \| Next` with `attachTimeout`; opt-in `whenNothingToCommit: CommitEmpty` that keeps the cause in the reason. Precise clock and zero rules. Supersedes the save-wait timer model. Open: the target's `maxDuration` value | diff --git a/docs/TODO.md b/docs/TODO.md index 2e8b8aa4..97b746b4 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -13,10 +13,9 @@ a request on a target that may not be written fails, whatever path ends it (`internal/git/write_gate.go`). Retry progress on quiet branches is fixed: a failed publication schedules its own bounded retry -(`internal/git/publication_retry.go`, [`push-cooldown.md`](design/push-cooldown.md) §7 option C). -Still open: when a rebuild's fetch fails, the open window is dropped, and unless the cause was a -missing parent nothing records the dropped scope or asks for a snapshot, so those writes wait for -the next resync. +(`internal/git/retry.go`, [`push-cooldown.md`](design/push-cooldown.md) §7 option C). A window whose +rebuild fails is no longer dropped: it stays pending for that retry +([`gittarget-branch-worker-pending-writes.md`](design/gittarget-branch-worker-pending-writes.md)). **Later, non-blocking:** diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index 40739f24..6d6cb31d 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -7,6 +7,101 @@ guidance that the changelog's breaking-change entries link to. We are pre-1.0, so breaking changes bump the **minor** version (release-please is configured with `bump-minor-pre-major`) rather than the major. Read the relevant entry before upgrading across it. +## `git_queue_drops_total` is now `git_queue_refusals_total` + +`gitopsreverser_git_queue_drops_total` is renamed `gitopsreverser_git_queue_refusals_total`, with the +same labels (`provider_namespace`, `provider_name`, `branch`, `kind`). A refused item is no longer +lost: the watch keeps its cursor and delivers a refused write again, the controller re-sends a save, +a refused resync is gathered again, and the next reconcile asks for a refresh again. Only a watch +cursor that expires before the write comes back costs something: that write's own commit, and under +`prune.mode: OnEvent` a deleted object's removal from Git. The counter counts both a full queue and +a branch that has paused intake through a Git outage. + +Update every dashboard and alert that names the old series; nothing emits it any more. The suggested +alert now fires on a healthy branch only: +`rate(gitopsreverser_git_queue_refusals_total[5m]) > 0 unless on(provider_namespace, provider_name, +branch) gitopsreverser_git_intake_paused == 1`, `for: 10m`. See +[Interpreting metrics](interpreting-metrics.md). + +## A `GitTarget` says when its branch cannot publish + +While a branch worker cannot publish to its Git remote (the remote cannot be reached, or it refuses +the push), every `GitTarget` on that branch reports `Ready=False` with reason `Progressing`, +`Reconciling=True` and `Stalled=False`. The message says since when, what the last attempt met, and +whether intake of new changes is paused. It used to report `Ready=True` through such an outage, +and, once intake paused, `Stalled=True` with `WatchError`, because the refused streams were graded +as broken watches. A stream waiting for its branch to reopen intake now reports `BranchIntakePaused`. +The message changes only when the cause or the pause does, so a long outage does not write status +on every retry. A missing parent branch is still reported as `ParentBranchNotFound`. + +`kubectl wait --for=condition=Ready` therefore waits through a Git outage, where it used to return, +and an alert on `Stalled=True` no longer fires for one. A save waiting for its push keeps +`WaitingForPush`, and its message now says why. + +New metrics describe the backlog without a status write per attempt: +`gitopsreverser_git_retained_bytes`, `gitopsreverser_git_retained_writes`, +`gitopsreverser_git_intake_paused`, `gitopsreverser_git_oldest_retained_write_timestamp_seconds`, +`gitopsreverser_git_next_retry_timestamp_seconds`, and the counter +`gitopsreverser_git_materialization_failures_total`. See +[Interpreting metrics](interpreting-metrics.md). + +## A `GitTarget` watches each object through one collection + +**Breaking.** One `GitTarget` may no longer watch a type both in all namespaces +(`sourceNamespace: "*"`) and in a named namespace, even with the same `objectSelector`; overlapping +collections with different selectors were already refused. The older rule keeps its collections; +the newer one is refused as a whole with `ResourcesResolved=False` and reason `CollectionOverlap`, +which replaces `ObjectSelectorConflict`. The message names both rules and both scopes. Exact +duplicates are still allowed and share one stream, and different `GitTarget`s never conflict. See +[overlapping collections](configuration.md#overlapping-collections). + +Before upgrading, find targets that hold a `"*"` item and a named-namespace item for the same type, +and remove the redundant one, or move the narrower one to its own `GitTarget`. Otherwise the newer +rule stops mirroring after the upgrade. Alerts or scripts that match `ObjectSelectorConflict` must +match `CollectionOverlap`. + +An update whose Git-visible content did not change is now also filtered right after a replay, so +`gitopsreverser_watch_events_total{outcome="unchanged"}` rises and fewer status-only updates reach +the branch worker. + +## A Git server that stops answering no longer holds a branch + +Every call to a Git server has a deadline: two minutes for one advertisement, fetch or push +session, and five minutes for a whole push cycle, contention retries included. A server that accepts +a connection and then stops answering used to hold the branch worker, and every `GitTarget` on that +branch, indefinitely: no saves, withdrawals or later writes were handled, and over SSH even a +shutdown waited for the connection to die. The same bound applies to the `GitProvider` controller's +connectivity check. + +A call that runs out of time fails like any other unreachable remote. The writes stay pending, a +save stays `WaitingForPush`, and the retry schedule attempts them again; the error names +`context deadline exceeded`. A push whose reply was lost after the server took it is recognized on +the next look at the remote, when the branch is still at the commits it sent, and published as is, +so a save's empty commit does not land twice. When another writer has pushed on top in the +meantime, the writes are replayed as for any other contention. + +## A save survives a remote that cannot be reached + +A `CommitRequest` whose window closes while the remote cannot be reached stays `WaitingForPush` and +resolves `Committed` once the remote is back, with its own message and author. It used to resolve +`Ready=False` with `FinalizeFailed`, and the window's changes waited for the next resync. The window +is kept as well when no request rides it, so `gitopsreverser_git_commit_failures_total` no longer +counts an unreachable remote. Automation that treated `FinalizeFailed` as "the remote is down, try +again" now sees the request wait instead. + +A configured `spec.parentBranch` that the remote does not carry holds writes back the same way: +they are kept and published once the parent exists, with their own authors and messages, instead of +being dropped and re-derived from a snapshot of the cluster. A save on such a target waits too. + +The kept work is bounded at intake. During an outage, work that would take a branch past its +retained-byte budget (`controllerManager.branchBufferMaxSize`) pauses the worker's intake: it refuses +new writes, saves and resyncs the way a full queue does until a push lands, so +`gitopsreverser_watch_events_total{outcome="route_failed"}` and +`gitopsreverser_git_queue_refusals_total` rise during such an outage. The budget counts queued work and +every kept item, empty saves included. A paused branch's watch streams wait for it to reopen instead +of reconnecting every two seconds, and deliver what was refused once it does. See +[Interpreting metrics](interpreting-metrics.md). + ## A save on a target that may not be written fails A `CommitRequest` on a suspended `GitTarget`, or on one whose render fidelity is not established, @@ -1527,9 +1622,8 @@ connections and a hundred list calls at warm-up, each with its own cursor and it apiserver watch cache; it is one of each now, and the saving grows with the cluster. And its failure mode is a clean 403 rather than a silently empty set. -A `"*"` item and a named-namespace item for the same type are **peers**, not duplicates. Each rule -carries its own `operations` filter, so a target holding both runs two streams over overlapping -objects. That is correct, not something to tune away. +A later release refuses a `"*"` item and a named-namespace item for the same type on one target; +see [A `GitTarget` watches each object through one collection](#a-gittarget-watches-each-object-through-one-collection). ## A GitTarget must cover exactly one kustomize render root diff --git a/docs/api-first-publication.md b/docs/api-first-publication.md index e0915c36..55146979 100644 --- a/docs/api-first-publication.md +++ b/docs/api-first-publication.md @@ -248,24 +248,24 @@ explains why these policies help with field authority without providing a concur ### Three separate recovery questions -The worker needs three pieces of state because a successful push cannot answer every question +The worker needs two pieces of state because a successful push cannot answer every question about its local checkout. | State | Meaning | Recovery consequence | | --- | --- | --- | | `baseTrusted` | The last observed base remains usable for optimistic planning | Fetch if untrusted and no writes are retained | -| `worktreeDirty` | A failed write may have left partial filesystem or index changes | Reset; replay retained writes if present | -| `replayRequired` | A reset discarded retained commits and their rebuild is incomplete | Block publication until the retained writes are rebuilt | +| `checkoutApplied` unknown | A failed write may have left partial filesystem or index changes | Reset; replay retained writes if present | +| `checkoutApplied` below the retained writes | A reset discarded retained commits and their rebuild is incomplete | Block commits and publication until the retained writes are rebuilt | “Trusted” does not mean the remote cannot have changed since the last observation. Detecting that change is the compare-and-swap's job. Similarly, a clean checkout can still be missing the commits represented by retained writes. The recovery rule is to fetch, reset, and replay before proceeding with retained work whose local -representation is unsafe. `replayRequired` is set **before** the reset rather than after it, +representation is unsafe. The count drops to zero **before** the reset rather than after it, because a reset can move the branch reference and then fail while rewriting the worktree: the commits behind the retained writes are already unreachable at that point, and a flag set only on -the success path would miss exactly that case. The cost of setting it too eagerly is one fetch on +the success path would miss exactly that case. The cost of dropping it too eagerly is one fetch on the next cycle. A fourth piece of state is not a flag: the trust above, like everything else the worker holds, is @@ -458,7 +458,7 @@ provider and branch so different workers do not hide one another's behavior. | Same counter, reason `bootstrap` | Instrumented bootstrap helper; currently called only by tests | | `gitopsreverser_git_pushes_total` and `gitopsreverser_git_push_retries_total` | Completed publication cycles and contention retries within them | | `gitopsreverser_git_push_duration_seconds` | Time inside a push cycle, including its inline replays | -| `gitopsreverser_git_queue_depth` and `gitopsreverser_git_queue_drops_total` | Outstanding work and queue overload | +| `gitopsreverser_git_queue_depth` and `gitopsreverser_git_queue_refusals_total` | Outstanding work and queue overload | A fetch counter counts calls. The ledger counts the HTTP requests each call produces. Push-cycle duration excludes diff --git a/docs/architecture.md b/docs/architecture.md index d76e8ee3..40eb92ca 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -65,11 +65,12 @@ Every write to that branch goes through the worker's single event loop and commi fetch and hard-reset before the first commit of every publication cycle. It no longer does: the push session reads the remote's ref advertisement on a connection the cycle was making anyway, and a cycle that commits nothing still reaches it, so a base the worker can vouch for needs no fetch to plan -against. Two flags on the worker carry that claim, and they are separate on purpose: `baseTrusted` -is dropped by anything that can no longer prove where the remote is, while `worktreeDirty` records -that a write failed part-way and is cleared **only by a reset** (a third, `replayRequired`, marks -retained writes whose local commits a reset discarded before the replay could rebuild them). -Collapsing them would let one kind of doubt clear another. Either flag makes the next cycle fetch. The +against. Two pieces of state on the worker carry that claim, and they are separate on purpose: +`baseTrusted` is dropped by anything that can no longer prove where the remote is, while +`checkoutApplied` counts how many retained writes the checkout holds on top of its root. A write +that failed part-way sets it to unknown, which **only a reset** clears; a reset sets it to zero, +which with writes retained says their local commits are gone and must be replayed. Collapsing the +two would let one kind of doubt clear another. Either one makes the next cycle fetch. The failure direction is safe by construction: a stale "untrusted" costs one fetch, while a stale "trusted" is caught by the compare-and-swap on the next push. For a new write branch, the worker checks the parent's advertised tip before sending the push. A moved parent triggers fetch, reset, and replay. @@ -501,9 +502,10 @@ Following the ConfigMap edit: `GitTarget`'s watch on `core/configmaps` in `team-a` delivers a `MODIFIED` event carrying the new object body. (On a cold start or after `410 Gone`, the same object instead arrives as an `ADDED` during the `sendInitialEvents` replay.) -2. **Relevance filter.** The event is sanitized (status, managedFields, and volatile metadata stripped), - checked for followability, and diffed against current Git content. A no-op (e.g. a `*/status` bump - whose desired-state projection is unchanged) is dropped here. +2. **Desired-state change filter.** The event is sanitized (status, managedFields, and volatile + metadata stripped) and compared with what the branch worker last accepted for this object on this + stream, from a live event or the stream's replay. A no-op (e.g. a `*/status` bump whose + desired-state projection is unchanged) is dropped here. 3. **Resolve the author.** When attribution is enabled, the resolver waits a bounded grace window (`--author-attribution-grace`, default `3s`) for a matching audit fact in the attribution index, joining by resourceVersion/UID. On a strong match the real user or named service account becomes the author. With @@ -1085,8 +1087,8 @@ and chooses one version per collection. The plan diff keeps unaffected watches r new collections, and replaces a collection only when its served version changes. Each managed watch has one goroutine. Its first attempt requests initial state, enqueues scoped -snapshot reconciliation, and then routes every live object event through unchanged-content -suppression and author resolution. +snapshot reconciliation, and then routes every live object event through its desired-state change +filter and author resolution. Later attempts may resume a cursor; see [Recovery: resume, replay, or list plus mark-and-sweep](#recovery-resume-replay-or-list-plus-mark-and-sweep). @@ -1267,12 +1269,12 @@ with no refresher. is untrusted or the worktree is dirty (see the ground rule above); a healthy publishing target plans straight onto its own last push. Once writes are retained the guard flips off entirely, because a reset would destroy the local commits those writes already produced. Anything that needs a fresh -tree with work in hand therefore resets **and replays** -([`refreshRemoteAndRebuildPendingWrites`](../internal/git/branch_worker.go)) rather than resetting -alone, re-planning the retained writes onto the new tip. Three things reach it: a forced recheck -calls it directly, while a worktree a failed write left dirty and the snapshot a resync judges -against go through `invalidateAndRefresh`, which drops base trust first because nothing has asked -the remote anything yet. +tree with work in hand therefore resets **and replays** rather than resetting alone, re-planning the +retained writes onto the new tip. One function on the event loop does that, +[`materialize`](../internal/git/branch_worker.go): every commit and push the loop makes runs it +first, and the loop's commit refuses a checkout that is not the projection of the retained writes. +A worktree a failed write left dirty, retained writes whose commits a reset discarded, a parent +change, and the snapshot a resync judges against all reach it. **What a reset has to leave behind, now that it is the only cleanup.** A hard reset restores tracked files and stops there, so a document a failed write created, and the placement directory it created @@ -1282,10 +1284,41 @@ explicitly discards untracked and newly staged leftovers and prunes the director ([`discardWorktreeLeftovers`](../internal/git/git.go)), and a write that fails while creating directories removes the ones it made on its way out. Only then is `worktreeDirty` cleared. +### When the remote cannot be reached + +A decided write is kept until a push lands. When a commit or a push cannot reach the remote, the +worker keeps every decided write pending ([`pending_writes_loop.go`](../internal/git/pending_writes_loop.go)), with +its author, its message, and any save riding it, and tries again on one schedule: 10 seconds, +doubling to 5 minutes ([`retry.go`](../internal/git/retry.go)). The retry does not wait for another +edit. Until it is due nothing else on the branch spends a connection: a write that needs one stays +pending, and a resync is answered with the failure the retry is waiting out and kept. Only a +failure of the write itself, such as a refused plan, drops an entry. Every call to a Git server is +bounded, two minutes per call and five per push cycle, so a server that stops answering cannot hold +the branch. + +The kept work is bounded at intake ([`intake.go`](../internal/git/intake.go)). While a retry is +pending, `--branch-buffer-max-size` counts everything accepted and not yet published: queued items, +the open window, the pending writes, deferred snapshots and waiting saves, each with a 1 KiB per-item charge. +The first payload that does not fit pauses the branch's intake until a push lands; room under the +budget again, or a remote that can be read but refuses the push, keeps it paused. A paused branch's +watch streams wait to be woken instead of reconnecting, and their cursors stay where the last +accepted event left them. Withdrawals, refreshes, and shutdown are lifecycle work and are never +paused. The budget counts serialized bytes; a queued object costs several times that in memory. + +Every `GitTarget` on the branch reports the outage from one worker report +([`publication.go`](../internal/git/publication.go)): `Ready=False`, `Reconciling=True`, +`Stalled=False` under `Progressing`, naming since when, what failed, and whether intake is paused. A +held save says the same in its `WaitingForPush` message, and the `git_retained_*`, +`git_intake_paused` and `git_next_retry_timestamp_seconds` metrics describe the backlog. What this +recovers, and what watch history and a restart cannot, is the +[recovery contract](design/gittarget-branch-worker-pending-writes.md#recovery-contract); the +[event pipeline overview](design/event-pipeline-overview.md) draws it. + ### Durability of the write queue (planned) A BranchWorker's queue (the open commit window's retained writes plus any local commits not yet pushed) -lives **only in process memory today**. It is about to be **materialized into Redis**, for two reasons: +lives **only in process memory today**, bounded as described above. Persisting it to **Redis** is +planned, for two reasons: - **High availability.** Multi-pod HA needs a durable, cross-pod write queue so a branch's accepted-but-unpushed work survives a failover: the pod that takes the branch-shard lease resumes that @@ -1505,12 +1538,12 @@ as one funnel and its loss paths as one selector: - **Ingest.** `gitopsreverser_watch_events_total{gittarget_*,group,version,resource,outcome}` counts every delivered watch event exactly once, at `routeLiveTargetWatchEvent`, the single switch carrying every terminal branch. `outcome` separates the pipeline working (`routed`, `unchanged`, - `bookmark`) from loss (`route_failed`). Beside it, + `bookmark`) from a refusal the watch delivers again (`route_failed`). Beside it, `_watch_event_handling_seconds` (stream occupancy, the head-of-line signal), `_watch_sessions_ended_total{reason}` and `_watch_replay_duration_seconds` (`410` pressure and what a rebuild costs), `_watch_recovery_total{mode}`, and `_watch_types{state}`. -- **Queue.** `gitopsreverser_git_queue_drops_total{kind}` counts work a full worker queue threw - away, and `_git_queue_depth` is read at scrape time rather than published by the worker loop. +- **Queue.** `gitopsreverser_git_queue_refusals_total{kind}` counts work a worker refused at enqueue, + which its producer offers again, and `_git_queue_depth` is read at scrape time rather than published by the worker loop. - **Write.** `gitopsreverser_git_documents_total{gittarget_*,group,version,resource,outcome}` is the per-document census (`written`, `deleted_live`, `deleted_sweep`, `unchanged`, `retained`), with `_placements_total`, `_placement_refusals_total` and `_placement_kustomization_entries_total` diff --git a/docs/configuration.md b/docs/configuration.md index 29db9e62..9b091d1d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -613,6 +613,10 @@ The most useful status fields are: - `Ready`: true when the target is valid, the Git path is accepted, and watched streams are running. - `Reconciling`: true while initial replay, a recheck, or another coarse progress step is in flight. - `Stalled`: true when the target is blocked until a human fixes configuration, RBAC, or Git path content. +- While the branch cannot publish to Git, `Ready=False` with reason `Progressing` says since when, + what failed, and whether intake of new changes is paused; `Reconciling=True` and `Stalled=False`, + because the retry needs nobody. Every target on the branch says the same until a push lands, and + its streams report `BranchIntakePaused` while they wait. - `Validated` and `EncryptionConfigured`: control-plane details. - `StreamsRunning`: true when the source watches are past initial replay and routing live events. - `GitPathAccepted`: true when the target Git path is safe to materialize. @@ -775,9 +779,8 @@ spec: - It recovers on its own once the parent is pushed, with no edit to the cluster and with periodic refresh on or off. The branch worker looks for the parent with one ref advertisement after 10s, then after twice as long each time, up to every 5 minutes, shared by every target on the branch. - When the parent is back it publishes the writes it held and asks for a fresh snapshot of the ones - it had to drop; until those are published the target reports `Ready=False` with reason - `RecoveringParentBranch`. + When the parent is back it publishes the writes it held, with their own authors and messages; + until those are published the target reports `Ready=False` with reason `RecoveringParentBranch`. - An omitted parent is the remote's default branch, as last discovered. In an empty repository (no refs at all, tags included) the first commit starts a branch with no history. A repository that is not empty but whose `HEAD` names no branch it carries also reports `ParentBranchNotFound`, and @@ -1778,9 +1781,9 @@ namespace. Connection count is bounded per matched type, but object count, traff still grow with the cluster. Access is bounded by the source credential's RBAC and by nothing else, which is why it is refused outright while `allowAnySourceNamespace` is false. -A `"*"` item and a named-namespace item for the same type are **peers**, not duplicates: each is its -own resource collection, so a target holding both runs two streams over overlapping objects. That is -correct rather than something to tune away. +A `"*"` item and a named-namespace item for the same type **overlap**, so one `GitTarget` cannot +hold both: the newer rule is refused (see [overlapping collections](#overlapping-collections)). Keep +the wider scope, or send the narrower one to its own `GitTarget`. The outcome for all items is aggregated into one `SourceNamespaceAuthorized` condition, also shown by `kubectl get watchrules -o wide`. A **denied** explicit name refuses the whole `WatchRule` @@ -1883,15 +1886,6 @@ Consequences to plan for: before writing, such as `kustomize.toolkit.fluxcd.io/name`; the labels do not need to survive in Git. -**Overlapping collections share one selector.** Two items on one `GitTarget` overlap when they -select the same type in the same namespace, or when one of them selects all namespaces. Overlapping -items must use the same selector, written in any equivalent form: `matchLabels: {team: a}` and -`matchExpressions: [{key: team, operator: In, values: [a]}]` are one selector. When they differ, the -older rule, then the rule with the lower name, keeps its collections, and the newer rule is refused -as a whole with `ResourcesResolved=False`, reason `ObjectSelectorConflict`. Different selectors in -different namespaces are fine. To select several values, use one set-based selector such as -`team in (a,b)`. - The field is the Kubernetes `LabelSelector` type, so the CRD schema checks only its shape. The rule compiler checks the rest (operators, values, and label syntax, exactly as Kubernetes parses a selector) and refuses the whole rule before it opens any watch, with `ResourcesResolved=False`, @@ -1917,6 +1911,21 @@ authored unresolved; a deletion that waited on a finalizer was already attribute requested. The document is removed either way. See the [attribution contract](spec/attribution.md#four-rules-that-are-easy-to-miss). +### Overlapping collections + +A `GitTarget` watches each object through one collection. Two items on one `GitTarget` overlap +when they select the same type in the same namespace, or when one of them selects all namespaces. +Only an exact duplicate is allowed: the same type, namespace scope, and selector, written in any +equivalent form (`matchLabels: {team: a}` and +`matchExpressions: [{key: team, operator: In, values: [a]}]` are one selector). It shares one stream. +Any other overlap is refused, whether the selectors are equal or differ: the older rule, then the +rule with the lower name, keeps its collections, and the newer rule is refused as a whole with +`ResourcesResolved=False`, reason `CollectionOverlap`. The message names both rules and both +scopes. A rule whose own items overlap is refused whatever its age. Different namespaces, and +different `GitTarget`s, never overlap. To select several values, use one set-based selector such as +`team in (a,b)`. The check re-runs when rules change or the source cluster starts serving a type. A +collection that leaves the plan stops before an overlapping one that replaces it starts. + ## `ClusterWatchRule` `ClusterWatchRule` is the **cluster-scoped** variant. Use it for cluster-scoped resources such as @@ -2077,7 +2086,7 @@ reason, without changing Git. | Only another author's window was open | no commit, `WindowMismatch` | no commit, `WindowMismatch` | | The target is suspended, or its render fidelity is not established | `FinalizeFailed`, the cause | `FinalizeFailed`, the cause | | The commit or the empty commit failed | `FinalizeFailed` | `FinalizeFailed` | -| The push failed | `WaitingForPush` until a push lands; `FinalizeFailed` if the worker stops first | the same | +| The push failed | `WaitingForPush`, with the cause, until a push lands; `FinalizeFailed` if the worker stops first | the same | The `Ready` reason keeps the cause, and `status.commit` with `Pushed=True` says the empty commit reached the remote. A hash proves the message is in Git; `NoWindow` says the request saw no writes, diff --git a/docs/definitions.md b/docs/definitions.md index 789611c5..9860e4c7 100644 --- a/docs/definitions.md +++ b/docs/definitions.md @@ -104,6 +104,13 @@ and storage call it a cursor. The first attempt after startup or watch replaceme current state even if a stored cursor exists; later attempts in that watch's loop may resume. A cursor records observation progress, not successful Git publication. +**Desired-state change filter.** Passes creates, deletes, and updates that change Git-visible +desired state, including data and retained labels and annotations. Drops updates whose sanitized +content matches the accepted baseline; a missing baseline passes. The metric outcome is +`unchanged`. Code: `desiredStateChangeFilter`, one per watch stream, seeded from the stream's +accepted replay; see +[step 5a2](design/gittarget-branch-worker-pending-writes.md#step-5a2-desired-state-change-filter-built). + **Catalog.** One source cluster's normalized discovery result. It holds no judgment about whether a type should be watched. @@ -145,6 +152,13 @@ replaces the window's timers with its own. A request attaches to at most one win one. One word for the act, in the API (`window.attach`, `attachTimeout`), the code and the docs: not "claim", which names a rule's demand on the read side. +**Pending write.** A write a branch worker has decided to make and not yet published: a closed +commit window, an atomic batch, a resync, a save's empty record, or a refusal's empty commit. The +branch worker keeps its pending writes in order, commits them, and pushes them on one retry +schedule. A pending write can be waiting to become a commit, committed locally and waiting for a +push, or replayed onto a remote that moved. `PendingWrite` and `pendingWrites` in Go. Not "log", +which also names diagnostic logs and suggests a durable journal, which this is not. + **Publication.** One commit-and-push cycle by a branch worker. The thing a `GitTarget` reports in `status.remote`. @@ -372,6 +386,7 @@ outage. | stream (for the managed lifecycle) | watch | reserve watch stream for delivered events; configuration kinds keep their full names | | initial sync, backfill, replay (for current state) | initial events or initialization | follows Kubernetes streaming-list terminology | | git worker, commit worker | branch worker | the tuple it owns is keyed by branch | +| log, journal (for what a branch worker holds) | pending writes | they are in memory and not a record of history; "log" also names diagnostic logs | | attribution (bare) | author attribution, render attribution | two unrelated concepts share the word | | `GitRepoConfig` | `GitProvider` | the kind was renamed; the string survived in two reasons | | failure, error (for a declined write) | refusal | the operator worked correctly | diff --git a/docs/design/branch-worker-event-model.md b/docs/design/branch-worker-event-model.md index 526ea30c..58c2f497 100644 --- a/docs/design/branch-worker-event-model.md +++ b/docs/design/branch-worker-event-model.md @@ -1,8 +1,9 @@ # Branch worker events, durable execution, and recovery A single failed push on a quiet branch used to leave a `CommitRequest` in `WaitingForPush` -indefinitely, even after Git recovered. #412 repairs the retry half of that liveness gap. Bounding -Git operations is the next step, with the branch worker still the owner of its checkout. +indefinitely, even after Git recovered. #412 repaired the retry half of that liveness gap, and #413 +finished the in-process half: bounded Git calls, a bounded outage backlog with explicit pause and +resume, and a status that says so. The branch worker remains the owner of its checkout. The longer-term direction is recoverable execution of accepted writes, save commands, window decisions, deadlines, and publication outcomes. This document owns the worker's event semantics, @@ -11,38 +12,49 @@ current failure analysis, and immediate implementation scope. The journal storage, acknowledgments, publication recovery, retention, and the persistence/HA rollout. The broader ownership contract is in [the architecture](../architecture.md#git-write-architecture). -Status: design proposal against the code after #407, #411 and #412 (publication retry, built). -Current-behavior sections describe that implementation. Event names and the transition boundary -are proposals. This document changes no runtime behavior and does not establish an outage or -exactly-once guarantee. +Status: design proposal, aligned 2026-10-04 with #413, which built every step of the +[pending-writes plan](gittarget-branch-worker-pending-writes.md): ordered pending writes and one materializer, one retry +schedule, the watch's admission boundary, Git call deadlines, intake pause and resume, and the +publication status. Event names and the transition boundary are proposals. This document changes +no runtime behavior and does not establish a restart or exactly-once guarantee. -## Next implementation: restore publication progress +## Publication progress (built in #412 and #413) -Complete retry scheduling and add Git operation deadlines before the journal work. Neither fix -depends on choosing a storage backend, changing the save contract, or implementing HA. +The write-path refactor in [`gittarget-branch-worker-pending-writes.md`](gittarget-branch-worker-pending-writes.md) built +what this section asked for, in the order it set: ordered pending writes, one materializer for the +checkout, and one retry deadline; then the watch's admission boundary (5a), Git operation deadlines +(6), intake pause and resume (5b), and the publication status (5c). None of it depends on a storage +backend or on HA. A saturated branch pauses intake while its recovery loop stays alive; a process +restart still loses the in-memory obligations. -1. **Built in #412.** Retained publication work has a bounded failure backoff (10s, doubling to - 5m) on the push timer, in [publication_retry.go](../../internal/git/publication_retry.go). +What comes next is the [transition boundary](#prepare-the-transition-boundary-for-later-durability): +recording timers, attaches, withdrawals, and refreshes as transitions, then the HA plan's journal. + +1. **Built in #412, one schedule since #413.** Retained publication work has a bounded failure + backoff (10s, doubling to 5m) on its own timer, in [retry.go](../../internal/git/retry.go). Commits before the deadline do not push; a success resets it. While parent recovery is open - its probe deadline is the retry, and new commits wait for it too, including after the parent + the same deadline is its probe, and new commits wait for it too, including after the parent returns. A stopped worker's timers are stopped with it. -2. Carry cancellation through ref listing and fetch, and impose operation deadlines across the - push cycle. Bound the cycle's total occupancy as well as its individual calls. Repeated - contention retries must not repeatedly reset the whole budget. Keep timed-out work pending; - a lost push response can mean the remote already accepted it. -3. Preserve held-save semantics. The worker must resolve publication or establish that the save - cannot publish before it reports a terminal failure. A controller timeout alone cannot prove - that. Keep `WaitingForPush` while a recoverable publication obligation remains. -4. Use an injectable clock and scheduler to test silence, continuous arrivals, cancellation, and - stale firings. Keep one owner of the checkout while testing the transport cancellation path. - Returning from a timeout wrapper while a goroutine still mutates the checkout is unsafe. +2. **Built in #413 (step 6).** Ref listing, fetch, and push carry the caller's context; each call + has a two-minute bound and the whole push cycle five minutes, contention retries included. A + dialer closes the connection when its call's context ends, because go-git's SSH transport + ignores the context past the dial. Timed-out work stays pending, and a push whose reply was lost + is settled as published when the remote is found at the commits it sent. +3. **Kept.** The worker resolves publication, or establishes that the save cannot publish, before it + reports a terminal failure; a controller timeout alone cannot. `WaitingForPush` holds while a + recoverable publication obligation remains, and since step 5c its message says why. +4. **Built.** The retry schedule takes an injectable clock; the stall tests run HTTP and SSH + servers that stop answering, and the call returns on the worker goroutine, which stays the only + owner of the checkout. The completion test is a save whose first push fails, followed by silence and remote recovery: without another resource edit, its commit reaches Git and the same request reaches its terminal status with the published SHA. `TestPublicationRetry_AHeldCommitRequestIsCommittedWithoutAnotherWrite` covers that on the event -loop. Still to prove that a request can safely outlive the controller's safety window while -retries continue, and that a stalled remote call returns control within the operation budget. +loop. `TestCommitRequestReconcile_AHeldRequestOutlivesTheBound` covers the controller's safety +window. `TestGitNetworkCalls_HTTPStallsReturnAtTheCallBound` and +`TestGitNetworkCalls_SSHStallsReturnAtTheCallBound` prove that a stalled remote call returns control +within the operation budget. Use a controllable remote for those failures and a fake clock for retry timing. This change improves progress within a running worker. Restart durability follows the separate @@ -89,14 +101,16 @@ not enforce that exactly one alternative is set. | Input | Producer or entry point | Loop handler | |---|---|---| | Live resource write | Watch stream through `Enqueue`; `CommitModePerEvent` | `handleLiveEvents` | -| Atomic resource batch | `EnqueueRequest`; `CommitModeAtomic` | `handleAtomicRequest` | +| Atomic resource batch | Unit tests only, through the unexported `enqueueRequest`; `CommitModeAtomic` | `handleAtomicRequest` | | Attach a save | `CommitRequest` controller through the event router | `handleAttachCommitRequest` | | Withdraw a save | `CommitRequest` controller through the event router | `handleWithdrawCommitRequest` | | Complete scoped snapshot | Watch replay through `EnqueueResync` | `handleResyncRequest` | | Observe remote and folder | `GitTarget` reconcile through `EnqueueRefresh` | `handleRefreshRequest` | This inventory is the starting point for extracting explicit transitions. The atomic handler is -supported, but this review found no non-test producer. `Refresh` normally observes an idle branch; +supported, but it has no non-test producer; step 5a of the +[pending-writes plan](gittarget-branch-worker-pending-writes.md#step-5a-make-refused-admission-safe-built) removed the +unused exported `EnqueueRequest`. `Refresh` normally observes an idle branch; during parent recovery its handler can service a due probe and publish retained work. Timer channels, shutdown, synchronous Git results, and shared configuration are additional inputs @@ -182,49 +196,52 @@ therefore defers to recovery's next probe deadline: new commits wait for it, and makes the attempt. Review of #412 found the handoff gap (each new commit retried at once); `TestPublicationRetry_AFailureAfterTheParentReturnsWaitsForRecovery` pins the fix. -The backoff schedules publication attempts, not every Git connection. A new window can still -fetch to rebuild retained writes before reaching `maybeSchedulePush`; if the rebuild fails, the -window is dropped and its attached save fails. Except for a missing parent, no dropped-scope -recovery requests a replacement snapshot. This existing gap belongs beside the deadline work, -before the durable-journal refactor. +The backoff schedules publication attempts, not every Git connection. A new window used to fetch +to rebuild retained writes before reaching `maybeSchedulePush`, and a failed rebuild dropped the +window and failed its save. A closed window is now a pending write +([`gittarget-branch-worker-pending-writes.md`](gittarget-branch-worker-pending-writes.md), step 2): an unreachable remote +leaves it for the publication retry with its save held. Decisions needing a connection wait, +resyncs included since step 5b: a resync is answered with the failure the retry is waiting out and +kept. Only a failure of the write itself drops an entry; a missing parent holds it back like +another remote failure. + +Since step 5c every `GitTarget` on the branch reports a publication failure, and a held save in +`WaitingForPush` carries the same cause, including a push-specific refusal such as branch +protection with working read access. `gitopsreverser_git_materialization_failures_total` counts a +rebuild that fails before entering the push cycle, which the push counter cannot see. -There is no worker publication-failure condition or retry deadline in `GitTarget` status. A -push-specific refusal, such as branch protection with working read access, can leave a save in -`WaitingForPush` without explaining the cause. Push-cycle failures appear in logs and -`gitopsreverser_git_pushes_total{outcome="failed"}`. A rebuild that fails before entering the push -cycle is not counted by that metric. Provider connectivity or credential failures can separately -surface through `GitProviderReady`, and parent recovery already has status. +### Slow Git operations stop the whole branch loop (bounded in #413) -### Slow Git operations stop the whole branch loop +Since step 6 every call has a deadline, so this delay ends within two minutes per call and five per +push cycle. The description below is what remains true inside that bound. Git calls run synchronously. While one is running, the branch cannot handle queued saves, withdrawals, snapshots, deadlines, or later writes. Every target on that worker shares the delay. Producers continue admitting work until the FIFO fills. -The gap covers both remote observation and fetch. In -[smart fetch](../../internal/git/git_smart_fetch.go), `listRemoteRefs` calls `remote.List` and -`SmartFetchFrom` calls `repo.Fetch` without passing the worker context to either network call. -The `ctx` parameter on `SmartFetchFrom` does not make those operations cancelable by the worker. -The separate connection check in [git.go](../../internal/git/git.go) also uses `remote.List`. - -The [atomic push](../../internal/git/git_atomic_push.go) does pass context through handshake, -advertisement, and upload, but `internal/git` sets no operation deadline. Worker shutdown -cancellation is not an elapsed-time budget. These code findings do not establish that every -transport lacks its own timeout; they establish that the worker has no end-to-end bound. -A timer on the event loop cannot interrupt a call that has not returned. +Before step 6, ref listing and fetch ran without the worker's context, and the push carried a +context with no deadline. [`network_bound.go`](../../internal/git/network_bound.go) now gives every +call a bound and closes an SSH connection when the call's context ends. A timer on the event loop +still cannot interrupt a call; the bound is what returns it. The values are fixed. -### Admission is volatile, and retained memory has no outage bound +### Admission is volatile (the outage bound is built in #413) Successful enqueue means an item entered process memory. The watch path can advance its resume -cursor before the write reaches Git. A crash between those steps can make a replacement resume -past work that existed only in RAM. The local checkout is disposable. +cursor before the write reaches Git. A crash loses that accepted execution state. A new stream +starts with a fresh replay, which can repair current content under its write/prune policy, but +cannot restore lost save decisions or intermediate history. The local checkout is disposable. The FIFO defaults to 1,000 items. The default 8 MiB branch buffer threshold closes a window early, -but closing moves its data into `pendingWrites`. Failed pushes retain that data. Queued payloads -and deferred snapshots also consume memory, so neither setting bounds total outage retention. +and since step 5b it is also the outage budget: while a retry is pending it counts everything +accepted and not yet published, queued payload, empty records, deferred snapshots, and waiting +saves included, and the first payload that does not fit pauses intake until a push lands. The +budget counts serialized bytes, not heap; on a healthy branch the FIFO is bounded by count only. Queue-full handling differs by input. Live writes return a refusal to the watch producer, which -keeps its cursor and reconnects. Attaches and withdrawals can be resent. Refreshes can be retried +keeps its cursor; on a paused branch it waits to be woken instead of reconnecting. Since step 5a the +filter records a baseline only after the worker accepted the event, so a refused UPDATE is offered +again on resume. An expired cursor requires a fresh scoped snapshot, with missed history and +deletions limited by prune policy. Attaches and withdrawals can be resent. Refreshes can be retried by a later reconcile. The supported atomic enqueue API logs a dropped request without returning an acceptance result. A durable admission contract needs explicit receipts for every supported producer. See the [durable queue backlog](../TODO.md). @@ -398,15 +415,15 @@ The current worker has five timer sources: | Deadline | Existing purpose | |---|---| | Commit window | Close on idle timeout or maximum duration | -| Push timer | Space successful publication cycles, or retry a failed one on its backoff | +| Push timer | Space successful publication cycles | +| Retry | Retry a failed publication on its backoff, or probe for a missing parent | | Attach timeout | Settle a save still waiting for an eligible window | | Refusal action | Recheck consent before an eligible empty commit | -| Parent recovery | Probe and service retained work or owed snapshots | -The publication retry (#412) shares the push timer with the success cooldown and keeps its own -backoff. Parent recovery has a separate timer, and owns the retry while its obligation is open. -A durable design must persist both deadlines and that ownership rule. This scheduling does not -limit the fetches other handlers can initiate. +The retry schedule has its own timer; the push timer is only the success cooldown. Parent +recovery has no timer of its own since #413: while its obligation is open, the retry deadline is +its probe. A durable design must persist that one deadline and what it is for. This scheduling +does not limit the fetches other handlers can initiate. An illustrative deadline protocol is: @@ -499,17 +516,19 @@ Journal crashes, retention, encryption, and HA failure tests belong to the | Scenario | Required observation | |---|---| -| Save's first push fails, then the branch goes quiet | Retry publishes and resolves the same request without another edit | -| Held save outlives the controller's safety window | Withdrawal cannot falsely fail it; recovery remains scheduled | -| Push fails under continuous arrivals | Arrivals cannot bypass the retry budget | -| Parent returns, but publication still fails | New commits respect the recovery deadline while the obligation remains open | -| Retained-write rebuild fails before a new window commits | Recovery accounts for the dropped window and its attached save | -| Ref listing, fetch, or push stalls | Operation and cycle deadlines return control to the worker | +| Save's first push fails, then the branch goes quiet | Built (#412): retry publishes and resolves the same request without another edit | +| Held save outlives the controller's safety window | Built (#412): withdrawal cannot falsely fail it; recovery remains scheduled | +| Push fails under continuous arrivals | Built (#412): arrivals cannot bypass the retry budget | +| Parent returns, but publication still fails | Built (#413): new commits respect the one retry deadline while the obligation remains open | +| Retained-write rebuild fails before a new window commits | Built (#413): decided window and attached save stay pending for retry | +| Intake fills during publication failure | Built (#413): accepted work stays; producers pause while the retry loop remains active | +| Rejected UPDATE is redelivered from the same cursor | Built (#413): dedup does not suppress work that never entered the worker | +| Ref listing, fetch, or push stalls | Built (#413): operation and cycle deadlines return control to the worker | | Worker is canceled during a failed attempt | No retry is rearmed and no abandoned task retains checkout access | | Timer moves or fires twice | Stale and duplicate firings do not close another window | | Recorded decisions are replayed | Identical state returns without external effects | | Fresh snapshot follows an accepted save | Coalescing preserves the save's membership and obligation | -| Parent disappears again during recovery | Earlier snapshot publication cannot clear newly owed work | +| Parent disappears again during recovery | Earlier publication cannot clear newer retained obligations | | Policy changes during a window or push | Its specified effective point remains unchanged | | Resource update and save arrive on independent paths | Only the documented attachment ordering is claimed | @@ -534,8 +553,8 @@ inspected production replay producer uses `Heal=false`. [Refresh handling](../../internal/git/refresh.go) normally skips a busy checkout. During parent recovery it can service a due probe and publish retained work. Disabling periodic refresh does -not disable commit windows or worker recovery. Keep `baseTrusted`, `worktreeDirty`, and -`replayRequired` separate: a clean checkout can still have lost the commits behind retained work. +not disable commit windows or worker recovery. Keep `baseTrusted` and `checkoutApplied` separate: +a clean checkout can still have lost the commits behind retained work. The detailed design records continue to own adjacent policy: diff --git a/docs/design/event-pipeline-overview.md b/docs/design/event-pipeline-overview.md new file mode 100644 index 00000000..4a2acea8 --- /dev/null +++ b/docs/design/event-pipeline-overview.md @@ -0,0 +1,576 @@ +# Event pipeline overview: from a watch event to a Git commit + +> **Snapshot, 2026-10-04, at the end of #413.** A picture-first tour of how a change +> in the cluster becomes a commit today, with the replay paths drawn out and an honest list of +> what is still missing. It describes behavior that exists. The plan that changes it is +> [`gittarget-branch-worker-pending-writes.md`](gittarget-branch-worker-pending-writes.md); the longer-term event model +> is [`branch-worker-event-model.md`](branch-worker-event-model.md). + +## 1. The whole pipeline on one page + +```mermaid +flowchart LR + subgraph SRC["Source cluster"] + API["kube-apiserver"] + AUD["audit webhook"] + end + + subgraph WATCH["Watch plane, one stream per (GitTarget, collection)"] + WS["target watch stream
replay, live, resume"] + DD{"desired-state
change filter"} + AT["attach author
from audit facts"] + PG["producer gate
(stream retired?)"] + CUR[("durable cursor
per GitTarget UID + collection")] + end + + subgraph ROUTE["Routing"] + ER["EventRouter"] + GES["GitTargetEventStream"] + end + + subgraph CTRL["Controllers"] + CRC["CommitRequest controller"] + GTC["GitTarget controller"] + end + + subgraph BW["Branch worker, one per (GitProvider, branch)"] + ADM{"intake
queue room and
budget gate"} + FIFO[["FIFO
default 1,000 items"]] + LOOP(("event loop
single goroutine")) + WIN["open commit window"] + PW[("pending writes, in order
in memory")] + MAT["materialize
(the only commit path)"] + CO[("local checkout")] + PUB["publish
(atomic push, CAS)"] + end + + REMOTE[("Git remote")] + + API -- "watch frames" --> WS + AUD -. "who did it" .-> AT + WS --> DD + DD -- "no-op UPDATE: skip" --> CUR + DD -- "changed" --> AT --> PG --> ER --> GES --> ADM + ADM -- "accepted" --> FIFO + ADM -. "refused: session ends,
stream waits for intake" .-> WS + GES -. "accepted: record content,
then the cursor" .-> CUR + WS -- "replay snapshot (resync)" --> ADM + CRC -- "attach / withdraw a save" --> ADM + GTC -- "refresh" --> ADM + FIFO --> LOOP + LOOP --> WIN -- "window closes" --> PW + LOOP -- "resync, save record,
refusal touch, atomic" --> PW + PW --> MAT --> CO --> PUB --> REMOTE + REMOTE -. "rejected: replay onto new tip" .-> MAT +``` + +Read it left to right: + +1. **One watch stream per collection.** A `GitTarget` that watches ConfigMaps in `apps` has one + stream for `(configmaps, apps)`. It starts with a replay of current state and then streams live + changes. +2. **A live change is checked, attributed, and routed.** The desired-state change filter passes + creates, deletes, and changes to Git-visible content, including retained metadata. It drops + unchanged UPDATEs, counted as `watch_events_total{outcome="unchanged"}`. The author comes from + audit facts. The producer gate prevents retired streams from enqueueing behind their replacements. +3. **The branch worker admits it or refuses it.** Admission is non-blocking: a full FIFO, a + stopping worker, or a paused intake gate refuses the item. A refused live event travels back up + as an error, the session ends, and the cursor stays put. When the branch has paused intake, the + stream waits for it to reopen before it reconnects. +4. **One goroutine owns everything after the FIFO.** It collects live events into a commit window, + decides pending writes, materializes them into commits, and publishes them. + +Several `GitTarget`s that write to the same provider and branch share one worker, and with it one +FIFO, one set of pending writes, one checkout, and one retry schedule. + +## 2. The watch stream: replay, live, and resume + +```mermaid +stateDiagram-v2 + [*] --> Replaying: stream started
(always replays first) + + Replaying --> Replaying: Added/Modified folded
into the desired set + Replaying --> SnapshotOffered: initial-events-end bookmark + Replaying --> ListFallback: sendInitialEvents unsupported + + ListFallback --> SnapshotOffered: LIST complete
(watch opened first, buffered) + + SnapshotOffered --> Live: worker accepted the resync
record cursor, mark stream replayed + SnapshotOffered --> Backoff: worker refused the resync
(no cursor, not replayed) + + Live --> Live: event accepted
record cursor + Live --> Backoff: event refused, watch error,
or server closed the watch + Live --> Backoff: 410 Gone (cursor expired) + + Backoff --> WaitingForIntake: the branch has paused intake + WaitingForIntake --> WaitingForIntake: woken, but paused again
(or 1 min fallback recheck) + WaitingForIntake --> Backoff: intake reopened + + Backoff --> Replaying: 2s later, stream has NOT replayed yet + Backoff --> Resuming: 2s later, stream has replayed + Resuming --> Live: watch opened from stored cursor + Resuming --> Replaying: 410 Gone on open
(fresh replay) + + Live --> [*]: stream retired or GitTarget gone +``` + +Four rules hold this together: + +- **A new stream always replays.** Starting a stream issues a fresh render-fidelity revision for + its collection; resuming an old cursor would never report that revision, so the scope would stay + pending forever. +- **A stream resumes only after its own replay was accepted** (step 5a). If the worker refuses the + snapshot, the stored cursor still belongs to a previous stream, so the reconnect replays again. +- **The cursor only moves past what the worker accepted.** A refused event ends the session before + the cursor is written; an event that a retiring stream never enqueued records no cursor at all. +- **A stream waits for a paused branch** (step 5b). Before every reconnect the stream asks + its branch worker whether intake is paused, and if so waits on a channel the worker closes when + it reopens. Reconnecting on the 2s backoff would gather a snapshot the branch refuses again, + for as long as the outage lasts. + +### The step 5a case, as a sequence + +```mermaid +sequenceDiagram + autonumber + participant API as kube-apiserver + participant S as watch stream + participant D as desired-state change filter + participant W as branch worker + participant C as cursor store + + API->>S: ADDED cm rv=10 data=A + S->>D: check: new object + S->>W: Enqueue + W-->>S: accepted + S->>D: record hash(A) + S->>C: cursor = 10 + + API->>S: MODIFIED cm rv=11 data=B + S->>D: check: B != A, changed + S->>W: Enqueue + W-->>S: refused (queue full or intake paused) + Note over S,D: nothing recorded:
the baseline stays A + S--xS: session ends with an error, cursor stays 10 + + Note over S,W: if intake is paused: wait until the worker reopens it + Note over S: 2s backoff, resume from rv=10 + API->>S: MODIFIED cm rv=11 data=B (redelivered) + S->>D: check: B != A, still a change + S->>W: Enqueue + W-->>S: accepted + S->>D: record hash(B) + S->>C: cursor = 11 +``` + +Before step 5a, the hash of B was recorded at step 8, so the redelivered frame matched it, was +skipped as unchanged, and the cursor moved to 11 without the worker ever seeing B. + +### The desired-state change filter + +The filter keeps status churn out of the FIFO. It also protects commit windows: `/status` updates +have no audit author, so passing one can close a named author's window even when Git has no diff. + +Each stream owns its baselines: a hash per UID of what the worker last accepted, from a live event +or from the stream's accepted replay snapshot, which replaces them all. A refused or unfinished +replay installs nothing. A `GitTarget` cannot hold two overlapping collections, so one stream is the +only producer for an object. See +[step 5a2](gittarget-branch-worker-pending-writes.md#step-5a2-desired-state-change-filter-built). + +## 3. Three different things called "replay" + +The word covers three mechanisms that guarantee different things. Most confusion about recovery +comes from mixing them up. + +```mermaid +flowchart TB + subgraph R1["1. Watch replay: re-observe the cluster"] + direction LR + A1["sendInitialEvents watch
or LIST + buffered watch"] --> A2["desired set for one collection"] + A2 --> A3["ResyncRequest
(scope = collection)"] + end + + subgraph R2["2. Resync: apply a snapshot as a write"] + direction LR + B1["resync decided as a pending write
(fetches first: judged against
the newest remote tree)"] --> B2["mark-and-sweep plan
under the target's prune mode"] + B2 --> B3["reply: created / updated / deleted
marks scope accepted + render-clean"] + end + + subgraph R3["3. Git replay: re-apply decided writes"] + direction LR + C1["remote moved, or checkout
lost the commits"] --> C2["reset to remote tip"] + C2 --> C3["re-plan each pending write in order
on the new tree"] + C3 --> C4["entry the new tree refuses:
settled alone (step 4)"] + end + + R1 --> R2 + R2 -. "the resync is now a pending write,
so it can be Git-replayed too" .-> R3 +``` + +| Replay | Input | What it recovers | What it cannot recover | +|---|---|---|---| +| Watch replay | Current cluster state | Current content of every object in scope | Intermediate versions, who made them, which save they belonged to | +| Resync | One collection's desired set | Git matches that set; with `prune.mode: always`, objects gone from the cluster are swept | With `onEvent` (the default), a delete that happened while nobody watched cannot be inferred from absence | +| Git replay | The pending writes, in memory | Every decided write, in order, with its message, author, and save | Anything lost with the process: pending writes are not persisted | + +### Resync coalescing and fences + +A resync rides the same FIFO as live events, so it lands in arrival order. Several snapshots for +the same scope collapse into one, but never across a live write for that scope: + +```mermaid +flowchart LR + N["new resync for scope S"] --> Q{"a resync for S
already queued?"} + Q -- "no" --> M["queue a marker at the tail"] + Q -- "yes, nothing for S
queued behind it" --> SW["swap in the newer payload
answer the old caller: superseded"] + Q -- "yes, but a write for S
is queued behind it (fenced)" --> REL["old marker keeps the payload it holds
new request takes a fresh marker"] +``` + +Swapping a payload behind a live write would let the older write overwrite the newer snapshot, so +the fence (`tailPassed`) forces a new position instead. The payload the old marker holds may be a +request swapped into it earlier; since step 5b that request runs, where before it was dropped and +its caller never answered. + +A resync marked `Heal` (a re-anchor that must not close another target's window) is parked until +the window is idle. No production producer sends one today; the watch replay sends `Heal=false`, +which closes the open window first to keep arrival order. + +## 4. Inside the branch worker: inputs and the loop + +```mermaid +flowchart TB + subgraph FIFO_IN["FIFO items (ordered, one at a time)"] + I1["Request: live events
CommitModePerEvent"] + I2["Request: atomic batch
(tests only)"] + I3["Attach a save"] + I4["Withdraw a save"] + I5["Resync"] + I6["Refresh"] + end + + subgraph TIMERS["Timers (in the same select)"] + T1["commit window:
idle / max duration"] + T2["push cooldown"] + T3["attach deadline"] + T4["refusal action"] + T5["retry deadline"] + end + + SEL(("select")) + FIFO_IN --> SEL + TIMERS --> SEL + + SEL --> H["handler"] + H --> AFTER["after every wake:
service waiting saves,
apply parked heals,
publish held bytes to the intake gate"] + AFTER --> SEL +``` + +Everything the loop does starts from one of these eleven inputs. Two more act from outside the +select: worker shutdown (context cancel) and `SetParentBranch`, an atomic setter that the push's +admission check reads. + +### How a commit window closes + +```mermaid +flowchart LR + E["live event"] --> G{"render fidelity
established for
this target?"} + G -- "no" --> DROP["dropped
(the pending replay repairs it)"] + G -- "open" --> CAN{"same author
and target as
the open window?"} + CAN -- "no" --> FIN1["close window
(identity change)"] --> NEW + CAN -- "yes" --> ADD["add to window"] + NEW["open window,
snapshot target's timers"] --> ATT["attach a waiting save"] --> ADD + ADD --> CAP{"retained bytes
over budget?"} + CAP -- "yes" --> FIN2["close window
(buffer limit)"] + CAP -- "no" --> ARM["arm idle / max timer"] +``` + +Other things that close a window: the idle or maximum-duration timer, an atomic batch or a +non-heal resync (to keep arrival order), a save with `attach: Next`, and shutdown. A closed window +becomes one decided write. + +The other write gate, `spec.suspend`, applies later, when the write is committed: a suspended +target's events are dropped, its folder is still scanned so its status stays fresh, and resuming +replays current state. + +## 5. Pending writes: decide, materialize, publish + +Every write takes one path, whatever produced it: a closed window, a resync, a save's empty +record, a refusal's empty commit, or an atomic batch. + +```mermaid +stateDiagram-v2 + [*] --> Decided: decide()
gates and build passed
save goes WaitingForPush + + Decided --> Decided: remote unreachable
or retry not due yet
(kept, retry scheduled) + Decided --> Committed: materialize()
one local commit + + Committed --> Committed: push failed
(kept, retry scheduled) + Committed --> Committed: push rejected, remote moved
Git replay re-plans it + Committed --> Published: push succeeded
save resolved with its SHA + + Decided --> Failed: the write itself failed
(plan refused, cannot apply) + Committed --> RefusedOnReplay: the moved remote's tree
refuses this entry + RefusedOnReplay --> Failed: settled alone (step 4)
refusal reported, its save fails + + Published --> [*] + Failed --> [*] +``` + +`decide` only appends to the pending writes. One driver, `advance`, runs at the end of every wake of +the loop: it materializes what was decided, schedules the push, and closes the retry and parent +recovery once nothing is pending. That last step is in one place, so no path that settles the last +pending write can leave intake paused. + +Three outcomes, classified in exactly one place, `materialize` in +[`pending_writes_loop.go`](../../internal/git/pending_writes_loop.go): + +- **Committed** (`settleCommitted`): the write is now a commit in the checkout, waiting to be pushed. +- **Failed for good** (`settleFailed`): the write itself was refused. Only that entry stops being pending. +- **Unreachable**: the remote could not be reached. Everything stays pending for the retry, and a + waiting resync caller is answered with the error (`replyToDeferredResyncs`). + +The checkout is a *projection* of the pending writes: the remote tip the writes were planned on, plus one +commit per materialized entry. The worker tracks how many entries the checkout holds +(`checkoutApplied`); when that is unknown, `materialize` resets to the remote tip and replays. + +### Publishing, with a rejection + +```mermaid +sequenceDiagram + autonumber + participant L as event loop + participant M as materialize + participant G as Git remote + + L->>M: pushPending() + M->>M: commit any decided-but-uncommitted entries + M->>G: atomic push (compare-and-swap on the branch) + G-->>M: rejected: branch moved + M->>G: fetch new tip + M->>M: reset, replay entries 1..n on the new tree + Note over M: entry k refused by the new tree:
undone, stamped, replay continues + M->>G: push again (at most 3 attempts per cycle) + G-->>M: accepted + M-->>L: pushed + L->>L: resolve saves on pushed entries
(each with its own SHA) + L->>L: clear the pending writes, clear the retry + L->>L: settle entry k as a refusal
(may decide an empty "touch" commit) +``` + +The refused entry is taken out before the pushed saves resolve and settled after the pending +writes are cleared, because settling it can decide new work. + +Every Git call in this sequence is bounded: two minutes per advertisement, fetch, or push session, +and five minutes for the whole cycle. When a push fails without a rejection (a deadline, a dropped +connection), the cycle probes the remote. If the branch is at the commits it sent, the push landed +and only the reply was lost, so the cycle settles as published instead of replaying them. + +## 6. Retry and intake + +One retry deadline covers everything the worker still owes: 10 seconds, doubling to 5 minutes. +While a parent branch is missing, the deadline's attempt is a single advertisement probe; +otherwise it materializes and pushes. The deadline records the failure it is waiting out and when +the outage began. The worker publishes both, with the intake pause, as its publication report: +every `GitTarget` on the branch shows it on `Ready`, and a held save in its message. The report +changes only when one of those does, so a retry that fails the same way writes no status. + +```mermaid +stateDiagram-v2 + [*] --> Healthy + Healthy --> RetryPending: an attempt failed
scheduleRetry(cause) + RetryPending --> RetryPending: new decisions wait
(no connection spent);
a resync is answered with the cause + RetryPending --> Attempt: retry deadline fires + Attempt --> Healthy: published, nothing owed
clearRetry(): intake reopens + Attempt --> RetryPending: failed again
backoff doubles + + state RetryPending { + [*] --> IntakeOpen + IntakeOpen --> IntakePaused: a payload would take
held + queued past the budget
(default 8 MiB) + } +``` + +The intake gate ([`intake.go`](../../internal/git/intake.go)) applies only while a retry is +pending; a healthy branch never pauses, because its pending writes drain at the next push. + +- **What it counts.** Payload on the FIFO, charged at enqueue so producers cannot fill the FIFO + between two loop iterations, plus what the loop holds: the open window, the pending writes, parked heals, + and saves waiting for a window. Every item is charged 1 KiB beyond its payload and message, so + empty saves and refusal commits count and the budget bounds the item count too. Sizes are + serialized YAML, estimated once by the producer. +- **Pausing is a latch.** The first payload that does not fit is refused and pauses the branch. + Only a cleared retry reopens it. Room under the budget again, a partial replay, or a remote that + can be read but refuses the push keep it paused. +- **Waking producers.** `BranchWorker.IntakePaused` returns a channel the gate closes when it + reopens; the watch streams wait on it. + +What each producer does when the gate refuses it: + +| Producer | On refusal | Comes back through | +|---|---|---| +| Live watch event | Session ends, cursor not advanced | Waits for intake, then the cursor resume redelivers the frame | +| Replay snapshot | Session ends, stream not marked replayed | Waits for intake, then the reconnect replays again | +| Snapshot larger than the whole budget | Refused during an outage, with both sizes in the message | Gathered again once the branch reopens; accepted on a healthy branch | +| Save attach | Dropped | The `CommitRequest` controller re-sends on its next poll | +| Save withdraw, refresh | Not refused by a pause: lifecycle work | Only a full FIFO or a stopping worker drops them; the sender asks again | + +## 7. A save (`CommitRequest`) from start to finish + +```mermaid +sequenceDiagram + autonumber + participant U as user + participant CR as CommitRequest controller + participant W as branch worker loop + participant G as Git remote + + U->>CR: create CommitRequest (author from admission) + CR->>W: EnqueueAttach + W->>W: register (first come, first served) + alt a same-author window is open or opens before the deadline + W->>W: attach to that window + W->>W: window closes, decided as a pending write
save: WaitingForPush + else no window by the attach deadline + W->>W: decide an empty "save record" commit
save: WaitingForPush + end + loop controller polls + CR->>W: LookupCommitRequestOutcome + end + W->>G: push + G-->>W: accepted + W->>W: resolve: Committed with this entry's SHA
(or AlreadyPresent if nothing changed) + CR->>CR: project status: Pushed=True +``` + +The controller never fails a save the worker still holds. Past its safety bound it sends a +withdraw, and the worker decides: an unattached request can be withdrawn, an attached one stays +`WaitingForPush` until it publishes. + +## 8. What is an event today, and what is not + +The direction in [`branch-worker-event-model.md`](branch-worker-event-model.md) is an +event-sourced state machine per branch. Today the worker is a single-owner event loop that is +partway there: + +```mermaid +flowchart LR + subgraph YES["Explicit, ordered inputs"] + Y1["6 FIFO item kinds"] + Y2["5 timers in one select"] + Y3["5 pending write kinds
(window, atomic, resync,
save record, refusal touch)"] + Y4["3 settle outcomes"] + end + subgraph PARTLY["Recorded, but only in memory"] + P1["the pending writes"] + P2["save registrations
and outcomes"] + P3["retry deadline
and intake pause"] + end + subgraph NO["Implicit inputs a decision still reads"] + N1["time.Now()"] + N2["live GitTarget / GitProvider config"] + N3["render-fidelity gate"] + N4["synchronous Git results"] + N5["SetParentBranch, shutdown"] + end + YES --> PARTLY --> NO +``` + +| Concern | Today | +|---|---| +| Single owner of branch state | Yes: one goroutine per worker | +| Ordered inputs | Yes, for what enters the FIFO; timers interleave by wall clock | +| Decisions as recorded facts | Partly: the pending writes record decided writes; window membership, save attachment, and timer firings are not recorded | +| Effects separated from decisions | Partly: `decide` and `materialize` are separate, but both run synchronously on the loop | +| Replayable without side effects | No: replaying would call Git and read current config | +| Durable | No: everything after the FIFO is in memory | + +## 9. What we do not have yet + +This is the honest list. Each item names who plans to close it, or says nobody does yet. + +### Durability + +- **Nothing after admission survives a restart.** The FIFO, the pending writes, open windows, save + registrations, and outcomes are in memory. The watch cursor *is* durable (Redis), so it can be + ahead of work the process lost. A restart replays every stream, which repairs current content + under each target's write gates and prune policy. It does not restore decided writes, + intermediate versions, authors, or save membership. Owner: the + [HA and durable delivery plan](../future/ha-gittarget-distribution-plan.md); deferred. +- **A save published shortly before a crash loses its receipt.** A re-sent request can create a + second empty commit. Owner: HA plan. + +### Bounded work + +- **Git call budgets are fixed.** Since step 6 every call to a Git server ends within two minutes + and a push cycle within five, over HTTP and SSH alike. The values are not configurable, so a + depth-1 fetch that needs longer fails every attempt. Local work inside a cycle has no bound of + its own. Owner: nobody yet; a flag waits for a measured need. +- **A lost push reply is settled by evidence only in the simple case.** When the remote is still at + the commits a failed push sent, step 6 publishes them as they are. When another writer pushed on + top in the meantime, the writes replay and a save's empty commit lands twice. Exactly-once is not + promised. Owner: HA plan. + +### Bounded memory and pause/resume + +Step 5b closed the gaps this section used to list: queued payload, empty records, parked heals, +and waiting saves are counted; the pause holds until a push lands; producers wait instead of +polling; resyncs respect the retry deadline; and an oversized snapshot has an explicit path. What +remains: + +- **The budget counts serialized bytes.** A queued object is held as an unstructured map of + about six times its YAML size, so the 8 MiB default is well below the memory it stands for. Owner: nobody yet; + documented in [Interpreting metrics](../interpreting-metrics.md). +- **A healthy branch's FIFO is bounded by count only.** The budget applies while a retry is pending; + until then, up to 1,000 queued items cost memory on top of the window and the pending writes. By design: a + healthy branch drains. +- **Work the loop derives can overshoot.** A refusal's empty commit is decided from work already + accepted, and work accepted before the outage began is kept, so the held total can pass the + budget by that much, and no further. + +### Status and observability + +- **The outage reason is generic.** Since step 5c every `GitTarget` on a branch that cannot publish + reports `Ready=False` under `Progressing`, with the start, the cause, and the pause in the message, + and a held save carries the same cause. A dedicated reason such as `PublicationFailing` waits for + a measured need. Owner: nobody yet. + +### Watch history and ordering + +- **Kubernetes watch history is not an archive.** An expired cursor forces a fresh snapshot. With + the default `prune.mode: onEvent`, a delete that happened while nothing was watching is not + inferred from absence. Inherent; documented, not planned. +- **Arrival order is not a save barrier.** A save can reach the worker before an earlier edit + from an independent watch. Only the documented attachment contract is claimed. Not planned. +- **A snapshot reply means "applied locally".** It is no publication receipt. Render fidelity + is proven against the fresh tree the resync fetched. By design. + +### Not yet event-sourced + +- **Decisions read implicit inputs**: the clock, live configuration, the fidelity gate, and Git + results. Replaying the same items tomorrow can produce different windows. Owner: the transition + boundary in [`branch-worker-event-model.md`](branch-worker-event-model.md), after the pending-writes plan. +- **Timers, attaches, withdrawals, and refreshes are not recorded as transitions.** Owner: the PR + after #413. +- **Known structural smells in the pending writes**: `checkoutApplied` living on the worker instead + of with the pending writes, and two replay functions (`replayOntoRemote` and its locked wrapper). Owner: the same + follow-up. + +### High availability + +- **One process owns a branch.** There is no failover, no handoff of a worker's obligations, and + no shared journal. Owner: HA plan; deferred. + +## Where to read the code + +| Piece | File | +|---|---| +| Watch streams, replay, resume, desired-state change filter | [`internal/watch/target_watch.go`](../../internal/watch/target_watch.go) | +| Routing, resync replies, fidelity marks | [`internal/watch/event_router.go`](../../internal/watch/event_router.go) | +| Enqueue into a worker | [`internal/reconcile/git_target_event_stream.go`](../../internal/reconcile/git_target_event_stream.go) | +| Admission, FIFO, loop, windows, push | [`internal/git/branch_worker.go`](../../internal/git/branch_worker.go) | +| Intake budget and pause | [`internal/git/intake.go`](../../internal/git/intake.go) | +| Pending writes: decide, materialize, settle | [`internal/git/pending_writes_loop.go`](../../internal/git/pending_writes_loop.go) | +| Retry deadline | [`internal/git/retry.go`](../../internal/git/retry.go) | +| Publication report and backlog gauges | [`internal/git/publication.go`](../../internal/git/publication.go) | +| Resyncs and heals | [`internal/git/resync_flush.go`](../../internal/git/resync_flush.go) | +| Saves | [`internal/git/commit_request_attach_loop.go`](../../internal/git/commit_request_attach_loop.go) | +| Missing parent probe | [`internal/git/parent_recovery.go`](../../internal/git/parent_recovery.go) | diff --git a/docs/design/gittarget-branch-worker-pending-writes.md b/docs/design/gittarget-branch-worker-pending-writes.md new file mode 100644 index 00000000..07d2f41b --- /dev/null +++ b/docs/design/gittarget-branch-worker-pending-writes.md @@ -0,0 +1,593 @@ +# Branch worker write path: pending writes with one materializer + +> **Built on #413**, reviewed 2026-10-03 at `373bf8d7` and completed 2026-10-04. Every step +> below is built, 5a2 included; step 3 was split into 3a, 3b, and 3c after review. The page stays +> here rather than in `finished/` because its recovery contract is still the reference for how a +> branch behaves through an outage. The +> [source review](gittarget-state-of-affairs.md#review-findings-and-remaining-gaps) records how each +> finding was closed. This is the write-path part of the "transition boundary" in +> [`branch-worker-event-model.md`](branch-worker-event-model.md#prepare-the-transition-boundary-for-later-durability), +> limited to the write path. Persistence stays with the +> [HA plan](../future/ha-gittarget-distribution-plan.md). Ships as one PR, one commit per step below. + +## Decision: pause intake, keep recovery running + +When a Git outage fills the branch's capacity, pause new payload admission for that branch. +Keep its worker alive: retain accepted work, service lifecycle commands, and retry on the existing +backoff. Resume intake after the accepted backlog settles. Other branch workers keep running; +targets sharing the affected worker share its publication failure. + +Backpressure protects memory. It cannot turn `kube-apiserver` into an indefinitely retained queue, +stop Kubernetes mutations, or preserve every mutation during an outage. A recoverable watch cursor +can redeliver unaccepted events; an expired cursor requires a fresh snapshot. That snapshot recovers +current state under the target's prune policy, with no reconstruction of missed intermediate +versions, authors, or save membership. With the default `prune.mode: onEvent`, a missed delete +cannot be inferred from absence alone; `always` permits a complete scoped snapshot to sweep it. + +The near-term design keeps pending writes in memory, with explicit recovery. Restarting a Pod +discards them and their save receipts, so restart is not the recovery mechanism. A durable Redis/Valkey journal +remains future work. Even that journal needs a capacity limit and an explicit admission-stop state. + +## Goal + +Fewer independent states and fewer execution paths. The worker already holds most of an +event-sourced write path, under other names; several helpers exist only to recreate the parts that +are not named. This plan makes those parts explicit and deletes the helpers. The finished change is +judged by the old machinery removed. Temporary growth is fine when +the deletions it enables are concrete, and comments are not trimmed to meet a number. + +## Starting point before step 1 + +This table records the original machinery; the built steps below describe what replaced it. + +| Event-sourcing role | Original representation | Helpers that compensated for it not being explicit | +|---|---|---| +| Ordered pending writes | `pendingWrites`: each entry keeps its events, message, author, policy snapshot and attached save | A decision becomes a pending write only after its local commit succeeds, so a failed commit discards the decision: `dropFailedWindow`, five drop branches, and parent recovery's dropped `scopes`, which re-derive lost decisions through snapshots | +| Projection of the pending writes | The checkout: a root commit plus one commit per entry, which replay already rebuilds from the pending writes | Three flags guess whether the projection is still valid (`worktreeDirty`, `replayRequired`, `rootParentStale()`); six `recoverRetainedWrites` call sites consult them, and `TestEveryLoopCommitPathRecoversADirtyWorktree` exists to catch a seventh that forgets; five functions rebuild the projection in slightly different ways | +| Deadline for an open obligation | Two clocks: `publicationRetry` and parent recovery's probe timer | `deferToRecovery`, the `parentProbeHold` atomic consulted in four places, `awaitingPush` | +| Outcomes | Saves resolve on push | `pcr.committed`, a flag that mirrors "this save rides a pending write" | + +The original window-loss defect followed from the first row: `finalizeOpenWindowWithReason` ran +the rebuild before building the window, so a failed rebuild dropped a window whose events, author, +message, and save were still intact. Step 2 fixes it. + +## Target shape + +Three pieces of loop-owned state: + +- **`pendingWrites`**: decided entries, in order, that are not yet published. Entry kinds are the existing + `PendingWrite` kinds (window, atomic, resync, request record, refusal touch). Accepted entries + survive saturation. There is no `owedSnapshot` entry replacing discarded accepted work. +- **`checkout`**: `{root, rootGen, applied}`. The checkout equals `root` plus the commits of + `pendingWrites[:applied]`; `applied` unknown means the projection must be rebuilt. +- **`retry`**: the single obligation deadline. Built fields are `backoff`, `due`, and its timer; + cause, first-failure time, and last error are proposed diagnostic state in step 5. + +Three operations: + +- **`decide(entry)`** appends to the pending writes. Write gates and building the write are checked here, + the decision point the event model names. A network failure cannot fail a decision. +- **`materialize()`** is the only function that changes the checkout for a write. On a valid + projection and a trusted base it commits the unmaterialized tail with no fetch; otherwise it + resets to the remote tip and replays the pending writes. +- **`publish()`** pushes the materialized pending writes. A rejection caused by a moved remote invalidates the + projection, re-materializes and retries (up to three times, as today). Success resolves the saves + riding the published entries and removes them from the pending writes. + +The pending writes and the retry deadline are the state the HA plan's journal would persist. This plan keeps +both in memory. + +## Recovery contract + +These are proposed operational states derived from the pending writes, retry schedule, and admission state. +Do not add a second authoritative phase field or another retry timer. + +| State | New payloads | Work the worker still performs | Exit | +|---|---|---|---| +| Running | Accept within capacity | Normal decisions, materialization, and publication | Retryable failure or capacity pressure | +| Retrying | Accept within remaining capacity | Local decisions; Git recovery only when due | Publication succeeds, or capacity fills | +| Paused | Refuse new writes, resyncs, and new saves | Already accepted FIFO work, window closure, lifecycle commands, due recovery | Accepted backlog settles | +| Recovering | Keep intake paused during the attempt | Probe if needed, materialize retained entries, publish | Failure returns to Paused; settled backlog reopens intake | + +`Recovering` is the due attempt while paused. A successful advertisement or fetch alone does not +reopen admission: read access does not prove that a branch-protected remote accepts pushes. A +completed publication, or explicit terminal settlement of all accepted obligations, does. Keep +admission closed across partial replay progress and a transient drop below the high-water mark. +Include already admitted FIFO payloads, the open window, and deferred work when deciding whether +that backlog has settled. A healthy FIFO that briefly fills waits for capacity; it must not wait +for a retry deadline that does not exist. + +The worker continues consuming accepted FIFO items into decisions, so controls behind them can +run. It does not discard those items or move withdrawals ahead of their attaches. Bound that +accepted set at enqueue; stopping intake after transferring it into the pending writes is too late to make +a strict memory promise. Check due recovery between items so a busy FIFO cannot starve it. + +The scope is the existing `(GitProvider namespace, name, branch)` worker. Do not stop the manager, +its status controllers, admission endpoints, or unrelated workers. Worker retirement and Pod +termination keep their explicit shutdown behavior; they cannot be described as pause/resume. + +### Resume and the three meanings of replay + +1. Replay **accepted decisions** from the pending writes onto the remote when the checkout requires + rebuilding. Preserve order, messages, attribution, and save boundaries. A terminal refusal + settles only its entry, as step 4 establishes. Each decision is judged against the tree the one + before it left, so commit count and SHAs can change, and a decision that changes nothing there + makes no commit. Two decisions to one object onto a remote that already holds the second one's + content replay as a revert and a reapply: the final tree is right, and each decision keeps its + commit and its save. This is accepted as is; collapsing it would cost a save on the first + decision its commit (`TestDecidedWrite_SameObjectEditsReplayAsDecidedOntoARemoteHoldingTheLast`). +2. Resume **unaccepted observations** from the last safely admitted watch cursor after intake + reopens. While paused, close the affected sessions and wait for branch capacity, with a + cancellation-safe wakeup and a bounded fallback recheck. Do not reconnect and collect rejected + snapshots every two seconds. A stale wakeup rechecks capacity before doing work. +3. If the cursor expired, gather a **fresh complete scoped snapshot** through the existing replay + path, after retained work. An incomplete snapshot never authorizes a sweep. Live work from that + session follows its snapshot. Existing scope, coalescing fences, and prune policy still apply. + +Watch catch-up is a separate observation from Git publication recovery. Admission can reopen while +a scope is still replaying; the existing stream/readiness state must continue showing that scope +as unproven. A published old backlog does not prove the cluster is current. Record cursor expiry +and the history limitation; do not turn a fresh snapshot into a receipt for a missed save. + +If a process restarts, `runTargetWatch` starts a fresh replay even with a stored cursor. This can +repair current object content, subject to write gates and pruning. It cannot restore lost decided +writes, intermediate history, or the identity of an empty save already published before the crash. + +### Review findings that gate the next implementation + +Source review at `373bf8d7` confirms the pending-writes path and step 4 refusal isolation. It also finds: + +- **Deduplication preceded acceptance** (fixed in step 5a; the filter is per stream since 5a2). + `skipUnchangedLiveUpdate` stored the content hash before enqueue, so a refused UPDATE was skipped + on cursor resume, which then advanced the cursor. +- **The byte budget was incomplete** (fixed in step 5b). `syncAdmission` used `pendingWritesBytes` + and a pending retry. `buildRequestRecordWrite` and `buildRefusalTouchWrite` leave `ByteSize` at + zero, so repeated empty saves grew the pending writes without reaching the byte threshold, and payload on the + FIFO was not counted at all. +- **Retry pacing had an exception** (fixed after the 2026-10-04 review). `decide` let resyncs + materialize during a pending retry, so each one spent a fetch before the retry deadline, and a + write decided behind a retained resync reached that resync's fetch too. Now the retry schedule + records the failure it waits out (`retrySchedule.cause`); a resync during backoff is answered + with it, stays pending, and is applied when the retry is due. +- **Watch recovery polled** (fixed in step 5b). `runTargetWatch` reconnected after its fixed + two-second backoff, with no wait for the worker to reopen admission, so queue saturation became + repeated work. The generic watch error it reports is step 5c's. +- **Git calls were unbounded** (fixed in step 6). **Failure status was incomplete** (fixed in step + 5c). + +These findings gated steps 5a to 5c; the markers above say where each was fixed. + +A second review, at `fc02f51d`, reproduced four more. All are fixed in the same branch: + +- **Recovery could stay open with nothing owed.** A resync held while the parent was missing could + find nothing to change once it was back. The last pending write settled without a push, so + neither recovery nor its retry closed, and intake stayed paused. `advance` now ends the outage + whenever nothing is pending (`TestParentRecovery_EndsWhenTheHeldWorkNeedsNoCommit`). +- **A handled item's charge was briefly counted nowhere.** The loop released its queued charge + before publishing what it held, and a producer in between found room the branch did not have. + `intakeGate.sync` now moves the charge in one step + (`TestIntake_AHandledItemsChargeMovesToHeldInOneStep`). +- **The FIFO kept coalesced snapshots.** A marker was the first request of its key, so the queue + kept every snapshot coalescing replaced, while the budget counted only the newest. The FIFO now + holds a `resyncMarker` that holds the current request and its charge, and `releasedResyncs` is + gone (`TestEnqueueResync_ACoalescedSnapshotIsNotKeptByTheQueue`). +- **A window under-charged what it kept.** A delete was charged nothing, and an attached save + stopped counting until its window closed. A window now charges each event it keeps and the save + attached to it. A decided write is never charged more than the window or save it came from + (`TestIntake_AWindowChargesTheEventsItKeeps`, `TestIntake_AnAttachedSaveStaysCharged`). + +The same review asked for one more structural step, also taken: `decide` only appends, and one +driver, `advance`, run once per wake, materializes, settles, publishes and closes the outage. That +removed the `materializing`, `followUps` and `deciding` flags and decision numbering. + +## Steps + +Each step is one commit. Its red tests are written first. + +### Step 1: one materializer for the checkout (built) + +`refactor(git)`, no behavior change; ledger byte-identical. + +`checkoutApplied` counts the retained writes the checkout holds on top of its root. Unknown is a +dirty worktree, which only a reset clears and a push never does; a reset sets it to zero, so with +writes retained it says their commits must be replayed. It replaced `worktreeDirtyState`, +`replayRequiredState` and the `hasPendingCommits` parameter. One `materialize` replaced +`recoverRetainedWrites` and its six call sites, `invalidateAndRefresh`, and `refreshRemoteForResync`. +`ResyncRequest.RefreshRemote` went with it: every resync already fetches. + +### Step 1b: clean a partly failed write locally (built) + +`perf(git)`. A batch that fails part-way is reset to the commit it started on, locally, with its +leftovers discarded and base trust kept. Only a failed local undo leaves the worktree dirty for a +reset from the remote, so `fetches_total{reason="recovery"}` drops. + +### Step 2: a decided window survives a failed materialization (built) + +`fix(git)`. A closed window becomes a pending write before it is committed, and its save is +`WaitingForPush` from then on. An unreachable remote leaves it for the publication retry; while that +retry is pending, a decision that would need a connection waits for it. Only a failure of the write +itself is terminal. Pinned by `TestDecidedWrite_SurvivesAFailedRebuildAndLandsThroughTheRetry` and +`TestAttach_AnUnreachableRemoteHoldsTheRequest`. + +### Step 3a: one write path (built) + +`refactor(git)`. Review of step 2 found two defects and a structural gap: two write lifecycles +(`decide → materialize` for windows and saves, `materialize → commit → retain` for the rest), +`pcr.committed` reused to mean "decided", and an executor that re-entered itself through refusal +handling. This step closes them, in [`pending_writes_loop.go`](../../internal/git/pending_writes_loop.go): + +- Every write kind (window, atomic batch, resync, a save's empty record, a refusal's empty commit) + is decided into the pending writes and committed by `materialize`. `l.commit`, `retain` and the commit guard + are gone, with the separate resync, atomic and refusal-touch commit paths. +- One place classifies an attempt: committed (`settleCommitted`), failed for good (`settleFailed`, + terminal for that write), or kept pending for the retry, with any waiting resync caller answered + (`replyToDeferredResyncs`). A write's origin (the resync caller, the atomic request, + the refusal a touch answers) rides with it so its outcome can be settled there. +- `materialize` is never re-entered. A refusal's empty commit decided while a refused write is being + settled is appended to the pending writes, and the running pass commits it in order. No push starts inside a + pass. +- Every commit re-reads its write's prune policy, as a replay does + (`TestDecidedWrite_ADeferredDeleteObeysATightenedPrunePolicy`, a review finding). The read comes + from the informer cache. +- Parent recovery never opens an obligation with nothing owed + (`TestParentRecovery_AFailedEmptySaveOwesNothing`, a review finding). +- `pcr.committed` is gone: `attached` means bound to a write, a window's or a save's own record. +- `rebuildPendingWrites` folded into `replayOntoRemote`. Two replay functions remain on purpose: + `replayOntoRemote` is the core the push cycle calls under its lock, and + `refreshRemoteAndRebuildPendingWrites` is the locked entry the loop calls. + +Still split: the pending writes are loop state, while `checkoutApplied` lives on the worker because the Git +effect functions (commit, replay, reset, push) and resets outside the loop (path bootstrap) update +it. A resync whose remote cannot be reached still answers its caller and is dropped, until 3c. + +### Step 3b: one retry schedule (built) + +`refactor(git)`, no retention change; ledger unchanged. One deadline, +[`retry.go`](../../internal/git/retry.go), replaces `publicationRetry` and parent recovery's +`backoff`, `nextProbeAt` and timer. When it fires with parent recovery open it probes with one +advertisement; otherwise it materializes and publishes. Whoever observes a failed attempt schedules +the next one, once; a new parent latch starts the schedule over, so the first probe is one initial +backoff away, as before. The push timer is only the success cooldown again. `publication_retry.go`, +`deferToRecovery` and the hand-off between the two clocks are gone. `parentProbeHold` stays: it is +how the worker-side base check, outside the loop, knows not to fetch for a parent the probe has not +found yet. + +### Step 3c: missing-parent retention, and admission backpressure (built) + +`fix(git)`. Writes, saves and resyncs decided while the parent is missing stay pending, like any +other a remote failure holds back, and are published when the probe finds the parent. A resync the +remote holds back answers its caller with the error at once, because its caller is waiting to hear +what it found, and stays in its place among the pending writes; when it is applied later, its outcome is reported +the way a live write's is. Two decisions, settled 2026-10-02: + +- **Admission backpressure.** Nothing decided is evicted for capacity. While the pending writes hold the retained-byte + budget and a failed attempt waits for its retry, the worker refuses new writes, saves and resyncs + at enqueue, through the existing queue-full contract. The intended producer behavior is cursor + resume for a refused event, controller retry for a save, and recollection for a resync. The review + above identifies the deduplication hole and the watch-history limit on that intent. A healthy + branch does not close this admission gate. Its threshold excludes the open window, queued + payloads, deferred snapshots, and zero-byte entries; it is not a process memory ceiling. +- **No snapshot replacement.** Resyncs replay in arrival order, so the writes decided between two of + them and every save boundary are kept. The admission threshold limits payload growth; the FIFO + still coalesces resyncs that are only queued. + +With nothing dropped, nothing is owed a snapshot: parent recovery's `scopes`, `awaitingPush`, the +snapshot-request sequence and the controller's snapshot-request tracker are gone. The parent +recovery tests now pin that writes decided across one or two outages are kept and published, and +that a failure leaving nothing pending opens no obligation. + +### Step 4: a refused replay drops only its own entry (built) + +`fix(git)`. A replay used to abort at its first error, so one retained write that the moved remote +now refused blocked every write behind it indefinitely. `replayPendingWrites` now undoes a refused +write on its own, goes on, and stamps the entry with its refusal; a replay that completes holds the +others, and `checkoutApplied` counts only those. The loop takes stamped entries out of the pending writes after +either replay (the loop's rebuild in `materializePrefix`, the push cycle's after a rejection) and +settles each as a refusal at first commit is settled: the refusal is reported, its save fails. Any +other failure still abandons the replay and keeps everything, refused write included, for the +retry. A committed resync is marked answered, so a replay that refuses it later reports the refusal +on the target instead of answering its caller twice. Pinned by `replay_refusal_test.go`. + +### Step 5a: make refused admission safe (built) + +`fix(watch)`. A live event changes the watch's state only once the branch worker accepted it. + +- **Deduplication records only accepted content.** The check reads the per-object content hash + before routing; the record happens after the worker's `Enqueue` returned true. A refused UPDATE + therefore stays a change when the cursor resume redelivers it. (5a shipped this on a process-wide + map with compare-and-swap between overlapping streams; 5a2 replaced both.) +- **An event a stopping stream never enqueued records no cursor.** The shutdown arm returned the + event's resourceVersion, and the cursor moved past an event the worker never saw. +- **A stream resumes from a cursor only after its own replay completed.** `runTargetWatch` used to + resume after the first session ended, however it ended. When the worker refused that session's + snapshot (admission backpressure) or the watch failed mid-replay, the reconnect resumed from a + previous stream's cursor (possibly one the shutdown arm had advanced) and skipped this stream's + snapshot, its sweep, and its render-fidelity report. +- **`EnqueueRequest` is gone.** It hid its enqueue boolean, and nothing outside `internal/git` tests + called it; those tests use the unexported `enqueueRequest`. + +Pinned by `refused_admission_test.go` (route + cursor through the producer, cancellation, and +resume-after-own-replay; each was reproduced red against the old behavior). + +Producer inventory, after this step: + +| Producer | Worker API | Refusal reaches | Redelivery | +|---|---|---|---| +| Live watch event | `Enqueue` → bool | `routeLiveTargetWatchEvent` error; session ends, cursor not advanced | Cursor resume redelivers the frame | +| Watch replay / LIST fallback snapshot | `EnqueueResync` → bool, plus the reply | `enqueueReplayResync` error; no cursor, stream not marked replayed | The reconnect replays again | +| CommitRequest attach / withdraw | `EnqueueAttach`, `EnqueueWithdraw`, fire-and-forget | Nothing synchronous | The controller re-sends on its next poll | +| Refresh | `EnqueueRefresh`, fire-and-forget | Nothing synchronous | The next `GitTarget` reconcile asks again | + +Remaining limits, for 5b: + +- Refusal still ends the watch session, and the reconnect retries after the fixed two-second + backoff. A refused snapshot is now gathered again on every attempt until the worker accepts one, + where before it resumed from a cursor it had not earned. That is correct, and it costs one LIST + or replay per attempt while intake is closed. 5b's wait for capacity removes the polling. +- The overlap and replay limits this step left are closed by 5a2. + +### Step 5a2: desired-state change filter (built) + +Two commits: `fix(watch)` refuses overlapping collections, then `fix(watch)` gives each stream its +own filter. + +**Overlap refusal.** One `GitTarget` watches each object through one collection. +[`collection_overlap.go`](../../internal/watch/collection_overlap.go) (was `selector_conflict.go`) +refuses, during rule resolution and before any subscription starts, every distinct collection that +overlaps one the target already selects: same type, and the same namespace or one of them all +namespaces, whatever the selectors. Exact duplicates are one collection and share one stream; +disjoint namespaces and other targets stay independent. Oldest-rule precedence and whole-rule +refusal are unchanged. The reason is `CollectionOverlap` (was `ObjectSelectorConflict`), and the +message names both rules and both scopes. Resolution re-runs on rule and catalog changes, and the +plan stops a leaving collection before starting its replacement. + +**Filter.** [`desired_state_change_filter.go`](../../internal/watch/desired_state_change_filter.go): +`desiredStateChangeFilter` is a plain UID-to-hash map owned by one stream, created by +`runTargetWatch`, kept across reconnects, and released with the stream. It passes CREATE, DELETE, +and UPDATEs that change sanitized content (data, spec, retained labels and annotations), and drops +the rest as `watch_events_total{outcome="unchanged"}`, before attribution and admission. A missing +baseline or an unhashable object passes. A live event's hash is recorded only after `Enqueue` +returned true; an accepted DELETE clears its UID. Each replay, initial-events or LIST fallback, +gathers hashes beside its desired set with the same sanitizer, and the map is replaced with them +only once the worker accepted the snapshot. A refused or unfinished replay installs nothing. + +Removed: `Manager.liveContentDedup`, `liveContentCheck` with its `prev`/`hadPrev` compare-and-swap, +`liveContentDedupKey`, and `live_content_dedup_test.go`. The step 5a producer guarantees are +unchanged. + +Pinned by `collection_overlap_test.go` (structural refusal with equal, different, and no +selectors; precedence; exact duplicates; separate targets; recovery when the older rule goes, with +the old stream retired before the new one launches; an overlap created by a catalog change) and +`desired_state_change_filter_test.go` (contract table; Git-visible versus stripped fields; and, +through both initial-events and LIST fallback, replay X→Z→X, a replayed object's status update +inside another author's window, refused and unfinished replays, an omitted UID, and the stream +lifetime). The replay and window cases fail without snapshot seeding. + +Remaining limits: + +- The window case is pinned at the producer: the worker receives only alice's edits, back to back. + That an unattributed event between them would split her commit is the worker's window rule, not + re-tested here. +- The baselines are in memory. A restart starts every stream with a replay, which seeds them again. + +### Step 5b: pause and resume under capacity pressure (built) + +`fix(git)`, `fix(watch)`. Built in five commits: the per-entry charge and the resync pacing after +the 2026-10-04 review, then a coalescing fix the accounting needed, the intake gate, and the +producer wait. The intake gate is [`intake.go`](../../internal/git/intake.go). + +- **One budget, bytes and count.** While a failed attempt waits for its retry, `--branch-buffer-max-size` + bounds everything accepted and not yet published: payload on the FIFO, charged at enqueue so + producers cannot fill the FIFO between two loop iterations; the open window; the pending writes; deferred + heals; and saves waiting for a window. Every item is charged `pendingWriteOverheadBytes` (1 KiB) + beyond its payload and message, so the byte budget also bounds the item count, empty records + included. The producer sizes each event and snapshot once, off the loop, and the loop reuses the + size. The charge is serialized YAML, not heap: a queued object is held as an unstructured map of + about six times that. Overshoot is limited to work accepted before the outage began and to work + the loop derives from accepted work, such as a refusal's empty commit. +- **Pause is a latch.** A payload that would cross the budget is refused and pauses the branch; + every later payload is refused until the retry clears, which means a publication landed or + nothing is owed. Room under the budget again, a partial replay, or a remote that can be read but + refuses the push keep it paused. A healthy branch never pauses. +- **Oversized items.** A snapshot larger than the whole budget is accepted on a healthy branch, + whose pending writes drain at the next push, and the window cap finalizes around it as before. During an + outage it can never fit: it is refused with a diagnostic naming its size and the budget, pauses + the branch, and is gathered again once the branch reopens. Its scope stays unproven meanwhile, + nothing of it is applied, and no sweep runs. +- **Lifecycle work is not payload.** A withdrawal, a refresh tick, and shutdown never pass the + gate. While the branch is paused nothing new enters the FIFO and the loop keeps draining it, so + they find room without a reserved allowance, in FIFO order. A duplicate attach allocates nothing: + the loop keeps the first registration, and the queued charge is released once it is handled. +- **Producers wait.** `BranchWorker.IntakePaused` returns a channel closed when intake reopens. A + target watch whose session ended waits on it instead of the two-second backoff, re-checks after + every wake-up, and falls back to a one-minute re-check. The cursor stays where the last accepted + event left it. +- **Retry pacing.** A resync during backoff is answered with the retry's known failure and stays + pending; a write behind it is no longer treated as local. Refresh ticks were already skipped while + writes are retained. Healthy-path refresh and pre-resync fetches are unchanged. +- **Found on the way:** a resync coalesced into a queued marker was lost when a write queued behind + the marker released its key: the marker ran the request already answered as superseded, and the + coalesced request was never run or answered. A released marker now runs the request it held. + +### Step 5c: project publication and intake state (built) + +`feat(status)`. One immutable worker observation supplies every target's status and held save: +`PublicationStatus` in [`publication.go`](../../internal/git/publication.go). + +- **One report per worker.** The loop publishes whether a failed attempt waits for its retry, since + when (`retrySchedule.since`, stable across retries), the last cause, whether the cause is the + parent branch, and whether intake is paused. It replaces the report, and tells the branch's + `GitTarget`s through `WorkerManager.PublicationEvents`, only when one of those changes. When the + next retry is due is deliberately left out of it. +- **GitTarget.** A failing report contributes `Ready=False`, `Reconciling=True`, `Stalled=False` + under the existing `Progressing` reason, with a message naming the start, the cause, and the + pause. It ranks after the dependency gates and ahead of the target's own data plane, which it + explains; terminal gates still win. A cause that is the parent branch is left to + `ParentBranchNotFound`. Every target on the branch reads the same report, so a sibling's success + cannot clear it, and nothing in it touches a refusal or a replay. +- **Streams.** A session that ends while its branch has paused intake is graded `Replaying` with + reason `BranchIntakePaused`, never `Blocked`/`WatchError`, so a Git outage no longer stalls the + target. +- **Saves.** A request in `WaitingForPush` carries the report's message (`FinalizeResult.Held`), and + its status is written again when that changes. The withdrawal path is unchanged. +- **Metrics.** `git_retained_bytes`, `git_retained_writes`, `git_intake_paused`, + `git_oldest_retained_write_timestamp_seconds`, `git_next_retry_timestamp_seconds` (scrape-time + gauges), and `git_materialization_failures_total{reason}`; documented in + [`interpreting-metrics.md`](../interpreting-metrics.md) with two alerts. + +### Step 6: deadlines on Git network calls (built) + +`fix(git)`. Measured first, against go-git v6.0.0-alpha.5 with a 300 ms context and servers that +accept a connection and then stall at one protocol phase: + +| Phase that stalls | HTTP, go-git's context API | HTTP, our code before | SSH, any API | +|---|---|---|---| +| Advertisement (`info/refs`, ls-refs) | Returns at the deadline | Hung: `Remote.List` takes no context | Hung | +| Fetch transfer (`upload-pack`) | Returns at the deadline | Hung: `Repository.Fetch` takes no context | Hung | +| Push (`receive-pack`) | Returns at the deadline | Returned, but the worker's context never expires | Hung | + +SSH uses the context only to dial: the SSH handshake (`gossh.NewClientConn`) and every read of the +git protocol ignore it, so a server that never sends its banner, and one that completes the +handshake and then says nothing, both hung past an expired context. That also meant worker shutdown +waited on a stalled SSH call. The measurement and its explanation live with the code, in +[`network_bound.go`](../../internal/git/network_bound.go). + +What changed: + +- **Context-taking API everywhere.** `listRemoteRefs`, the fetch inside `SmartFetchFrom`, + `CheckRepo`, and `advertiseRemoteBranch` use `ListContext`/`FetchContext` and take a context; the + advertisement's three callers (refresh, the parent probe, the push-failure probe) pass theirs. +- **A connection-level bound for SSH.** Each call adds a dialer whose connection closes when the + call's context ends (`context.AfterFunc`). Closing the socket fails the blocked read on the calling + goroutine; nothing runs the operation elsewhere, so nothing outlives it with the checkout. go-git + cancels the context it hands the dialer once the dial returns, so the dialer captures the call's + own context instead. +- **Two budgets.** `gitCallTimeout` (2 minutes) bounds each advertisement, fetch, or push session, + inside the library functions, so the `GitProvider` controller's connectivity check is bounded + too. `gitPublishTimeout` (5 minutes) is one deadline over the whole push cycle, contention retries + and their replays included. Both are package variables, not flags; a flag waits for a measured + need. +- **A lost push reply is settled by evidence.** A push that fails without a typed rejection already + kept its local commits and probed the remote. When the probe finds the branch at our local head, + the push landed and only its answer was lost: the cycle settles as published instead of replaying, + which would have planned the writes onto a tree that holds them and landed a save's empty commit + twice. Only the exact head counts. A remote that moved on past our commits replays as any other + contention does, because proving ancestry needs history a depth-1 fetch does not carry. + +Pinned by [`network_bound_test.go`](../../internal/git/network_bound_test.go): every call above +returns at its bound over HTTP and over both SSH stalls, an SSH stall ends on cancellation, one +deadline covers the whole push cycle, and a push whose reply is swallowed after the server applied +it lands exactly once (reproduced red without the evidence check: it pushed again and the remote +ended at a replayed commit). + +Remaining limits: + +- Exactly-once is not promised. A lost reply followed by another writer's push replays our writes, + and an empty save then lands twice. +- The bounds are fixed. A depth-1 fetch that needs more than two minutes fails every attempt. +- Local work inside a cycle (planning, replay, Kubernetes reads for prune policy) counts against + the cycle's budget but has no bound of its own. + +### Step 7: documentation (built) + +Updated the event model's "Next implementation" and "Problems" sections, the findings in the +state-of-affairs page, the paragraph after the gate table in +[`commitrequest-design.md`](../spec/commitrequest-design.md#when-the-target-may-not-be-written), +and [`architecture.md`](../architecture.md), which gained a section on a remote that cannot be +reached. The effective-point comment in `write_gate.go` already said what was built and is +unchanged. `UPGRADING.md` gained its entries with each step. The PR body carries the argument; this +page is marked built. + +## What is gone so far + +| Removed | By | +|---|---| +| `worktreeDirtyState`, `replayRequiredState`, the `hasPendingCommits` parameter | step 1 (one count) | +| `recoverRetainedWrites` and its six call sites, `invalidateAndRefresh`, `refreshRemoteForResync`, `prepareBaseForResync` | steps 1 and 3a | +| `ResyncRequest.RefreshRemote`, end to end | step 1 | +| The fetch that cleaned a partly failed write | step 1b | +| `dropFailedWindow` and the network-failure drop branches | step 2 | +| The second write lifecycle (`l.commit`, `retain`, the commit guard), `pcr.committed`, `rebuildPendingWrites`, executor re-entry | step 3a | +| `publication_retry.go`, `deferToRecovery`, parent recovery's own backoff and timer | step 3b | +| Parent recovery's `scopes`, `awaitingPush`, `noteResyncApplied`, `recoveryTargets`, the snapshot-request sequence, the controller's snapshot-request tracker, and the per-scope drop paths | step 3c | +| `Manager.liveContentDedup`, its compare-and-swap between overlapping streams, and overlapping streams within one `GitTarget` | step 5a2 | +| The `admissionClosed` flag, the budget read only once per loop iteration, and the paused watch's two-second reconnect | step 5b | +| A Git outage reported as `Ready=True`, or as `Stalled=True` with `WatchError` once intake paused | step 5c | + +## What stays + +The open window and its timers, FIFO admission and resync coalescing at enqueue, deferred heals +(still needed so a heal never closes a sibling target's window), the attach loop, the +refusal-touch schedule, `baseTrusted`, the standby state (`newBranchParent`), and the push's +compare-and-swap with the parent-change admission check. + +## Invariants to keep + +- `TestGitRoundTripLedger`: existing rows byte-identical; only explained rows added. +- Parent recovery's single probe budget per worker. +- The push admission boundary for a parent change (`admitPushRoot`). +- The write gates' effective points: on the normal path the decision and the local commit happen + in the same step, as today. Under a failed materialization the gates were read at the decision, + and a decided entry is materialized later like a retained commit is pushed after a gate closes. +- The controller never fails a request the worker holds. +- Compare-and-swap and refused-upload tests run against `startRealGitServer`, never `file://`. + +## Resyncs among the pending writes + +A resync arrives on the same FIFO as every other write, so it belongs among the same pending writes. It does two +separate jobs, and only one of them can wait: + +- **A write**: make one scope of the folder match a desired set. It is decided into the pending writes in + arrival order and materialized against a fresh base (the deliberate pre-resync fetch stays, so + ledger row 10 is unchanged). A committed resync is already retained and replayed today. When it + cannot be materialized, it stays pending like a window does. Its age does not matter: writes + that arrived after it sit after it in order, and a replay applies them on top. +- **A measurement**: its reply is what marks the scope clean for render fidelity and accepted for + its Git path (`drainScopedResync` in + [`event_router.go`](../../internal/watch/event_router.go)). That proof holds only against a fresh + tree, and the caller waits at most five minutes. So a resync that cannot be materialized now + still replies with the error, as today, and the scope stays unproven, which is the conservative + direction. If the deferred write is refused when it finally materializes, step 4 reports that + refusal on the target's status. + +## Decisions + +Settled 2026-10-02: + +1. **Resyncs stay pending**, as described in [Resyncs among the pending writes](#resyncs-among-the-pending-writes). Capacity + closes admission; it never replaces accepted entries with an owed snapshot. +2. **A partly failed write is cleaned locally** (step 1b). +3. **Follow repository validation and push rules.** This changes the Git write and watch paths, + so the high-risk exception in `AGENTS.md` requires a local e2e pass before an implementation + push. Documentation-only changes use `task lint-docs`. + +Updated 2026-10-03: pause/resume is per worker, accepted work stays in memory, and watch catch-up +has a weaker guarantee than replaying accepted decisions. Redis remains deferred. + +## Acceptance scenarios for the remaining steps + +| Scenario | Required observation | +|---|---| +| UPDATE refused, then replayed from the unchanged cursor | Built (5a): it is enqueued; dedup cannot skip it or advance the cursor past it | +| Push outage under continuous events and empty saves | Built (5b): the budget stops intake at enqueue; accepted entries and saves remain | +| Saturation across two targets on one branch and a second branch | Built (5c): every target on the branch is told; another branch is not | +| Paused producers, with no new Kubernetes edits | Built (5b): one due retry recovers; producers wake without a restart | +| Resync arrives during backoff | Built: no early fetch; the caller hears the failure and the resync stays | +| Read access returns but pushes are still rejected | Built (5b): intake stays paused and backoff continues | +| Cursor expires during pause | Fresh scoped snapshot follows accepted work; no missing-history claim | +| DELETE history expires under each prune mode | Only permitted deletions occur; retained stale objects remain observable | +| One replay entry is refused | Only that entry settles; later work can publish | +| Capacity stays full during withdrawal and shutdown | Controls progress in order; held saves never falsely time out | +| A snapshot exceeds the entire payload budget | Built (5b): refused with its size; no retry storm or partial sweep | +| Remote stalls, or accepts a push and loses its reply | Built (6): deadline returns control; publication evidence governs save outcomes | + +## What comes next + +Nothing in this plan. Recording timers, attaches, withdrawals, and refreshes as transitions is the +next PR, under [`branch-worker-event-model.md`](branch-worker-event-model.md#prepare-the-transition-boundary-for-later-durability); +persistence and HA stay with the [HA plan](../future/ha-gittarget-distribution-plan.md). + +## Out of scope + +Persisting the pending writes or the deadlines, HA, an asynchronous executor, the success push cooldown, and +recording the other FIFO inputs (attach, withdraw, refresh) as transitions. diff --git a/docs/design/gittarget-parent-branch.md b/docs/design/gittarget-parent-branch.md index 1b0fc2ba..bbf02c0b 100644 --- a/docs/design/gittarget-parent-branch.md +++ b/docs/design/gittarget-parent-branch.md @@ -60,13 +60,13 @@ check is what protects the first write. | no | no, omitted, empty repository (no refs at all) | unborn `HEAD`; the first commit is a root commit on the write branch | - **Recovery from a missing parent** is an obligation on the branch worker, not an observation. It - latches when a cycle fails on the parent and clears only when the work is published: retained - writes when they are pushed, and dropped writes per `(GitTarget, collection)` when a resync of - that scope is pushed or settles as a no-op. The worker probes with one advertisement after 10s, - doubling to 5m, shared by every target on it; before a deadline nothing spends a connection on - the parent, and reconciles never probe. Once found, it retries the retained writes and raises - a per-target snapshot request, which the controller turns into one recheck. Meanwhile the target - reports `RecoveringParentBranch`. + latches when an attempt fails on the parent and clears only when the work is published. Writes, + saves and resyncs decided while the parent is missing are kept as the worker's pending writes, + not dropped, and admission backpressure bounds them + ([`gittarget-branch-worker-pending-writes.md`](gittarget-branch-worker-pending-writes.md), step 3c). The worker probes + with one advertisement after 10s, doubling to 5m, on its one retry schedule shared by every target + on it; before a deadline nothing spends a connection on the parent, and reconciles never probe. + Once found, it publishes the pending writes. Meanwhile the target reports `RecoveringParentBranch`. - **A parent change on a live worker** takes effect at a push admission boundary: every operation reads one `{name, generation}` snapshot, trust is scoped to the generation, and a push is admitted only when its root was chosen under the current one. A push already admitted is not recalled. diff --git a/docs/design/gittarget-state-of-affairs.md b/docs/design/gittarget-state-of-affairs.md index 71a764c4..b65afa98 100644 --- a/docs/design/gittarget-state-of-affairs.md +++ b/docs/design/gittarget-state-of-affairs.md @@ -1,156 +1,152 @@ # GitTarget and the branch worker: state of affairs -> **Snapshot**, taken 2026-10-02 at `230d602a` (#411 on `main`), and reviewed against #412 at -> `3708b529`. That branch fixes gap 1 and most of gap 2 below, closes the 0.51 docs gaps, and adds -> the rewritten [`branch-worker-event-model.md`](branch-worker-event-model.md) and HA plan. It describes -> what is built, which page owns which decision, which edge cases are known not to work, and a -> recommended order for what comes next. It decides nothing by itself; each linked page stays the -> owner of its own contract. Current-behavior claims below refer to the reviewed branch; fixed -> gaps describe the earlier behavior explicitly. +> **Source review**, 2026-10-03 at `373bf8d7` on the step-4 branch, updated 2026-10-04 when #413 +> built the remaining steps. This page distinguishes implemented behavior from what is left. It +> reports source and test coverage inspected; it does not claim current PR/CI status. -## Where the release stands +## Where the implementation stands -Release PR [#406](https://github.com/ConfigButler/gitops-reverser/pull/406) (`0.51.0`) is open and -carries four merged changes since `0.50.0`: +Every step of the [branch-worker pending-writes plan](gittarget-branch-worker-pending-writes.md) is built. Decided +writes take one materialization path, retry uses one schedule, and a refused replay entry no longer +blocks unrelated retained work. On top of that: the watch's admission boundary, bounded Git calls, +an explicit publication pause, and a status that operators can read. -| PR | What it changed | Breaking | -|---|---|---| -| #403 | A save's delay restarts when it claims a window; #388 restored to the changelog | no | -| #404 | One commit window surface: `spec.commit.window.{idleTimeout,maxDuration}` on `GitTarget`, `spec.window` on `CommitRequest`, `closeDelay` gone | **yes** | -| #407 | `GitTarget.spec.parentBranch`; a target on standby creates its write branch from the parent's current tip, checked at publication; the hardening pass | no (additive) | -| #411 | A `CommitRequest` on a suspended target, or one whose render fidelity is not established, fails instead of reporting `Ready=True` with nothing written | no, but user-visible | - -The release PR body includes #411. #412 remains a separate open fix PR at this snapshot. +Recovery stays inside the existing worker. It pauses intake for the affected branch when capacity +fills, keeps accepted work, retries with backoff, and wakes producers once a push lands. +Restarting the Pod loses in-memory decisions and save receipts. Redis persistence remains in the +[future HA plan](../future/ha-gittarget-distribution-plan.md). ## What is built -| Capability | Code | Pinned by | +| Capability | Code | Evidence inspected | |---|---|---| -| Failed publication retries without new writes, with 10s to 5m backoff; while parent recovery is open, its deadline paces the retry | [`publication_retry.go`](../../internal/git/publication_retry.go) | `TestPublicationRetry_*` (6) | -| One commit window per branch worker, with idle and maximum-duration timers; a save's timers replace the target's for the window it attaches to | [`open_window.go`](../../internal/git/open_window.go), [`commit_request_attach_loop.go`](../../internal/git/commit_request_attach_loop.go) | `TestCommitEmpty_*`, `TestAttach_*` | -| `whenNothingToCommit: CommitEmpty` records a save's message in an empty commit; outcomes `Committed` and `AlreadyPresent` are decided by the push | [`branch_worker.go`](../../internal/git/branch_worker.go) (`resolvePushedCommitRequests`) | `TestCommitEmpty_*` | -| The controller never fails a save the worker holds; past its safety window it only asks the worker to withdraw it | [`commitrequest_controller.go`](../../internal/controller/commitrequest_controller.go) | `TestParentRecovery_AHeldCommitRequestIsCommittedAfterALongRecovery` | -| Write gates: suspension and render fidelity are read when the commit is made; a commit made before a gate closed is still pushed | [`write_gate.go`](../../internal/git/write_gate.go) | `TestWriteGate_*` (6), `TestSuspend_*`, `test/e2e/suspend_e2e_test.go` | -| Standby: an absent write branch is created from the parent's tip as advertised on the push session; a no-op write leaves it absent | [`git_atomic_push.go`](../../internal/git/git_atomic_push.go), [`branch_worker.go`](../../internal/git/branch_worker.go) | 20 `TestBranchWorker_*Parent*` tests, `test/e2e/new_branch_parent_e2e_test.go` (3 contexts) | -| A parent change on a live worker takes effect at the push admission boundary (`{name, gen}` snapshot) | [`branch_worker.go`](../../internal/git/branch_worker.go) | `TestBranchWorker_AParentChange*` (8) | -| An empty repository that gains a ref is never orphaned; a dangling `HEAD` is `Missing` and only a repository with no refs is `Unborn` | [`git_atomic_push.go`](../../internal/git/git_atomic_push.go), [`git_smart_fetch.go`](../../internal/git/git_smart_fetch.go) | `TestBranchWorker_AnEmptyRepositoryThatGains*`, `TestBranchWorker_ADanglingRemoteHeadIsMissingNotUnborn` | -| Parent recovery is an obligation, not an observation: retained writes plus per-`(target, collection)` dropped scopes, one probe per worker (10s doubling to 5m) | [`parent_recovery.go`](../../internal/git/parent_recovery.go), [`gittarget_parent_recovery.go`](../../internal/controller/gittarget_parent_recovery.go) | `TestParentRecovery_*` (8), the missing-parent e2e context | -| Cost gate: connections per operation, including the four new standby and probe rows | [`git-roundtrip-ledger.golden`](../../internal/git/testdata/git-roundtrip-ledger.golden) rows 13 to 16 | `TestGitRoundTripLedger` | -| Idle refresh observes without writing content and skips a branch mid-cycle; during parent recovery it can service a due probe and publish retained writes | [`refresh.go`](../../internal/git/refresh.go) | `TestRefresh_*` (25) | -| `status.remote.parent` (`Found`, `Missing`, `Unborn`) while the write branch is absent; `ParentBranchNotFound` (stalled) and `RecoveringParentBranch` (reconciling) | [`gittarget_types.go`](../../api/v1alpha3/gittarget_types.go), [`gittarget_controller.go`](../../internal/controller/gittarget_controller.go) | `internal/controller/gittarget_parent_branch_test.go` | -| Red status, first slice: refusal messages republish when the offender changes; a scoped refusal clears only on the same scope's success; "Why a `GitTarget` is not committing" in [`configuration.md`](../configuration.md#why-a-gittarget-is-not-committing) | [`git_path_acceptance.go`](../../internal/watch/git_path_acceptance.go) | `TestGitTargetKstatusContract`, the unsupported-folder e2e | - -## The documents on this topic - -| Page | Status | Owns | Note from this review | -|---|---|---|---| -| [`gittarget-parent-branch.md`](gittarget-parent-branch.md) | built (#407) | The parent-branch contract as built | Accurate against the code and ledger. One open question (parent per `GitProvider.status.branches` entry). | -| [`gittarget-parent-hardening.md`](gittarget-parent-hardening.md) | done (#407) | The races, recovery and cost rules the contract depends on | Every §4 item has a named test and its ledger row. Mostly history now, including agent handoff instructions in §0; a candidate for `docs/finished/` after the release, with §7 kept as the "do not re-raise" list. | -| [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md) | deferred | Which branch an empty repository's first commit may create | Blocked on go-git discarding the server's unborn `HEAD`. The upstream issue and PR are the only step that can start now, and it runs in the background. | -| [`gittarget-parent-observation.md`](gittarget-parent-observation.md) | deferred | The parent after the write branch exists; ancestry status | Informational only. Step 1 (availability) is the current backlog item in [`TODO.md`](../TODO.md). | -| [`gittarget-red-status-plan.md`](gittarget-red-status-plan.md) | partly built | Why a target is red, and what the message must say | Step 5 (the operator section) shipped and the page does not say so. The step 1 cross-package reason-agreement test was not found. Open question 2 (refusal state across a restart) is still open: `GitPathAccepted` lives in watch-manager memory. | -| [`gittarget-configuration-freshness.md`](gittarget-configuration-freshness.md) | deferred | Whether a target's running watch plan matches its configuration | Deferred with explicit pick-up triggers; neither trigger has fired. | -| [`gittarget-api-wave.md`](gittarget-api-wave.md) | effectively done | The `GitTarget` field boundaries | Every step shipped, was dropped, or was declined. A candidate for `docs/finished/`. | -| [`branch-worker-event-model.md`](branch-worker-event-model.md) | proposal; retry built in #412 | Worker execution semantics: the FIFO inputs, deadlines, saves, the transition boundary, and the operation-deadline fix still to do | Leads with the liveness fix, names both unbounded network calls, and hands the journal to the HA plan. | -| [`push-cooldown.md`](push-cooldown.md) | partly built | Success cooldown versus failure backoff | §7 option C is built in #412, including the parent-recovery handoff. Success-cooldown changes still need measurement. | -| [`../spec/commitrequest-design.md`](../spec/commitrequest-design.md) | spec | The save lifecycle | Current as of #412. | -| [`../future/ha-gittarget-distribution-plan.md`](../future/ha-gittarget-distribution-plan.md) | future | The durable journal, publication recovery, outage retention, and the persistence and HA rollout | The single owner of the journal since #412. | -| [`../TODO.md`](../TODO.md) | backlog | Follow-ups | Its "Before the next release" list is closed by #412. | - -## Known gaps, ranked by what a user would hit - -Fixed behavior is marked separately from gaps that remain at `3708b529`. - -1. **Fixed in #412: a failed push on a quiet branch stranded the work, and any save riding it - waited forever.** - Before #412, `pushPending` stopped its timer and armed nothing. Refresh skipped retained work, - and withdrawal could not fail a held save. A transient failure could therefore strand the - commit and its request until another write arrived. The worker now schedules publication - retries itself; parent recovery retains its own schedule. -2. **Fixed in #412: a down remote under steady traffic cost one failed push per commit.** - The publication retry now holds new commits until its deadline. Review found one case the - first version missed: after a missing parent returns, recovery stays open but its hold is gone, - so each new commit retried a failing push (three attempts where one was due, on `main` too). - A failure while recovery is open now defers to recovery's deadline, pinned by - `TestPublicationRetry_AFailureAfterTheParentReturnsWaitsForRecovery`. The backoff paces - publication attempts only; fetches a new window or snapshot needs are not on that schedule. -3. **No Git network call has a deadline.** `remote.List` (`listRemoteRefs`, `CheckRepo`) and - `repo.Fetch` (`SmartFetchFrom`) take no context; the push uses the worker's context, which has - no deadline; nothing in `internal/git` sets a timeout. A stalled server can block the whole - branch loop, and every target on that worker with it. Whether go-git's transports apply a - default timeout was not established here. -4. **A failed rebuild drops the open window.** When retained writes need a replay and its fetch - fails, `recoverRetainedWrites` returns an error and the finalize drops the window - (`dropFailedWindow`). Only a missing parent records the dropped scope and asks for a snapshot, - so under any other failure those writes wait for the next resync. While the rebuild keeps - failing, every new window spends a fetch on it. -5. **Publication failures have no dedicated status.** The normal retry delay starts at 10s and - reaches a 5m cap after repeated failures. A save riding retained work stays in `WaitingForPush`. - The worker exposes no publication-failure condition or retry deadline on `GitTarget`. - Branch-protection refusals with working read access can therefore look like a slow save. - Provider connectivity and credential failures can separately surface through - `GitProviderReady`; parent recovery also has status. Logs and - `gitopsreverser_git_pushes_total{outcome="failed"}` cover push-cycle failures, but a failed - rebuild before the cycle does not increment that push metric. -6. **Accepted work is volatile.** The FIFO (1,000 items) and retained writes are process memory. - The watch cursor can advance before a write reaches Git, so a crash in between can skip work on - restart. The 8 MiB buffer threshold closes a window early; it does not bound retained memory. -7. **A save's identity does not survive a restart.** A push that succeeds immediately before a crash - loses the worker's receipt, and a resent save can create a second empty commit. The outcome - cache keeps entries 15 minutes; the admission author store keeps records one hour. -8. **Refusal state does not survive a restart.** After a manager restart a refused folder reads as - accepted until the next write or resync refuses it again. -9. **An empty repository with a non-default write branch** gets the write branch as its first, - parentless branch (documented as today's bootstrap contract; deferred policy). -10. **The parent disappears from status once the write branch exists**, so an operator cannot see - that the parent has moved ahead of an open review branch. -11. **Two `GitProvider`s naming one repository** are two workers and two checkouts on the same - remote; nothing coordinates them beyond the server's compare-and-swap. The advice remains one - `GitProvider` per repository. -12. **With `parentBranch` omitted, a push cannot see a switch of the remote's `HEAD`**; the next - refresh follows it with one fetch. This is a documented choice, not a defect. - -Gaps 1 to 5 are liveness and visibility defects in the current worker and need no persistence; -1 and 2 are fixed. Gaps 6 and 7 are the durable-execution -problem. Gaps 8 to 12 are separate, smaller decisions. - -## Review of `branch-worker-event-model.md` - -An earlier version of this page reviewed the uncommitted rewrite and found six issues: the missing -user-visible `WaitingForPush` consequence, the understated timeout gap, the buried near-term fix, a -dated status line, the dropped input inventory, and no INDEX entry. The version in #412 addresses -all six, and moves the journal, publication recovery and outage retention into the HA plan, so each -topic now has one owner. The follow-up review corrected the failure-status claim (gap 5) and found -the retry handoff in gap 2, since fixed. Neither the backoff nor serialization establishes a budget -for every Git connection. +| All write kinds become pending writes before materialization | [`pending_writes_loop.go`](../../internal/git/pending_writes_loop.go) | `decided_write_test.go`, `branch_worker_split_test.go` | +| One checkout materializer and explicit applied-prefix tracking | [`branch_worker.go`](../../internal/git/branch_worker.go) | `dirty_worktree_recovery_test.go`, `branch_worker_split_test.go` | +| One retry deadline, 10s doubling to 5m, including parent recovery | [`retry.go`](../../internal/git/retry.go), [`parent_recovery.go`](../../internal/git/parent_recovery.go) | `retry_test.go`, `parent_recovery_test.go` | +| Failed materialization retains decided writes and their saves | `pending_writes_loop.go` | `TestDecidedWrite_SurvivesAFailedRebuildAndLandsThroughTheRetry` | +| Every accepted item counts against the outage budget, and the pause holds until a push lands | [`intake.go`](../../internal/git/intake.go) | `intake_test.go`, `TestDecidedWrite_EmptySavesCountAgainstTheBudgetDuringAnOutage` | +| A resync during backoff spends no connection | `pending_writes_loop.go`, `retry.go` | `TestDecidedWrite_AResyncDuringBackoffWaitsForTheRetry` | +| A stream on a paused branch waits to be woken, graded `BranchIntakePaused` | [`target_watch.go`](../../internal/watch/target_watch.go) | `TestRunTargetWatch_APausedBranchWaitsForIntakeToReopen` | +| Every call to a Git server is bounded; a lost push reply is settled by evidence | [`network_bound.go`](../../internal/git/network_bound.go) | `network_bound_test.go` | +| One publication report feeds every `GitTarget` on the branch, its held saves, and the backlog metrics | [`publication.go`](../../internal/git/publication.go), [`gittarget_publication.go`](../../internal/controller/gittarget_publication.go) | `publication_test.go`, `gittarget_publication_test.go` | +| Replay refusal settles only the affected entry | `pending_writes_loop.go`, `branch_worker.go` | [`replay_refusal_test.go`](../../internal/git/replay_refusal_test.go) | +| Save outcomes follow remote publication; the controller cannot time out a held save | [`commit_request_attach_loop.go`](../../internal/git/commit_request_attach_loop.go), [`commitrequest_controller.go`](../../internal/controller/commitrequest_controller.go) | Held-request retry and controller safety-bound tests | +| Parent selection and compare-and-swap remain publication guards | [`git_atomic_push.go`](../../internal/git/git_atomic_push.go) | Parent-change tests and `TestGitRoundTripLedger` | +| New watch streams start with a fresh replay; reconnects resume a cursor only after the stream's own replay completed | [`target_watch.go`](../../internal/watch/target_watch.go) | `runTargetWatch`, `TestRunTargetWatch_ResumesOnlyAfterItsOwnReplayCompleted` | +| A live event changes the dedup baseline and the cursor only once the worker accepted it | [`target_watch.go`](../../internal/watch/target_watch.go) | `refused_admission_test.go`, `desired_state_change_filter_test.go` | +| Each stream owns its filter baselines, seeded from its accepted replay; one `GitTarget` holds no overlapping collections | [`desired_state_change_filter.go`](../../internal/watch/desired_state_change_filter.go), [`collection_overlap.go`](../../internal/watch/collection_overlap.go) | `desired_state_change_filter_test.go`, `collection_overlap_test.go` | + +The pending writes live in memory. They do not yet reconstruct windows, timers, or outcomes after +process loss, and `PendingWrite` contains live interfaces and process references. + +## Review findings and remaining gaps + +### 1. A refused UPDATE could be skipped on reconnect (fixed in step 5a) + +`routeLiveTargetWatchEvent` stored the sanitized-content hash before routing, so a refused UPDATE +redelivered by the cursor resume was skipped as unchanged and the cursor advanced past it. Step 5a +records the baseline only after the worker accepts the event, records no cursor for an event a +stopping stream never enqueued, and resumes a stream from a cursor only after its own replay +completed. The [step 5a section](gittarget-branch-worker-pending-writes.md#step-5a-make-refused-admission-safe-built) +lists the producer inventory and the limits left for 5b. + +### 2. The retention threshold was not a total memory bound (fixed in step 5b) + +`syncAdmission` closed intake when a retry was pending and `pendingWritesBytes` reached the branch +budget. That improved ordinary outage behavior, but: + +- Empty request records and refusal touches have zero `ByteSize`; repeated empty saves bypass it. +- The FIFO caps item count (default 1,000), not payload bytes. A snapshot can be large. +- The open window, deferred resyncs, registered requests, and upstream snapshot collection also + consume memory. An 8 MiB retained threshold does not cap these allocations. +- Admission can reopen when retained bytes fall below the threshold even if publication still + fails. There is no explicit backlog-settlement latch. + +Step 5b charges every item at enqueue, with a fixed per-item overhead, counts everything the loop +holds, latches the pause until a push lands, refuses an oversized snapshot during an outage with +both sizes in the message, and keeps lifecycle work outside the gate. Accepted decisions are never +evicted. The budget counts serialized bytes, not heap. + +### 3. Backoff did not suppress every failing Git connection (fixed in step 5b) + +`decide` exempts resyncs from the ordinary retry hold. A resync can fetch during publication +backoff unless the missing-parent hold stops it. The producer also reconnects after a fixed +two-second delay when admission refuses it. That can repeatedly gather snapshots which the +worker cannot accept. + +Producers now wait for the branch to reopen intake, and a resync during backoff is answered with +the failure the retry is waiting out and kept pending. The single retry schedule remains the +recovery driver. + +### 4. Git network calls had no operation budget (fixed in step 6) + +Ref listing and fetch ran without the caller's context, and push ran under a worker context with +no deadline. Measurement showed worse: go-git's SSH transport ignores the context past the dial. +Step 6 bounds every call (two minutes) and the whole push cycle (five minutes), closes an SSH +connection when its call's context ends, and settles a push whose reply was lost by finding the +remote at the commits it sent. The +[step 6 section](gittarget-branch-worker-pending-writes.md#step-6-deadlines-on-git-network-calls-built) has the +measurement and the limits left. + +### 5. Publication and saturation had no coherent operator state (fixed in step 5c) + +Parent recovery has status, and push failures have logs and a counter. General publication failure +has no shared worker observation projected to all its targets. A held save can say `WaitingForPush` +without explaining that the remote rejects pushes or intake has stopped. A pre-push rebuild failure +also misses the push-failure counter. Intentional producer pause currently looks like a watch error. + +Step 5c publishes one worker report and projects it onto every `GitTarget` on the branch as +`Ready=False`/`Progressing`, with a message that changes only when the cause or the pause does, and +onto a held save's `WaitingForPush` message. A paused stream is `BranchIntakePaused`, and gauges +describe the backlog and the next retry. Independent target refusals and source replay stay +visible; a successful push cannot clear them. + +### 6. Watch replay recovers a weaker contract than accepted-write replay + +Backpressure cannot stop Kubernetes mutations or retain watch history indefinitely. A cursor that +expires forces a fresh snapshot; intermediate versions, deleted short-lived objects, and their +authors cannot be reconstructed. With `prune.mode: onEvent`, an expired DELETE can leave a Git +object retained. A complete scoped snapshot may sweep it under `always`; recovery must not silently +change that policy. + +A new process already starts a fresh watch replay even with a stored cursor, so the old assertion +that every restart resumes past volatile work was inaccurate. Initial replay repairs current +content where policy permits. It does not restore accepted windows or save outcomes. A push that +lands immediately before a crash can still be followed by a duplicate empty save after restart. + +## What the refactor got right + +Keep the separation between deciding, materializing, and publishing. It lets a failed fetch keep a +save obligation without guessing what a later snapshot would have meant. Step 4's refusal isolation +also prevents one invalid folder from blocking another target on the same branch. + +The earlier `owedSnapshot` proposal and a separate parent retry timer are unnecessary for the +chosen capacity policy. Keep accepted entries and reject new admission explicitly. The next change +should repair the acceptance boundary, not replace the pending writes or introduce a generic workflow engine. ## Recommended order -### Before cutting 0.51.0 - -Both steps are in #412: the docs gaps in `UPGRADING.md`, `configuration.md`, the CRD descriptions -and the CommitRequest spec, and the publication retry with its tests. CI lint, unit tests, and all -six e2e legs passed at `3708b529`; the handoff fix in gap 2 landed after that run. - -### Next, after the release - -1. **Operation deadlines (gap 3).** Pass the worker context to `List` and `Fetch` and give each - remote operation a bounded deadline. Measure first how go-git's HTTP and SSH transports behave - today, against a server that stalls. -2. **Rebuild recovery and publication visibility (gaps 4 and 5)**: decide what a failed rebuild - owes a dropped window and expose a publication failure with its retry deadline. These improve - the current worker without waiting for a journal or a broad transition refactor. -3. **The transition boundary**: extract the event model's explicit transitions and test them - from recorded state. Agree the save contract the HA plan asks for: final-state convergence, - or preserved save outcomes too. Start the journal only after both decisions. -4. **Cheap, independent items**: open the go-git unborn-`HEAD` issue and PR; parent observation - step 1 (availability after the write branch exists, no extra connections); the red-status - reason-agreement test. -5. **Housekeeping**: move the hardening plan and the API-wave page to `docs/finished/`, and mark - red-status step 5 as shipped. - -### Leave deferred - -Configuration freshness, ancestry classification, the empty-repository bootstrap policy, and the -durable journal (the HA plan's phases) all wait on a decision or on evidence that does not exist -yet. None of them is needed to make 0.51.0 better than 0.50.0. +1. **Step 5a (built):** fix producer deduplication after rejected admission and test cursor resume. +2. **Step 6 (built):** bound network calls and test stalls and lost push responses. +3. **Step 5b (built):** complete capacity accounting and per-worker pause/resume, including producer waits. +4. **Step 5c (built):** expose publication failure, intake pause, and recovery through status and metrics. +5. **Step 7 (built):** update runtime documentation and validate the final branch. + +What is left belongs elsewhere: recording timers, attaches, withdrawals, and refreshes as +transitions ([event model](branch-worker-event-model.md#prepare-the-transition-boundary-for-later-durability)), +and persistence and HA ([HA plan](../future/ha-gittarget-distribution-plan.md)). + +## Document ownership + +| Page | Role after this review | +|---|---| +| `gittarget-branch-worker-pending-writes.md` | The steps as built, the pause/resume contract, and the review findings | +| [`branch-worker-event-model.md`](branch-worker-event-model.md) | Transition semantics and the later boundary for reconstructing execution | +| [`push-cooldown.md`](push-cooldown.md) | Healthy publication cadence; the success cooldown remains unchanged | +| [`commitrequest-design.md`](../spec/commitrequest-design.md) | Current save contract, including retained versus unaccepted requests | +| `ha-gittarget-distribution-plan.md` | Future durable acceptance, publication evidence, storage budgets, and HA | + +Parent observation, empty-repository bootstrap policy, configuration freshness, and refusal state +across restart remain separate follow-ups. Two providers naming one remote still create separate +workers; only remote compare-and-swap coordinates their writes. These do not need to be solved to +make an in-process outage explicit and recoverable. diff --git a/docs/design/label-selection-follow-ups.md b/docs/design/label-selection-follow-ups.md index 0c7b75a5..3458dc1e 100644 --- a/docs/design/label-selection-follow-ups.md +++ b/docs/design/label-selection-follow-ups.md @@ -65,29 +65,20 @@ exactly that fact. **Today.** `markResyncTailForResyncLocked` fences every other pending resync of the same GitTarget whose sweep boundary overlaps the new one's. This is what keeps a selector changed A → B → A in -the order A1, B, A2. It also fences pairs that may legitimately run side by side, such as -`configmaps in team-a` and `configmaps` cluster-wide with the same selector. +the order A1, B, A2. -**Cost.** During a relist storm on such a pair, each snapshot takes a fresh queue position -instead of coalescing. The worker's event queue is bounded, and a full queue answers a resync with -`ErrFinalizeQueueFull`. +**Cost.** None left to pay. A GitTarget can no longer run two overlapping collections side by side +(see [overlapping collections](../configuration.md#overlapping-collections)), so the fence only +orders successive collections of one scope, which must not coalesce anyway. -**Why it was kept.** The fence protects that pair too. Without it, an older cluster-wide snapshot -could run after a newer `team-a` snapshot and put stale `team-a` state back. Coalescing across the -pair would need snapshots that can be ordered by resourceVersion across two streams, and the -ordering spec says not to infer a global order from two streams' arrivals. +### 3. Overlap ties within one second go to the lower name -**Revisit if** queue-full resyncs show up in practice for a GitTarget with overlapping collections. - -### 3. Selector-conflict ties within one second go to the lower name - -**Today.** When two rules select overlapping collections with different selectors, the older rule -keeps them. `selectingRule.olderThan` compares `CreationTimestamp`, which has one-second -resolution, and breaks ties by name. See -[`selector_conflict.go`](../../internal/watch/selector_conflict.go). +**Today.** When two rules select overlapping collections, the older rule keeps them. +`selectingRule.olderThan` compares `CreationTimestamp`, which has one-second resolution, and breaks +ties by name. See [`collection_overlap.go`](../../internal/watch/collection_overlap.go). **Cost.** Say rule `b` is created and running, and rule `a` is applied within the same second. -`a` wins the tie and takes the collections. `b` is refused loudly (`ObjectSelectorConflict`), and +`a` wins the tie and takes the collections. `b` is refused loudly (`CollectionOverlap`), and under `prune.mode: Always` the next snapshot sweeps with `a`'s selector. **Why it was not fixed.** The order is deterministic and documented, and the loser is told. A diff --git a/docs/design/metrics-observability-plan.md b/docs/design/metrics-observability-plan.md index d55d1ac7..6361bb70 100644 --- a/docs/design/metrics-observability-plan.md +++ b/docs/design/metrics-observability-plan.md @@ -278,7 +278,7 @@ flowchart TB AI -->|"attribution_resolutions_total{tier,actor_kind}
_resolution_wait_seconds{tier,event_kind}"| RES subgraph GITPATH["Git: internal/git"] RES --> Q["worker queue"] - Q -->|"git_queue_drops_total{kind}"| DROPQ(["LOSS: queue full"]) + Q -->|"git_queue_refusals_total{kind}"| DROPQ(["REFUSED: queue full,
offered again"]) Q -->|"git_documents_total{outcome}"| F["files on disk"] F -->|"placement_refusals_total{reason}"| DROPP(["LOSS: not mirrored"]) F --> C["local commits"] @@ -293,10 +293,10 @@ drawn apart, and why an audit outage is never a mirror outage. ### 4.2 The object-state stages -| Stage | Counter | Expected values | Loss | +| Stage | Counter | Expected values | Loss, or refusal offered again | |---|---|---|---| -| 1 Ingest | `watch_events_total` | `routed`, `unchanged`, `bookmark`, `shutdown` | `route_failed` | -| 2 Queue | `git_queue_drops_total` | (no increment) | every increment | +| 1 Ingest | `watch_events_total` | `routed`, `unchanged`, `bookmark`, `shutdown` | `route_failed` (refusal) | +| 2 Queue | `git_queue_refusals_total` | (no increment) | every increment (refusal) | | 3 Write | `git_documents_total` | `written`, `deleted_live`, `deleted_sweep`, `unchanged`, `retained` | `placement_refusals_total`, beside it | | 4 Commit | `git_commits_total` | all: recorded on a successful push, so it cannot claim a commit the remote never took |: | | 5 Push | `git_pushes_total` | `pushed` | `failed` is **recoverable**, not loss: the writes are retained and retried | @@ -343,7 +343,7 @@ is not one panel: sum by (outcome) (rate(gitopsreverser_watch_events_total{outcome="route_failed"}[5m])), "reason", "$1", "outcome", "(.*)"), "stage", "ingest", "", "") or label_replace(label_replace( - sum by (kind) (rate(gitopsreverser_git_queue_drops_total[5m])), + sum by (kind) (rate(gitopsreverser_git_queue_refusals_total[5m])), "reason", "$1", "kind", "(.*)"), "stage", "queue", "", "") or label_replace( sum by (reason) (rate(gitopsreverser_placement_refusals_total[5m])), @@ -429,7 +429,7 @@ on the caps rather than as a count of lost joins. | `watch_sessions_ended_total` | counter | `group`, `version`, `resource`, `reason` | `reason`: `expired` (the cursor fell out of history, forcing a full rebuild), `error`, or `stopped` (the plan retired the stream: routine). Watch stability and `410` pressure | | `watch_replay_duration_seconds` | histogram | `group`, `version`, `resource` | the cost of a replay, which is what a `410` storm charges | | `git_commit_failures_total` | counter | `provider_*`, `branch`, `kind`, `reason` | the largest remaining hole (§2.4). `kind`: `window` / `atomic`; `reason`: `refused` (a Git path a human must fix, which will not clear on its own) / `error`. Every increment is a window's worth of events lost until the next resync | -| `git_queue_drops_total` | counter | `provider_*`, `branch`, `kind` | §2.4's first silent drop. `kind`: `write` / `attach` / `resync`. It **overlaps** `watch_events_total{outcome="route_failed"}`: a full queue is one of the ways a route fails, so one dropped event increments both. Two views of one event, never two events | +| `git_queue_refusals_total` | counter | `provider_*`, `branch`, `kind` | §2.4's first silent drop, a refusal since #413: the producer keeps the item and offers it again. `kind`: `write` / `attach` / `resync` / `refresh`. It **overlaps** `watch_events_total{outcome="route_failed"}`: a refused enqueue is one of the ways a route fails, so one refused event increments both. Two views of one event, never two events | | `git_pushes_total` | counter | `provider_*`, `branch`, `outcome` | §2.4's second. `outcome`: `pushed` / `failed`, counted once per push cycle at its terminal end. `failed` is **recoverable**, not loss: the writes are retained and a later push carries them, which is why the alert on it needs the second arm in §7 | | `git_push_retries_total` | counter | `provider_*`, `branch`, `reason` | `reason`: `remote_moved` / `error`. A replay round is not a terminal outcome, so it is its own counter rather than a third `outcome` value. `rate(retries) / rate(pushes)` is the contention signal | | `git_push_duration_seconds` | histogram | `provider_*`, `branch` | push latency, re-added **with** a recording site this time | @@ -596,7 +596,7 @@ Alerts, as rules rather than sketches this time: | Alert | Expression | Meaning | |---|---|---| | Mirror stopped | `sum by (provider_namespace,provider_name,branch) (rate(gitopsreverser_git_pushes_total{outcome="failed"}[15m])) > 0 unless sum by (provider_namespace,provider_name,branch) (rate(gitopsreverser_git_pushes_total{outcome="pushed"}[15m])) > 0`, for 15m | this branch is failing to push AND landing nothing. The `unless` arm is required: a failed push retains its writes and is retried, so an occasional failure beside successful ones is contention, not an outage, and paging on any failure trains people to ignore it | -| Work dropped | `rate(gitopsreverser_git_queue_drops_total[5m]) > 0` | the queue is saturated and writes are on the floor | +| Work refused on a healthy branch | `rate(gitopsreverser_git_queue_refusals_total[5m]) > 0 unless on(provider_namespace, provider_name, branch) gitopsreverser_git_intake_paused == 1` | the queue is too small for its bursts; producers offer the work again, so this is lag until a watch cursor expires | | Ingest loss | `rate(gitopsreverser_watch_events_total{outcome="route_failed"}[10m]) > 0` | observed changes are not reaching the writer. Often the same events as the row above, seen from the other end | | Audit misdirected | `rate(gitopsreverser_audit_eventlist_duration_seconds_count{outcome=~"bad_path\|bare_endpoint_disabled"}[15m]) > 0` | an apiserver is posting audit somewhere this operator will not read it (§2.6) | | Fact loss | `rate(gitopsreverser_attribution_fact_stream_gaps_total[10m]) > 0 or rate(gitopsreverser_attribution_fact_stream_decode_errors_total[10m]) > 0` | facts are gone for good. Two series in an `or`, never a sum: one counts occurrences and the other entries (§5.2) | @@ -700,7 +700,7 @@ first, so nothing new is built beside a duplicate. one. Fix the three stale comments in §2.3 and the broken cache-effectiveness query. No new capability. The attribution loss counters are **not** merged: see §5.2 for the unit error that would have been. -2. **The exceptions and the honest gauges.** `git_queue_drops_total`, `git_pushes_total`, +2. **The exceptions and the honest gauges.** `git_queue_refusals_total`, `git_pushes_total`, `git_push_retries_total`, `git_push_duration_seconds`, moving `git_commits_total` to the push site, converting every gauge to an observable callback, and turning the dirty-set age into a timestamp. Two data-loss paths and one stalled-mirror path stop being log lines, and the two diff --git a/docs/design/push-cooldown.md b/docs/design/push-cooldown.md index 1fa87529..690aa10d 100644 --- a/docs/design/push-cooldown.md +++ b/docs/design/push-cooldown.md @@ -1,9 +1,9 @@ # The push cooldown: what it still buys, and what removing it would cost > **design**: open. Option C (§7), the failure backoff, is built in -> [`publication_retry.go`](../../internal/git/publication_retry.go); the success cooldown is +> [`retry.go`](../../internal/git/retry.go), now one schedule with parent recovery's probe; the success cooldown is > unchanged. Option D needs measurement; option E remains rejected without that evidence. -> Index: [`../INDEX.md`](../INDEX.md). Reviewed against #412 on 2026-10-02. +> Index: [`../INDEX.md`](../INDEX.md). Reviewed at `373bf8d7` on 2026-10-03. > Related: [`../api-first-publication.md`](../api-first-publication.md), > [`push-notification-and-reconcile-trigger.md`](push-notification-and-reconcile-trigger.md), > [`../spec/commit-window-refactor.md`](../spec/commit-window-refactor.md) @@ -17,19 +17,32 @@ defaults already space commits enough to avoid the success cooldown. Identity ch with shorter timers, and buffer-limit closures can produce commits sooner. The cooldown batches those commits when they arrive within five seconds of a successful push. -Removing the success wait would remove `lastPushAt` and its scheduling branch. The shared push -timer and retained-write recovery would remain because publication retries now use them too. +Removing the success wait would remove `lastPushAt`, `pushTimer`, and their scheduling branches. +The separate retry timer and retained-write recovery would remain. Option C is built in #412. The remaining decision is whether changing the success cooldown earns its extra publication cost; that needs the measurements in §9. +## An outage pause is a separate decision + +Keep the success cooldown unchanged while completing the +[branch-worker recovery contract](gittarget-branch-worker-pending-writes.md#recovery-contract). Failure backoff +paces attempts at Git; capacity admission controls how much new work the process accepts. Neither +one supplies storage durability. + +The proposed saturation behavior pauses intake for the affected worker, retains accepted entries, +and uses the existing retry deadline to attempt recovery. A successful read alone cannot reopen +intake when pushes still fail. Producers resume after the accepted backlog settles, using their +cursor or a fresh scoped snapshot if history expired. This does not require another cooldown, +a manager restart, or Redis. All targets sharing that worker share the pause; other workers run. + ## 1. What it is today `PushCooldown` is a fixed `5s` per worker. `maybeSchedulePush` runs after every local commit: ```go if len(l.pendingWrites) == 0 { return } -if l.awaitingPublicationRetry() { return } // a failed publication has its own deadline +if l.awaitingRetry() { return } // a failed attempt has its own deadline if l.lastPushAt.IsZero() { l.pushPending(); return } // never pushed: go now if time.Since(l.lastPushAt) >= PushCooldown { l.pushPending(); return } if l.pushTimer == nil { l.pushTimer = time.NewTimer(...) } // otherwise wait out the remainder @@ -139,21 +152,22 @@ is half right. | --- | --- | | `lastPushAt` field | 1 field | | Successful-push wait in `maybeSchedulePush` | The elapsed-time check and cooldown scheduling | +| `pushTimer` and its select arm | Success-cooldown scheduling only since step 3b | | `PushCooldown` constant and its documentation | 1 constant, several doc paragraphs | -The failure backoff shares `pushTimer`, so its stop helper and select arm remain. The worker still -has five timer sources: window, push, attach, refusal action, and parent recovery. +The failure backoff uses its own timer in `retrySchedule`. The worker currently has five timer +sources: window, success push, attach, refusal action, and retry. Parent recovery uses the retry +deadline. Removing the success cooldown would leave four sources. **Stays exactly as it is:** | Item | Why it is not the cooldown's | | --- | --- | -| `publicationRetry`, `pushTimer`, and `stopPushTimer` | Failed publication still needs a scheduled attempt | +| The retry schedule (`retry.go`) | A failed publication still needs a scheduled attempt | | `pendingWrites` retention | A push can fail or be rejected; the writes must survive to be replayed | | `baseTrusted` | The head-of-cycle fetch decision, unrelated to push cadence | -| `worktreeDirty` | A write that failed part-way, unrelated to push cadence | -| `replayRequired` | A reset that discarded local commits, unrelated to push cadence | -| `recoverRetainedWrites`, `refreshRemoteAndRebuildPendingWrites`, the whole replay path | Contention recovery | +| `checkoutApplied` | A write that failed part-way, or a reset that discarded local commits, unrelated to push cadence | +| `materialize`, `refreshRemoteAndRebuildPendingWrites`, the whole replay path | Contention recovery | ```mermaid flowchart TD @@ -163,12 +177,11 @@ flowchart TD end subgraph FAILURE["Owned by push failure - would remain"] - PT["publicationRetry + pushTimer + stopPushTimer"] + PT["retry schedule"] PW["pendingWrites retention"] BT["baseTrusted"] - WD["worktreeDirty"] - RR["replayRequired"] - RE["reset, replay, recover"] + CA["checkoutApplied"] + RE["materialize: reset and replay"] end style COOLDOWN fill:#e8f5e9,stroke:#43a047 @@ -211,8 +224,8 @@ backlog that drains after a slow push. | Option | What it is | Complexity | Verdict | | --- | --- | --- | --- | | **A. Keep the success cooldown** | Preserve successful-push spacing | Unchanged | Current default alongside C | -| **B. Delete the success cooldown** | Push after every commit unless failure backoff applies | Removes `lastPushAt` and the success wait; keeps the timer | §3 can become one push per identity change | -| **C. Add a failure backoff, keep the cooldown** | Schedule failed publications independently of new writes | Reuses `pushTimer`; adds backoff state | Built in #412, including the parent-recovery handoff | +| **B. Delete the success cooldown** | Push after every commit unless failure backoff applies | Removes `lastPushAt` and `pushTimer`; keeps the retry timer | §3 can become one push per identity change | +| **C. Add a failure backoff, keep the cooldown** | Schedule failed publications independently of new writes | One retry timer shared with parent recovery | Built in #412; schedule unified in step 3b of #413 | | **D. Add the backoff AND shorten or drop the success cooldown** | C, plus reducing the `5s` wait once §9 has priced it | Depends on the outcome | The measured follow-up to C | | **E. Make it conditional or configurable** | Engage only when more than one identity is active, or expose it on `GitProvider` | **+complexity, +API surface** | Rejected unless D measurably fails | @@ -227,9 +240,10 @@ and it belongs in option D behind the numbers §9 asks for. Before #412, `pushPending` stopped its timer on failure and armed no replacement outside parent recovery. A quiet branch retained its work indefinitely, while continued arrivals could provoke a failed push per commit because `lastPushAt` advances only on success. #412 adds the independent -failure backoff. Its ordinary publication path follows this lifecycle; while parent recovery is -open, recovery's deadline takes the backoff's place. The worker event model records the fetches -outside that schedule: +failure backoff. Step 3b then unified the publication and parent-recovery deadlines. While parent +recovery is open, the due attempt services it. The following diagram describes publication retries; +the linked recovery contract adds admission pause and producer resume. Resync fetches can still +bypass backoff today; step 5b closes that gap. ```mermaid stateDiagram-v2 @@ -258,8 +272,8 @@ Ordered by how much they should worry you. [`push-notification-and-reconcile-trigger.md`](push-notification-and-reconcile-trigger.md), which spent a whole change removing two requests per publication. 2. **Hosted Git rate limits.** With serialization the bound is one push per push-duration, which on - a fast remote is several per second sustained. GitHub's secondary rate limits are real and are - not documented as a fixed number, so this needs observation rather than arithmetic. + a fast remote is several per second sustained. Measure throttling on the supported hosts before + increasing publication frequency. 3. **`commit.window.idleTimeout: 0s` becomes a push per event.** On a healthy remote the cooldown spaces publications even with that setting. Anyone who set `0s` for prompt commits did not necessarily ask for prompt *pushes*. @@ -279,7 +293,7 @@ The ledger built in PR #382 answers most of this, and turns the argument into nu | # | Operation to add to the ledger | What it settles | | --- | --- | --- | | 1 | One author, one target, a burst at default window, with no early closure | Confirms §2 for ordinary timer closures | -| 2 | Two authors alternating on one branch | Prices the identity-boundary case that §3 claims is the real one | +| 2 | Two authors alternating on one branch | Prices the shared-branch identity changes described in §3 | | 3 | Three `GitTarget`s on one branch, edited together | The multi-tenant bill for option B | | 4 | `commit.window.idleTimeout: 0s`, ten events | Risk 3, as a number | | 5 | A failing remote with continued arrivals, including parent recovery | Confirms retry pacing, including across the parent-recovery handoff | @@ -292,8 +306,10 @@ held-request retry test, to measure those workloads. 1. **Keep the success cooldown while evaluating its cost.** Changing publication cadence needs a separate change and its own validation. -2. **Option C is complete.** #412 built the failure backoff and, after review, the - parent-recovery handoff: new commits respect retry pacing after a missing parent returns. +2. **Option C's retry schedule is built.** Step 3b unified it with parent recovery. The outage work + that followed it (admission correctness, bounded Git calls, pause/resume, and visible status) is + built too, in the branch-worker pending-writes plan. Removing the success cooldown would not have + closed any of those gaps. 3. **Build the event-loop ledger rows in §9 before changing the success wait.** Retry tests prove progress, but the ledger still needs the cost of these scheduled workloads. Row 2 prices the multi-identity case. diff --git a/docs/design/push-notification-and-reconcile-trigger.md b/docs/design/push-notification-and-reconcile-trigger.md index 2758d1e4..9bd51640 100644 --- a/docs/design/push-notification-and-reconcile-trigger.md +++ b/docs/design/push-notification-and-reconcile-trigger.md @@ -163,6 +163,12 @@ stateDiagram-v2 at a SHA that is on no remote. `worktreeDirty` cannot cover this, because the reset **clears** `worktreeDirty` at exactly the moment the problem starts. +The second and third questions are now one count, `checkoutApplied`: how many retained writes the +checkout holds on top of its root. Unknown is `worktreeDirty`; a count below the retained writes is +`replayRequired`, and a reset sets it to zero, so the reset no longer hides the problem it starts. +The loop's `materialize` is the one place that acts on it +([`gittarget-branch-worker-pending-writes.md`](gittarget-branch-worker-pending-writes.md), step 1). + A cycle plans straight onto the worktree only when the base is trusted and the tree is clean: ```mermaid diff --git a/docs/design/source-scope-simplification.md b/docs/design/source-scope-simplification.md index 3041f0eb..2276fbb4 100644 --- a/docs/design/source-scope-simplification.md +++ b/docs/design/source-scope-simplification.md @@ -167,9 +167,9 @@ That is a deletion in `watchrule_compile.go`, not new machinery. One trap, and the code already records it. A cluster-wide collection is a **peer** of a named-namespace collection on the same type, never a replacement. `CollectionKey`'s doc comment names the bug from a previous attempt: collapsing the two "widened the named rule's stream to every -namespace its credential could read". So a target carrying both `*` and a named rule for one type -runs two streams over overlapping objects, and that is correct rather than something to optimize -away. +namespace its credential could read". A target carrying both `*` and a named rule for one type ran +two streams over overlapping objects; since step 5a2 of the branch-worker pending-writes plan the newer rule is +refused instead (see [overlapping collections](../configuration.md#overlapping-collections)). It is what a Kubernetes reader expects `*` to mean, and its failure is a clean 403 rather than a silent empty set. diff --git a/docs/future/ha-gittarget-distribution-plan.md b/docs/future/ha-gittarget-distribution-plan.md index f5d23045..39d6a897 100644 --- a/docs/future/ha-gittarget-distribution-plan.md +++ b/docs/future/ha-gittarget-distribution-plan.md @@ -2,10 +2,16 @@ Status: **proposed** (not started) +Reviewed 2026-10-03 alongside branch-worker steps 1 to 4 at `373bf8d7`. Redis/Valkey queue work +remains deferred. Its prerequisite, the +[in-memory pause/resume contract](../design/gittarget-branch-worker-pending-writes.md#recovery-contract), is +built in #413: capacity limits, visible failure state, and one recovery schedule. A durable queue +keeps all three; a growing external queue does not remove the need for them. + This is the owning document for the durable journal, publication recovery, retention, and the persistence/HA rollout. The [branch worker event model](../design/branch-worker-event-model.md) owns worker transitions, deadline semantics, the publication retry (built in #412), and the -operation-timeout fix, which can ship independently of this plan. +operation timeouts (built in #413), which shipped independently of this plan. ## Scope and definition of done @@ -22,10 +28,11 @@ The first HA release uses active/passive ownership: - Losing one controller Pod must not silently drop a Kubernetes-to-Git state change. The replacement may replay a change or create a no-op Git attempt. -The baseline durability contract is eventual state convergence: after recovery, Git matches the -watched Kubernetes state. The proposed extension preserves accepted save obligations, including -window membership, attribution, messages, deadlines, and outcomes. The event model defines those -execution semantics; this plan owns how their records survive failure. +The baseline durability contract is eventual state convergence under the configured write and +prune policies. A missed DELETE cannot be inferred from a snapshot under `prune.mode: onEvent`; +an `always` target permits scoped absence-based pruning. The proposed extension preserves accepted +save obligations, including window membership, attribution, messages, deadlines, and outcomes. +The event model defines those execution semantics; this plan owns how their records survive failure. Before implementing the extension, agree on retention, target replacement, and the response to an indeterminate publication result. Existing coalescing still applies, and rebuilding unpublished @@ -58,12 +65,18 @@ The repository has useful foundations, but it is not HA today. GitTargetEventStream into an in-memory BranchWorker FIFO. See [event_router.go](../../internal/watch/event_router.go) and [git_target_event_stream.go](../../internal/reconcile/git_target_event_stream.go). -- Redis currently stores resume cursors and author-attribution data. A cursor is - written after the event enters the in-memory FIFO, but before the eventual Git - push. A Pod crash in that interval can make a replacement resume past work - that only existed in RAM. This is the blocker for lossless failover. +- Redis currently stores resume cursors and author-attribution data. A cursor is written after + admission to the in-memory FIFO, before Git publication. A new watch stream starts with a fresh + replay even with a stored cursor; later reconnects may resume it. Startup can repair current + content under the write/prune policy, but cannot recover lost intermediate events or accepted + save decisions. Persisted cursors alone do not establish lossless failover. - BranchWorker keeps open commit windows, local commits, unpushed writes, and CommitRequest outcomes in memory. Its local clone is disposable. +- The pending-writes plan provides ordered pending writes, one materializer, one retry schedule, + isolated replay refusals, and an intake budget: during a retry, admission closes on everything + accepted and not yet published, queued payloads and empty records included, and producers offer + what it refuses again. That budget is not a durable retention contract or a total memory + ceiling. - Git pushes already use a remote reference compare-and-swap. PushAtomic remains the final protection against a stale owner or an external remote update. See [git_atomic_push.go](../../internal/git/git_atomic_push.go). @@ -273,6 +286,21 @@ acknowledge the journal or prove a request's historical execution. ## Surviving hours without Git +Use the same pause/recover/resume behavior as the in-memory worker, with durable admission as the +acceptance boundary. Storage changes how long accepted obligations survive and how much backlog +fits; it does not change which outcomes the worker owes. + +| Boundary | During an outage | At capacity | After recovery | +|---|---|---|---| +| Current in-memory path | Retain accepted writes while the process lives | Refuse new admission; explicit producer pause is planned | Replay retained decisions, then resume watches | +| Future durable journal | Persist accepted inputs and workflow decisions; page the active working set | Stop durable admission before eviction or storage failure | Recover accepted obligations, then catch up sources | + +If Git is unavailable but the durable store has capacity, producers can keep appending without +loading that backlog into the worker's RAM. When the store fills or cannot guarantee a durable +write, stop admission and keep the source cursor unchanged. Report whether Git publication or +journal admission is blocked; they are different dependencies and can recover independently. +Never acknowledge and trim accepted work merely to free capacity for newer observations. + State an outage budget in workload and storage terms. A useful first estimate is admitted bytes per second multiplied by outage duration, plus snapshot, index, encryption, and retention costs. Recovery also needs enough publication throughput to drain the backlog while new work arrives. @@ -282,6 +310,12 @@ policy from the first durable implementation. Stop accepting new payloads when the durable store cannot honor its contract. This does not guarantee recovery of every later Kubernetes mutation: a prolonged ingestion stop can outlast watch history. +Retained accepted work drains in branch order before a fresh source snapshot can supersede current +state. An expired watch cursor records a continuity gap; the complete scoped snapshot repairs only +what the current write/prune policy permits. Missed intermediate versions and expired delete +events are not recreated. An accepted save keeps its original obligation through this catch-up; +an unaccepted save has no durable receipt. State these limits in status and operator documentation. + Give recovery and lifecycle work a way to run under saturation, with its causal ordering intact. Reserved capacity must not let a withdrawal overtake an already accepted attach. Bound the active in-memory working set independently of journal size; keeping every persisted payload in @@ -302,9 +336,10 @@ healthy outage backlog from a worker that has stopped making progress. ## Implementation phases -The event model owns the operation-deadline fix (its publication retry shipped in #412) and -extraction of deterministic worker transitions. These phases own persistence and HA after that -boundary is available. Each phase needs its own failure tests; adding leadership must not be the +The branch-worker pending-writes plan owns admission correctness, operation deadlines, and in-memory recovery +visibility. The event model owns extraction of deterministic worker transitions. These phases own +persistence and HA after that boundary is available. Do not start them as part of finishing the +current pending-writes refactor. Each phase needs its own failure tests; adding leadership must not be the first recovery test of the journal. ### HA-0: Specify and expose branch ownership @@ -420,7 +455,9 @@ Add fault tests for each durable boundary before enabling the supported HA mode: | Empty save is redelivered | Original result is recovered; missing evidence remains explicit | | Snapshot lacks a complete marker | No sweep runs | | Watch cursor expires | Fresh replay preserves the old journal tail and scoped deletion rules | -| Journal quota fills during an outage | Admission stops explicitly; recovery retains capacity | +| Journal quota fills during an outage | Admission stops explicitly; accepted records remain; recovery retains capacity | +| Store is full when watch history expires | Gap is explicit; fresh snapshot obeys prune policy and preserves accepted saves | +| Git is down while the journal still has capacity | Durable backlog grows while the active worker memory stays bounded | | Target or provider is recreated | Old work cannot bind to the replacement by name | | Old owner completes an in-flight push | Replacement recovers the outcome and rejects stale journal transitions | | Remote branch moves | Compare-and-swap and recorded rebuild attempts preserve the pending work | diff --git a/docs/interpreting-metrics.md b/docs/interpreting-metrics.md index b7887ddf..e7f36846 100644 --- a/docs/interpreting-metrics.md +++ b/docs/interpreting-metrics.md @@ -201,12 +201,18 @@ boundary, the commit, the push. Background: | `git_pushes_total` | counter | `provider_namespace`, `provider_name`, `branch`, `outcome` | One per push cycle: `pushed` or `failed`. | | `git_push_retries_total` | counter | `provider_namespace`, `provider_name`, `branch`, `reason` | Replay rounds inside a cycle. `reason` is `remote_moved`. | | `git_push_duration_seconds` | histogram | `provider_namespace`, `provider_name`, `branch` | One cycle end to end, retries included. | -| `git_fetches_total` | counter | `provider_namespace`, `provider_name`, `branch`, `reason` | Every call that reads the remote through a `SmartFetch`. `reason` is `bootstrap` (repository preparation; **no production caller reaches it today**, so the series stays at zero) / `publication` (the head of a publication cycle) / `recovery` (re-establishing a base that could not be trusted, including a worktree a failed write left dirty) / `contention` (the reset onto the new tip after a push was rejected because somebody else moved the branch) / `push_failure_probe` (a push that failed without the remote saying anything, looking up where the branch is) / `forced_recheck` (a full re-read outside the publication cycle: a forced recheck, or the snapshot a resync judges against) / `refresh` (the periodic top-up of an idle branch, and only on the intervals where the branch had actually moved). See the note below. | -| `git_queue_drops_total` | counter | `provider_namespace`, `provider_name`, `branch`, `kind` | Work a full queue threw away. `kind` is `write` / `attach` / `resync` / `refresh`. Every increment is lost work — except `refresh`, which the next reconcile asks for again; a standing rate there means the branch's status and placement stop being topped up. | -| `git_commit_failures_total` | counter | `provider_namespace`, `provider_name`, `branch`, `kind`, `reason` | A window or request that died between routing and pushing. `kind` is `window` / `atomic`; `reason` is `refused` (a Git path a human must fix) / `error`. Every increment is a window's events lost until the next resync. | +| `git_fetches_total` | counter | `provider_namespace`, `provider_name`, `branch`, `reason` | Every call that reads the remote through a `SmartFetch`. `reason` is `bootstrap` (repository preparation; **no production caller reaches it today**, so the series stays at zero) / `publication` (the head of a publication cycle) / `recovery` (re-establishing a base that could not be trusted, including a worktree a failed write left dirty that could not be undone locally) / `contention` (the reset onto the new tip after a push was rejected because somebody else moved the branch) / `push_failure_probe` (a push that failed without the remote saying anything, looking up where the branch is) / `forced_recheck` (a full re-read outside the publication cycle: a forced recheck, or the snapshot a resync judges against) / `refresh` (the periodic top-up of an idle branch, and only on the intervals where the branch had actually moved). See the note below. | +| `git_queue_refusals_total` | counter | `provider_namespace`, `provider_name`, `branch`, `kind` | Work refused at enqueue: the queue was full, intake was paused through an outage, or the worker was stopping. `kind` is `write` / `attach` / `resync` / `refresh`. The producer keeps a refused item and offers it again: the watch delivers a write again from its cursor, the controller re-sends a save, a resync is gathered again, and the next reconcile asks for a refresh. What it can cost, when its watch cursor expires before the write comes back, is the write's own commit and, under `prune.mode: OnEvent`, a deleted object's removal from Git. | +| `git_commit_failures_total` | counter | `provider_namespace`, `provider_name`, `branch`, `kind`, `reason` | A window or request that died between routing and pushing. `kind` is `window` / `atomic`; `reason` is `refused` (a Git path a human must fix) / `error`. Every increment is a window's events lost until the next resync. An unreachable remote does not count: a decided window waits for the publication retry instead. | | `git_commit_windows_total` | counter | `gittarget_namespace`, `gittarget_name`, `close_reason`, `timer_source` | One per closed commit window, whether or not it then produced a commit. `close_reason` is `idle_timeout` / `max_duration` / `attach_next` / `identity_change` / `buffer_limit` / `resync_before_apply` / `atomic_before_apply` / `shutdown`; `timer_source` is `target` or `commit_request`. See [tuning commit windows](#tuning-commit-windows). | | `git_commit_window_duration_seconds` | histogram | `gittarget_namespace`, `gittarget_name` | How long each window collected: from the write that opened it to the start of its finalize. Its count equals `git_commit_windows_total` summed over `close_reason` and `timer_source`. | | `git_queue_depth` | gauge | `provider_namespace`, `provider_name`, `branch` | Pending + in-flight + committed-but-unpushed. Read at scrape time. | +| `git_retained_bytes` | gauge | `provider_namespace`, `provider_name`, `branch` | What the worker holds until a push lands, in the units of the retained-byte budget: the open window, the retained writes, deferred snapshots and waiting saves, each with a 1 KiB per-item charge. Serialized bytes, not heap. | +| `git_retained_writes` | gauge | `provider_namespace`, `provider_name`, `branch` | Decided writes waiting for a push. | +| `git_intake_paused` | gauge | `provider_namespace`, `provider_name`, `branch` | 1 while the worker refuses new writes, saves and resyncs because an outage filled its budget; 0 otherwise. | +| `git_oldest_retained_write_timestamp_seconds` | gauge | `provider_namespace`, `provider_name`, `branch` | When the oldest write still waiting for a push was decided, as a Unix timestamp. No sample while nothing waits. | +| `git_next_retry_timestamp_seconds` | gauge | `provider_namespace`, `provider_name`, `branch` | When the next attempt at owed work is due, as a Unix timestamp. No sample while nothing is owed. | +| `git_materialization_failures_total` | counter | `provider_namespace`, `provider_name`, `branch`, `reason` | Attempts to commit decided writes that stopped before any push: `reason` is `unreachable` or `parent_unavailable`. Nothing is lost; the writes wait for the retry. A failure `git_pushes_total` cannot see, because no push cycle started. | | `git_branch_targets` | gauge | `provider_namespace`, `provider_name`, `branch`, `gittarget_namespace`, `gittarget_name`, `source_cluster` | Always 1. The **join** between the GitTarget-labeled half of the pipeline and the branch-labeled half. Published per configured GitTarget, whether or not its worker runs. | | `commit_requests_total` | counter | `outcome`, `gittarget_namespace`, `gittarget_name` | One per `CommitRequest` terminal **decision**. `outcome` is `committed` / `no_window` / `window_mismatch` / `already_present` / `failed`. A target that never resolved publishes the empty pair. See the counting note below. | | `placements_total` | counter | `source`, `disposition`, `gittarget_namespace`, `gittarget_name`, `group`, `version`, `resource` | One per new document at a resolved path. | @@ -256,6 +262,12 @@ the plain reset when nothing is retained, and the reset-and-replay when somethin two happens depends only on whether a push was in cooldown at the time, so splitting them would hide half of every recovery from the series you are meant to read this way. +A write that fails part-way no longer reaches this series at all in the common case: the worker +undoes the failed batch locally, back to the commit it started on, and the next commit plans on the +checkout without a fetch. Only a failed write that cannot be undone locally leaves the worktree +dirty and costs a `recovery` fetch, so a `recovery` fetch now points at a reset that went wrong, a +parent change, or a replay that did not finish. + `contention` tracks `git_push_retries_total` one-for-one, because both count a confirmed moved remote. `push_failure_probe` moving on its own is pushes failing for a reason that is not another writer — check credentials and connectivity, not the branch. @@ -277,15 +289,24 @@ operator questions, and the alternative (a metric per call site) would multiply telling anyone anything the `reason` label does not. Nothing here should grow an eighth series without a question it is the only way to answer. -### Sizing the branch worker queue against `git_queue_drops_total` - -`git_queue_drops_total{kind="write"}` is the one loss the end state hides. The enqueue onto a -branch worker is deliberately non-blocking, so a full queue throws the write away rather than -stalling the watch path behind a slow remote; the next resync's mark-and-sweep then heals the -mirror, and the target goes on reporting `RenderMatchesLive=True`. What does not come back is the -live, attributed commit for the dropped write — the change happened, and the history does not say -so. Treat any sustained increment as a signal to raise -`controllerManager.branchWorkerQueueDepth`. +### Sizing the branch worker queue against `git_queue_refusals_total` + +`git_queue_refusals_total{kind="write"}` on a healthy branch is a queue too small for its bursts. The +enqueue onto a branch worker is deliberately non-blocking, so a full queue refuses the write rather +than stalling the watch path behind a slow remote. The watch does not advance its cursor past a +refused write: its session ends, it reconnects, and it delivers the write again. So nothing is lost +while that cursor lives, but each refusal costs the stream a reconnect, and a burst that keeps the +queue full keeps the stream reconnecting. If the cursor expires first, the stream rebuilds from a +fresh snapshot, and that recovery follows the target's prune policy. It restores the current +content of every object it lists, but the write's own attributed commit is never made: the change +happened, and the history does not say so. A delete is weaker still: under the default +`prune.mode: OnEvent` a snapshot cannot infer one from absence, so an object deleted meanwhile keeps +its file in Git until `Always` sweeps it or someone removes it; see the [recovery contract][recovery]. +Treat any sustained increment on a healthy branch as a signal to raise +`controllerManager.branchWorkerQueueDepth`. During an outage, refusals are the intake pause instead; +see the backpressure paragraph below. + +[recovery]: design/gittarget-branch-worker-pending-writes.md#resume-and-the-three-meanings-of-replay **The depth gauge will not warn you first.** `git_queue_depth` shows the traffic you have, and the bursts that overrun the queue are usually the traffic you do not: an unpaced `kubectl delete` of a @@ -296,16 +317,17 @@ administrative one. **Size it by burst, not by rate.** A queue slot holds one write request, so what has to fit is the number of concurrent write requests a bounded burst can produce — roughly `(concurrent writers) × (GitTargets sharing the branch worker)`. A burst that cannot exceed the -depth cannot drop at all, whatever its arrival shape. Note the second factor: workers are keyed by +depth is never refused, whatever its arrival shape. Note the second factor: workers are keyed by `(GitProvider namespace, GitProvider name, branch)`, so pointing a second GitTarget at a branch halves the headroom of a worker already in use, and nothing in the API surface says so. Confirm which targets share one worker by joining on `git_branch_targets` — several targets against a single `git_queue_depth` series is one worker serving all of them. -**What it costs.** `--branch-buffer-max-size` does **not** cover this queue. That cap bounds the -open commit window plus the writes retained for replay until a push succeeds, and it is accounted -only once an item *leaves* the queue — so queue depth × payload is additional pod memory, on top of -it. +**What it costs.** On a healthy branch `--branch-buffer-max-size` does **not** cover this queue. That +cap bounds the open commit window plus the writes retained for replay until a push succeeds, and it +is accounted only once an item *leaves* the queue, so queue depth × payload is additional pod +memory, on top of it. During a Git outage the same cap is charged at enqueue and covers the queue +as well; see the backpressure paragraph below. Budget it as **queue depth × serialized size × ~6**. A queued object is held as an unstructured map, which measures around six times the bytes it serializes to; the channel itself is negligible @@ -324,11 +346,26 @@ red trains people to ignore it. Four classes: | --- | --- | --- | | **expected** | `routed`, `unchanged`, `bookmark`, `shutdown`, `written`, `deleted_live`, `deleted_sweep`, `retained`, `cached` | the pipeline working. `unchanged` is a document considered and found identical; `refused` is a document the writer declined to place, which is loss | | **degraded** | `author_kind="unresolved"`, `mode="list_fallback"`, `not_object`, `stream_error`, weak attribution tiers | working, on weaker evidence | -| **recoverable** | `git_pushes_total{outcome="failed"}` — the writes are retained and retried | alert on it **sustained with no successes**, never on one occurrence | -| **loss** | `route_failed`, any `git_queue_drops_total`, any `git_commit_failures_total`, `placement_refusals_total`, `git_documents_total{outcome="refused"}` | an observed change that did not reach Git | - -`route_failed` and a queue drop **overlap**: a full worker queue is one of the ways a route fails, so -one dropped event increments both. They are two views of one event, and summing them double-counts. +| **recoverable** | `git_pushes_total{outcome="failed"}` (the writes are retained and retried); `route_failed` and any `git_queue_refusals_total` (the producer keeps the refused item and offers it again) | alert on it **sustained**: pushes with no successes, refusals on a branch whose intake is not paused. Never on one occurrence | +| **loss** | any `git_commit_failures_total`, `placement_refusals_total`, `git_documents_total{outcome="refused"}` | an observed change that did not reach Git | + +`route_failed` and a queue refusal **overlap**: a refused enqueue is one of the ways a route fails, +so one refused event increments both. They are two views of one event, and summing them +double-counts. A refused event becomes loss only when its watch cursor expires before it comes back: +the fresh snapshot then restores the object's content without that change's own commit, and, under +`prune.mode: OnEvent`, does not remove a deleted object's file. + +**Backpressure during an outage looks like loss and is not.** A branch worker keeps every decided +write while its remote cannot be reached, and bounds that at intake: while a failed publication +waits for its retry, work that would take what it holds past the retained-byte budget (queued work, +the open window, retained writes, deferred snapshots and waiting saves) pauses its intake. It then +refuses new writes, saves and resyncs the way a full queue does, and both series rise. Intake stays +paused until a push lands. The watch does not advance its cursor past a refused event; its stream +waits for the branch to reopen intake and then delivers the event again. Only a cursor that expires +during a long outage turns it into a fresh snapshot, which converges the folder under the target's +prune policy and without that event's own commit. Read these increments beside `git_intake_paused == 1`, or +`git_pushes_total{outcome="failed"}` with no successes: that is an outage holding work back, and +every `GitTarget` on the branch says so on `Ready`. A healthy branch never refuses for this reason. `placement_refusals_total` and `git_documents_total{outcome="refused"}` **overlap the same way**: a resource the writer declines to place increments both, so the loss class holds two views of one @@ -409,12 +446,13 @@ sum by (provider_namespace, provider_name, branch) ( rate(gitopsreverser_git_pushes_total[5m])) ``` -**Is work being thrown away?** Both should be flat zero. A queue drop is a saturated worker; a -commit failure takes a whole window with it, and `reason="refused"` will not clear until someone -fixes the Git path: +**Is work being refused, or thrown away?** Both should be flat zero on a healthy branch. A queue +refusal is a saturated worker or a paused intake, and the producer offers the work again; a commit +failure takes a whole window with it, and `reason="refused"` will not clear until someone fixes the +Git path: ```promql -sum by (kind) (rate(gitopsreverser_git_queue_drops_total[5m])) +sum by (kind) (rate(gitopsreverser_git_queue_refusals_total[5m])) sum by (kind, reason) (rate(gitopsreverser_git_commit_failures_total[5m])) ``` @@ -578,7 +616,7 @@ picture towards `max_duration`. - `buffer_limit` means the worker's retained-write budget was reached, and the window was flushed to stay within it. Writes committed but not yet pushed count towards that budget, so this reads as "the branch is holding too much", not as a measurement of pod memory. See - [sizing the branch worker queue](#sizing-the-branch-worker-queue-against-git_queue_drops_total). + [sizing the branch worker queue](#sizing-the-branch-worker-queue-against-git_queue_refusals_total). - `resync_before_apply` and `atomic_before_apply` close the window so a resync or an atomic write can apply after it; `shutdown` is the worker stopping. @@ -1248,8 +1286,10 @@ sum by (outcome) (rate(gitopsreverser_secret_encryptions_total[5m])) | `time() - gitopsreverser_watch_plan_oldest_dirty_since_timestamp_seconds > 120`, `for: 5m` | A GitTarget cannot be planned — its source cluster or catalog is unreachable — and its mirror is not converging. The `time() -` is not optional: the gauge is a Unix timestamp, so a bare `> 120` is true of every instant since 1970 and would fire on ordinary planning. | | `rate(gitopsreverser_watch_plan_passes_total{outcome="timed_out"}[15m]) > 0` sustained | Plan passes are hitting their per-target deadline; the target stays dirty and installs nothing. | | `rate(gitopsreverser_secret_encryptions_total{outcome="failed"}[10m]) > 0` | Secret writes are being rejected by the encryption path. | -| `rate(gitopsreverser_git_queue_drops_total[5m]) > 0` | The queue saturated and work was thrown away. | +| `rate(gitopsreverser_git_queue_refusals_total[5m]) > 0 unless on(provider_namespace, provider_name, branch) gitopsreverser_git_intake_paused == 1`, `for: 10m` | A healthy branch keeps refusing work: its queue is too small for its bursts. Producers offer the work again, so this is lag; only a watch cursor that expires first loses commits, and under `OnEvent` deletes. During an outage the intake-paused alert below covers it. | | `gitopsreverser_git_queue_depth` rising and not draining | A branch worker is backing up against a stalled remote. | +| `time() - gitopsreverser_git_oldest_retained_write_timestamp_seconds > 900`, `for: 5m` | A branch has not published for 15 minutes: its writes, and any save riding them, are held. The `time() -` is not optional, as for the plan-dirty gauge above. | +| `gitopsreverser_git_intake_paused == 1`, `for: 10m` | An outage filled a branch's budget: new changes on its `GitTarget`s are refused until a push lands. | | `gitopsreverser_resource_condition{type="Ready", status="False"} == 1`, `for: 15m` | A declared object has not been accepted. The `== 1` is not optional: each condition publishes one series per status, so the selector alone matches the two zeroes too. | | `gitopsreverser_resource_condition{type="Stalled", status="True"} == 1` | Permanently wedged: `Stalled` is the kstatus "nothing will retry" signal, so this needs a human and will not clear on its own. `reason` names the gate. | | any `gitopsreverser_*` series with `otel_metric_overflow="true"` | A metric family exceeded its per-instrument cardinality cap, so the labels identifying those series are gone and every query over that family is under-reporting. Raise the cap or cut a label. | diff --git a/docs/spec/commitrequest-design.md b/docs/spec/commitrequest-design.md index 337651af..bddcc21b 100644 --- a/docs/spec/commitrequest-design.md +++ b/docs/spec/commitrequest-design.md @@ -1,6 +1,7 @@ # CommitRequest window finalization -> **spec** — current behaviour. The code depends on this document; change one, change the other. +> **spec**: current behavior, reviewed at `373bf8d7` on 2026-10-03. +> The code depends on this document; change one, change the other. > Index: [`../INDEX.md`](../INDEX.md) A `CommitRequest` is a one-shot “save now” command for one `GitTarget`. It does not mirror the @@ -100,7 +101,11 @@ object is visible. | No same window before deadline | `Ready=True`, `Pushed=False`, reason `NoWindow` or `WindowMismatch` | | Nothing to commit, with `whenNothingToCommit: CommitEmpty` | `Ready=True`, `Pushed=True`, reason `NoWindow` or `AlreadyPresent`; `status.commit` is the empty commit | | Window produced no diff, and the remote agreed | `Ready=True`, `Pushed=False`, reason `AlreadyPresent` | -| Finalize or push error, or a target that may not be written | `Ready=False`, `Pushed=False`, `Stalled=True`, reason `FinalizeFailed` | +| Terminal write failure, withdrawal before the worker holds it, shutdown with an unresolved held write, or a target that may not be written | `Ready=False`, `Pushed=False`, `Stalled=True`, reason `FinalizeFailed` | + +A retryable materialization or push failure keeps a held request in `WaitingForPush`; it is not a +terminal outcome. Publication has to settle the request or the worker must establish that it will +no longer execute it before the controller can report failure. `Reconciling=True` is the normal in-progress state, with the phase the worker reports as its reason: `Progressing`, `WaitingForWorker`, `WaitingForWindow`, `CollectingWindow`, `WaitingForPush`. Once the @@ -127,13 +132,13 @@ produced no diff used to resolve `AlreadyPresent` at finalize, on the strength o That was only sound while every cycle fetched before it planned. It no longer does (see [inbound push notification](../design/push-notification-and-reconcile-trigger.md) §3), so the plan may have run against a tree the remote has moved past, and the replay that follows a rejected push can turn the -same captured object into a real commit. "Already present" is a claim about the remote, so the +same captured object into a commit. "Already present" is a claim about the remote, so the remote settles it: a no-diff request resolves `AlreadyPresent` when the push confirms there was nothing to add, and `Committed` when the replay produced a commit after all. -**A request whose window has committed stays identifiable until then.** It is neither pending nor -resolved in that interval, and the controller keeps re-sending its attach every couple of seconds -until it reads an outcome. The worker marks such a request committed rather than forgetting it, so +**A request whose write is decided stays identifiable until publication.** It remains attached to +that write, and the controller keeps re-sending its attach every couple of seconds until it reads +an outcome. The worker keeps that attachment even when local materialization is deferred, so the re-send is recognized as the same request: it cannot register afresh and expire into `NoWindow` while its commit waits out the push cooldown, and it cannot claim the next same-author window and stamp its message on somebody else's commit. Its attach deadline is spent and no longer @@ -147,28 +152,60 @@ remote" says what happened. Two gates stop a GitTarget being written at all: `spec.suspend`, and a render-fidelity check that has not passed. One branch worker serves every GitTarget on its branch, and each gate applies per -GitTarget, so a sibling target's save is unaffected. A third condition, a failed push, is not a gate -but a retry: +GitTarget, so a sibling target's save is unaffected. A third condition, an unreachable remote, is +not a gate but a retry: -| Path | Suspended | Render fidelity not established | Push fails | +| Path | Suspended | Render fidelity not established | Remote unreachable (commit or push) | |---|---|---|---| -| Live window, no request | The window still scans, writes nothing | Writes dropped on arrival, and the window dropped at close; the next resync re-derives them | The commit is kept and published by a later push | +| Live window, no request | The window still scans, writes nothing | Writes dropped on arrival, and the window dropped at close; the next resync re-derives them | The decided write is kept and published by a later push | | Window a request is attached to | The request fails; the window still scans | The request fails | The request stays `WaitingForPush` | | Request no window reached, `Resolve` or `CommitEmpty` | The request fails; no empty commit | The request fails; no empty commit | `CommitEmpty`: the record is kept, `WaitingForPush` | -| A gate closes after the request's local commit | The commit is pushed and the request resolves with it | Same | Same, once a push lands | +| A gate closes after the request's window closed | The write is pushed and the request resolves with it | Same | Same, once a push lands | A request fails with `FinalizeFailed` and the gate as its message, under either `whenNothingToCommit`: `NoWindow` would say the save saw no writes, and a target that drops or suppresses its writes cannot say that. `WindowMismatch` is the exception, because another author's window refused that request and its reason says so. -The gates are read when the commit is made, as `suspend` already was for every write. A local commit -made before a gate closed is pushed rather than kept back, because a commit that never left the -operator's checkout would surface later, out of order. - -A failed push schedules its own retry (10s, doubling to 5m), with parent recovery owning attempts -while its obligation remains active. A request riding the retained write stays `WaitingForPush` -until publication resolves it or worker shutdown fails it. An unavailable remote can leave it -pending indefinitely, but retries no longer depend on another commit arriving. +The gates are read when the window closes, which is when the write is decided and normally when it +is committed. A write decided before a gate closed is pushed rather than kept back, because a write +that never left the worker would surface later, out of order. + +A decided write is kept when the remote cannot be reached, whether that happens at its commit (a +rebuild of the checkout that needs a fetch) or at its push. The worker retries on its own schedule +(10s, doubling to 5m), with parent recovery owning attempts while its obligation remains active. A +request riding the write stays `WaitingForPush` until publication resolves it or worker shutdown +fails it. An unavailable remote can leave it pending indefinitely, but retries no longer depend on +another commit arriving. Only a failure of the write itself fails the request: a refused plan, or a +write that cannot be made. A missing parent branch is a remote that cannot take the write yet, and +holds the request the same way, and its `WaitingForPush` message says why the push has not landed, +in the words its `GitTarget` uses. While an outage has paused the branch's intake, a new request is +refused at admission and the controller sends it again; past its safety window, it fails with +`FinalizeFailed` only after withdrawal confirms that the worker does not hold it. + +### Capacity, replay, and restart limits + +The budget charges every kept item its payload bytes, its message, and a fixed overhead for its +bookkeeping, so a stream of empty saves fills it too. It counts queued work, retained writes, and +requests still waiting for a window. A refusal pauses the branch's intake until a push lands; see +the [branch-worker pending-writes plan](../design/gittarget-branch-worker-pending-writes.md#recovery-contract). A refused +request is sent again by the controller. + +For a request the worker already holds, saturation does not change its window membership or +replace its decided write with a newer snapshot. A refused replay entry fails its own request; +unrelated retained writes can still publish. Requests accepted but still waiting for a window +keep their existing attach deadline and no-window behavior. + +An unaccepted request is retried within the controller's safety bound. Neither an API object +created successfully nor a later source snapshot proves that the worker accepted that save. +Resource watches and save commands arrive independently: a save is not a barrier covering every +Kubernetes mutation before its creation. + +Pausing the live worker preserves accepted work in memory. Restarting the process does not preserve +its attachments or outcome receipts. Startup replays current watched state, subject to pruning, +but cannot recreate the same save history. A push accepted immediately before a crash can therefore be +followed by another empty commit when the request is resent. A lost push response also needs remote +evidence; equal tree content alone cannot prove that an empty save's message was published. Durable +save recovery belongs to the [future HA plan](../future/ha-gittarget-distribution-plan.md). The complete status vocabulary is in the [status conditions guide](status-conditions-guide.md). diff --git a/internal/controller/clusterwatchrule_admission_test.go b/internal/controller/clusterwatchrule_admission_test.go index f6f8bbc4..5ef2d900 100644 --- a/internal/controller/clusterwatchrule_admission_test.go +++ b/internal/controller/clusterwatchrule_admission_test.go @@ -33,8 +33,8 @@ import ( type cwaWatchManager struct { replans int onReconcile func() - // selectorConflict, when set, is the refusal message both ObjectSelectorConflict methods report. - selectorConflict string + // collectionOverlap, when set, is the refusal message both CollectionOverlap methods report. + collectionOverlap string } func (m *cwaWatchManager) TriggerRuleChange(internaltypes.ResourceReference) { @@ -63,16 +63,16 @@ func (m *cwaWatchManager) ResolveClusterWatchRuleResources( return true, "resolved" } -func (m *cwaWatchManager) ObjectSelectorConflictForWatchRule( +func (m *cwaWatchManager) CollectionOverlapForWatchRule( configbutleraiv1alpha3.WatchRule, ) (bool, string) { - return m.selectorConflict != "", m.selectorConflict + return m.collectionOverlap != "", m.collectionOverlap } -func (m *cwaWatchManager) ObjectSelectorConflictForClusterWatchRule( +func (m *cwaWatchManager) CollectionOverlapForClusterWatchRule( configbutleraiv1alpha3.ClusterWatchRule, ) (bool, string) { - return m.selectorConflict != "", m.selectorConflict + return m.collectionOverlap != "", m.collectionOverlap } func (m *cwaWatchManager) StreamSummaryForGitTarget(internaltypes.ResourceReference) watch.StreamSummary { diff --git a/internal/controller/clusterwatchrule_controller.go b/internal/controller/clusterwatchrule_controller.go index 6594f673..b8dc5904 100644 --- a/internal/controller/clusterwatchrule_controller.go +++ b/internal/controller/clusterwatchrule_controller.go @@ -361,8 +361,8 @@ func (r *ClusterWatchRuleReconciler) setResourceResolutionCondition( st *reconcileStatus, clusterRule *configbutleraiv1alpha3.ClusterWatchRule, ) { - if refused, message := r.WatchManager.ObjectSelectorConflictForClusterWatchRule(*clusterRule); refused { - st.set(ConditionTypeResourcesResolved, metav1.ConditionFalse, watch.ReasonObjectSelectorConflict, message) + if refused, message := r.WatchManager.CollectionOverlapForClusterWatchRule(*clusterRule); refused { + st.set(ConditionTypeResourcesResolved, metav1.ConditionFalse, watch.ReasonCollectionOverlap, message) return } resolved, message := r.WatchManager.ResolveClusterWatchRuleResources(ctx, *clusterRule) diff --git a/internal/controller/commitrequest_controller.go b/internal/controller/commitrequest_controller.go index 434743e0..a99ef732 100644 --- a/internal/controller/commitrequest_controller.go +++ b/internal/controller/commitrequest_controller.go @@ -281,7 +281,7 @@ func (r *CommitRequestReconciler) attachCommitRequest( log.V(1).Info("CommitRequest attach not yet serviceable; will retry", "name", req.NamespacedName, "err", serviceErr.Error()) } - return r.pollCommitRequest(ctx, commitRequest, attribution, result.Phase) + return r.pollCommitRequest(ctx, commitRequest, attribution, result.Phase, result.Held) } if result.Err != nil { @@ -312,7 +312,7 @@ func (r *CommitRequestReconciler) withdrawCommitRequest( "name", req.NamespacedName, "err", withdrawErr.Error()) } // Not answered yet, or the worker holds the request: keep polling. - return r.pollCommitRequest(ctx, commitRequest, attribution, result.Phase) + return r.pollCommitRequest(ctx, commitRequest, attribution, result.Phase, result.Held) } if !errors.Is(result.Err, git.ErrCommitRequestWithdrawn) { @@ -349,8 +349,9 @@ func (r *CommitRequestReconciler) pollCommitRequest( commitRequest *configbutleraiv1alpha3.CommitRequest, attribution commitRequestAttribution, phase git.CommitRequestPhase, + held string, ) (ctrl.Result, error) { - if err := r.recordWorkerPhase(ctx, commitRequest, attribution, phase); err != nil { + if err := r.recordWorkerPhase(ctx, commitRequest, attribution, phase, held); err != nil { return ctrl.Result{}, err } return ctrl.Result{RequeueAfter: commitRequestPollInterval}, nil @@ -367,7 +368,7 @@ func (r *CommitRequestReconciler) stampFirstSightConditions( if findCondition(commitRequest.Status.Conditions, ConditionTypeReady) != nil { return nil } - markCommitRequestProgressing(commitRequest, attribution, "") + markCommitRequestProgressing(commitRequest, attribution, "", "") if err := r.Status().Update(ctx, commitRequest); err != nil { return err } @@ -453,15 +454,29 @@ func (r *CommitRequestReconciler) recordWorkerPhase( commitRequest *configbutleraiv1alpha3.CommitRequest, attribution commitRequestAttribution, phase git.CommitRequestPhase, + held string, ) error { - current := findCondition(commitRequest.Status.Conditions, ConditionTypeReconciling) - if current != nil && (phase == "" || current.Reason == string(phase)) { + if progressRecorded(commitRequest, phase, held) { return nil } - markCommitRequestProgressing(commitRequest, attribution, phase) + markCommitRequestProgressing(commitRequest, attribution, phase, held) return r.Status().Update(ctx, commitRequest) } +// progressRecorded reports whether the request's status already says this phase, and why it is +// held, so a poll writes only what changed. An empty phase says nothing new. +func progressRecorded(cr *configbutleraiv1alpha3.CommitRequest, phase git.CommitRequestPhase, held string) bool { + current := findCondition(cr.Status.Conditions, ConditionTypeReconciling) + if current == nil { + return false + } + if phase == "" { + return true + } + reason, message := progressFor(phase, held) + return current.Reason == reason && current.Message == message +} + // loadActionableCommitRequest fetches the CommitRequest and short-circuits // everything that must not reach the finalize: a deleted object, a terminal // outcome, and — via an uncached re-read — a stale cache echo of our own diff --git a/internal/controller/commitrequest_finalize.go b/internal/controller/commitrequest_finalize.go index 61ac2f46..313a31ae 100644 --- a/internal/controller/commitrequest_finalize.go +++ b/internal/controller/commitrequest_finalize.go @@ -189,8 +189,10 @@ const ( pushPendingMessage = "the commit has not been pushed yet" ) -// progressFor maps a worker-reported phase to its progress reason and message. -func progressFor(phase git.CommitRequestPhase) (string, string) { +// progressFor maps a worker-reported phase to its progress reason and message. held is why a +// request waiting for its push has not reached the remote yet: the branch worker's publication +// report, in the words its GitTarget uses. +func progressFor(phase git.CommitRequestPhase, held string) (string, string) { switch phase { case git.PhaseWaitingForWorker: return string(phase), waitingForWorkerMessage @@ -199,6 +201,9 @@ func progressFor(phase git.CommitRequestPhase) (string, string) { case git.PhaseCollectingWindow: return string(phase), collectingWindowMessage case git.PhaseWaitingForPush: + if held != "" { + return string(phase), waitingForPushMessage + ". " + held + } return string(phase), waitingForPushMessage default: return ReasonProgressing, progressingMessage @@ -221,10 +226,11 @@ func markCommitRequestProgressing( cr *configv1alpha3.CommitRequest, attribution commitRequestAttribution, phase git.CommitRequestPhase, + held string, ) { cr.Status.ObservedGeneration = cr.Generation setCommitRequestAttributed(cr, attribution) - reason, message := progressFor(phase) + reason, message := progressFor(phase, held) setCommitRequestProgress(cr, reason, message) } diff --git a/internal/controller/constants.go b/internal/controller/constants.go index 5270612c..ae80699a 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -34,11 +34,11 @@ type WatchManagerInterface interface { ResolveWatchRuleResources(ctx context.Context, rule configv1alpha3.WatchRule) (bool, string) ResolveClusterWatchRuleResources(ctx context.Context, rule configv1alpha3.ClusterWatchRule) (bool, string) - // ObjectSelectorConflictForWatchRule and ObjectSelectorConflictForClusterWatchRule report - // whether a compiled rule is refused because its collections overlap another rule's on the - // same GitTarget with a different objectSelector; the older rule keeps the collection. - ObjectSelectorConflictForWatchRule(rule configv1alpha3.WatchRule) (bool, string) - ObjectSelectorConflictForClusterWatchRule(rule configv1alpha3.ClusterWatchRule) (bool, string) + // CollectionOverlapForWatchRule and CollectionOverlapForClusterWatchRule report + // whether a compiled rule is refused because its collections overlap a different collection on + // the same GitTarget; the older rule keeps the collection. + CollectionOverlapForWatchRule(rule configv1alpha3.WatchRule) (bool, string) + CollectionOverlapForClusterWatchRule(rule configv1alpha3.ClusterWatchRule) (bool, string) StreamSummaryForGitTarget(gitDest types.ResourceReference) watch.StreamSummary StreamSummaryForWatchRule(rule configv1alpha3.WatchRule) watch.StreamSummary StreamSummaryForClusterWatchRule(rule configv1alpha3.ClusterWatchRule) watch.StreamSummary @@ -185,10 +185,10 @@ const ( // has streams pending replay completion. Stream status is computed during reconcile, so // this keeps status.streams fresh while watches converge. RequeueStreamSettleInterval = 10 * time.Second - // RequeueSelectorConflictInterval is how often a rule refused for an ObjectSelectorConflict + // RequeueCollectionOverlapInterval is how often a rule refused for a CollectionOverlap // re-checks the conflict. It is a fallback: the stream-state event of the sibling rule that // cleared it usually arrives first, but that event is best-effort. - RequeueSelectorConflictInterval = 30 * time.Second + RequeueCollectionOverlapInterval = 30 * time.Second // RequeueWriteLostInterval is how soon a reconcile whose status write lost the optimistic-lock // race comes back. It is short because there is nothing to wait FOR: the write was rejected // against a resourceVersion this reconcile had already read, so the only thing that has to diff --git a/internal/controller/gittarget_controller.go b/internal/controller/gittarget_controller.go index 5b50e503..17c382e7 100644 --- a/internal/controller/gittarget_controller.go +++ b/internal/controller/gittarget_controller.go @@ -144,10 +144,6 @@ type GitTargetReconciler struct { // value is usable. reconcileRequests reconcileRequestTracker - // snapshotRequests remembers which of a branch worker's snapshot requests were acted on, so a - // missing parent's recovery forces one recheck per request rather than one per reconcile. - snapshotRequests snapshotRequestTracker - // remotePublications is what each GitTarget last wrote to status.remote: when, and about which // repository. It is internal by design; see remotePublicationLedger. remotePublications remotePublicationLedger @@ -188,10 +184,10 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( st := beginStatus(r.Client, r.Recorder, &target) gitPathWasRefused := conditionIsFalse(target.Status.Conditions, GitTargetConditionGitPathAccepted) - // Writes are dropped while the parent is missing, so its return is recovered the way a refused - // folder is: by a recheck that re-derives what the cluster holds. The branch worker decides - // when, and asks; this acts once per request. - parentRecheck, recoveringParent := r.parentRecovery(&target, target.Namespace) + // The branch worker keeps the writes decided while the parent is missing and publishes them when + // it returns, so nothing has to be re-derived; this only reports the progress. + recoveringParent := r.parentRecovering(&target, target.Namespace) + publication := r.branchPublication(&target, target.Namespace) providerNS := target.Namespace // One read of the GitProvider for everything below it; see getGitProvider. @@ -277,7 +273,7 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // A standing reconcile request forces the same re-check a refused Git path does: the watch // plane re-anchors the target's streams, which is what makes it re-read the folder rather than // wait for the periodic pass. Taken once per distinct annotation value. - forceRecheck := gitPathWasRefused || parentRecheck || r.reconcileRequests.take( + forceRecheck := gitPathWasRefused || r.reconcileRequests.take( types.NewResourceReference(target.Name, target.Namespace), reconcileRequestedAt(&target)) observed := r.observeDataPlane(&target, sourceProvider, forceRecheck, log) st.setValue(GitTargetConditionStreamsRunning, observed.axes.Streams) @@ -289,7 +285,7 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( rd := newGitTargetReadiness() convergeAsSuspended(rd, &target) gitTargetReadinessGates(rd, observed, r.parentBranchReadiness(&target, providerNS, repoIdentityOf(gitProvider)), - refs.gitProvider, refs.clusterProvider, refs.sourceCluster) + refs.gitProvider, refs.clusterProvider, refs.sourceCluster, publicationCondition(publication)) parentRecoveryReadiness(rd, recoveringParent) st.applyReadiness(rd) @@ -775,7 +771,7 @@ func (r *GitTargetReconciler) publishReferenceReadiness( func gitTargetReadinessGates( rd *readiness, observed dataPlaneObservation, - parent, provider, clusterProvider, sourceReach conditionValue, + parent, provider, clusterProvider, sourceReach, publication conditionValue, ) { // Terminal, most specific first. Each of these needs a human: the folder holds content the // operator will not manage, the write branch has no parent to be created from, a watch is @@ -800,6 +796,10 @@ func gitTargetReadinessGates( // An unconfirmed source has not been established at all, so Ready is Unknown rather than False. rd.progressingIf(sourceReach.Status == metav1.ConditionUnknown, metav1.ConditionUnknown, sourceReach.Reason, sourceReach.Message) + // The branch cannot publish. Ahead of this target's own data plane, because it explains it: a + // branch that paused intake keeps its streams waiting, and they say only that they wait. + rd.progressingIf(publication.Status == metav1.ConditionFalse, metav1.ConditionFalse, + publication.Reason, publication.Message) rd.progressingIf(observed.axes.Render.Status == metav1.ConditionUnknown, metav1.ConditionFalse, observed.axes.Render.Reason, observed.axes.Render.Message) rd.progressingIf(observed.axes.Streams.Status != metav1.ConditionTrue, metav1.ConditionFalse, @@ -1465,7 +1465,6 @@ func (r *GitTargetReconciler) cleanupDeletedGitTarget( // condition gauge is released on the same terms and for a sharper reason: a condition series // that outlives its object reports Ready=False forever and the alert on it never clears. r.reconcileRequests.forget(gitDest) - r.snapshotRequests.forget(gitDest) // Same terms: the publication ledger is this reconciler's memory of what the object's status // said, and an entry that outlives the object is a rate limit held against a name that may be // recreated tomorrow with nothing published. @@ -1599,6 +1598,16 @@ func (r *GitTargetReconciler) SetupWithManager(mgr ctrl.Manager) error { ). Named("gittarget") + // React to a change of the branch worker's publication report (an outage starting, intake + // pausing, a push landing), so the status says so within one reconcile instead of at the next + // steady requeue. + if r.WorkerManager != nil && r.WorkerManager.PublicationEvents() != nil { + b = b.WatchesRawSource(source.Channel( + r.WorkerManager.PublicationEvents(), + &handler.EnqueueRequestForObject{}, + )) + } + // React to a data-plane GitPath acceptance TRANSITION (refused/recovered) so GitPathAccepted // is re-projected within one reconcile instead of lagging up to RequeueSteadyInterval (5m). // The watch manager records acceptance asynchronously and pushes a GenericEvent here. See diff --git a/internal/controller/gittarget_layout_test.go b/internal/controller/gittarget_layout_test.go index bab5a610..8fd9a0e2 100644 --- a/internal/controller/gittarget_layout_test.go +++ b/internal/controller/gittarget_layout_test.go @@ -206,7 +206,8 @@ func TestGitTargetReadiness_StalledFollowsGitPathAccepted(t *testing.T) { }, Render: conditionValue{Status: metav1.ConditionTrue}, }, - }, healthyDependency(), healthyDependency(), healthyDependency(), healthyDependency()) + }, healthyDependency(), healthyDependency(), healthyDependency(), healthyDependency(), + healthyDependency()) trio := rd.trio() assert.Equal(t, wantStalled.Status, string(trio.Stalled.Status)) diff --git a/internal/controller/gittarget_parent_branch_test.go b/internal/controller/gittarget_parent_branch_test.go index 134526c1..96dea49d 100644 --- a/internal/controller/gittarget_parent_branch_test.go +++ b/internal/controller/gittarget_parent_branch_test.go @@ -159,7 +159,7 @@ func TestParentBranchReadiness(t *testing.T) { Streams: conditionValue{Status: metav1.ConditionTrue}, GitPath: conditionValue{Status: metav1.ConditionTrue}, Render: conditionValue{Status: metav1.ConditionTrue}, - }}, got, healthyDependency(), healthyDependency(), healthyDependency()) + }}, got, healthyDependency(), healthyDependency(), healthyDependency(), conditionValue{}) trio := rd.trio() assert.Equal(t, metav1.ConditionTrue, trio.Stalled.Status, "it needs somebody to create the branch") assert.Equal(t, GitTargetReasonParentBranchNotFound, trio.Ready.Reason) @@ -199,40 +199,20 @@ func TestParentBranchReadiness_AnOmittedParentCanStall(t *testing.T) { "an observation made under a configured parent is about another configuration") } -// TestParentRecovery_ASnapshotRequestForcesOneRecheck: the controller acts on the branch worker's -// snapshot requests, not on the Ready reason. A request forces exactly one recheck; requests made -// entirely between two reconciles are still acted on; an unchanged sequence forces nothing. -func TestParentRecovery_ASnapshotRequestForcesOneRecheck(t *testing.T) { - r, workers := startParentBranchWorkers(t) - repo := git.RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/repo.git"} +// TestParentRecovering_ReadsTheWorkerOnly: the reconciler reports recovery from the branch worker's +// memory and nothing else. No worker manager, no worker, or a worker that owes nothing is not +// recovering. +func TestParentRecovering_ReadsTheWorkerOnly(t *testing.T) { target := parentBranchTarget("apps", "edits", "apps", "release", time.Now()) - _, err := r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) - require.NoError(t, err) - worker, _ := workers.GetWorkerForTarget("repo1", "shop", "edits") - ref := types.NewResourceReference(target.Name, target.Namespace) - - recheck, _ := r.parentRecovery(target, "shop") - assert.False(t, recheck, "nothing asked") + assert.False(t, (&GitTargetReconciler{}).parentRecovering(target, "shop"), "no worker manager") - worker.BumpSnapshotRequestForTest(ref) - recheck, _ = r.parentRecovery(target, "shop") - assert.True(t, recheck) - recheck, _ = r.parentRecovery(target, "shop") - assert.False(t, recheck, "exactly one recheck per request") + r, _ := startParentBranchWorkers(t) + assert.False(t, r.parentRecovering(target, "shop"), "no worker for the target yet") - worker.BumpSnapshotRequestForTest(ref) - worker.BumpSnapshotRequestForTest(ref) // a whole episode between two reconciles - recheck, _ = r.parentRecovery(target, "shop") - assert.True(t, recheck, "still acted on") - - for range 10 { - recheck, _ = r.parentRecovery(target, "shop") - assert.False(t, recheck) - } - - r.snapshotRequests.forget(ref) - recheck, _ = r.parentRecovery(target, "shop") - assert.True(t, recheck, "a forgotten target acts on the standing request again") + repo := git.RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/repo.git"} + _, err := r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) + require.NoError(t, err) + assert.False(t, r.parentRecovering(target, "shop"), "a worker that owes nothing") } func TestParentRecoveryReadiness_IsProgressNotAStall(t *testing.T) { diff --git a/internal/controller/gittarget_parent_recovery.go b/internal/controller/gittarget_parent_recovery.go index 33266bf4..dddc1b4f 100644 --- a/internal/controller/gittarget_parent_recovery.go +++ b/internal/controller/gittarget_parent_recovery.go @@ -3,69 +3,32 @@ package controller import ( - "sync" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" - "github.com/ConfigButler/gitops-reverser/internal/types" ) // GitTargetReasonRecoveringParentBranch is the Ready reason while the parent branch has been found -// again but the branch worker has not yet published the work it held back. It is progress, not a -// stall, and keeps the not-Ready requeue cadence so the controller acts on the worker's snapshot -// requests within one requeue. +// again but the branch worker has not yet published the writes it kept while the parent was +// missing. It is progress, not a stall. const GitTargetReasonRecoveringParentBranch = "RecoveringParentBranch" -// snapshotRequestTracker remembers, per GitTarget, the last snapshot request of its branch worker -// that this reconciler acted on. A missing-parent episode that starts and ends between two -// reconciles still leaves the worker's sequence higher, so it is never lost; an unchanged sequence -// forces nothing. Its memory is empty after a restart, which is harmless: every stream starts with -// an initial list, which is itself the snapshot. Its zero value is usable. -type snapshotRequestTracker struct { - mu sync.Mutex - acted map[string]uint64 -} - -// take reports whether seq is a request not acted on yet, and records it as acted on. -func (t *snapshotRequestTracker) take(ref types.ResourceReference, seq uint64) bool { - t.mu.Lock() - defer t.mu.Unlock() - if seq == 0 || t.acted[ref.Key()] >= seq { - return false - } - if t.acted == nil { - t.acted = map[string]uint64{} - } - t.acted[ref.Key()] = seq - return true -} - -// forget drops a deleted object's record. -func (t *snapshotRequestTracker) forget(ref types.ResourceReference) { - t.mu.Lock() - defer t.mu.Unlock() - delete(t.acted, ref.Key()) -} - -// parentRecovery reads the branch worker's obligation for work a missing parent held back. The -// first result is whether it asks for a new snapshot of this target (true once per rise of its -// sequence), the second whether the parent is back while work is still owed. It reads memory only, -// and costs no connection. -func (r *GitTargetReconciler) parentRecovery( +// parentRecovering reports whether the branch worker has found the parent branch again and still +// owes the writes it kept while the parent was missing. It reads memory only, and costs no +// connection. +func (r *GitTargetReconciler) parentRecovering( target *configbutleraiv1alpha3.GitTarget, providerNS string, -) (bool, bool) { +) bool { if r.WorkerManager == nil { - return false, false + return false } worker, ok := r.WorkerManager.GetWorkerForTarget(target.Spec.GitProviderRef.Name, providerNS, target.Spec.Branch) if !ok { - return false, false + return false } - ref := types.NewResourceReference(target.Name, target.Namespace) open, found := worker.ParentRecovery() - return r.snapshotRequests.take(ref, worker.SnapshotRequestSeq(ref)), open && found + return open && found } // parentRecoveryReadiness is the progress gate for a target whose parent is back but whose diff --git a/internal/controller/gittarget_publication.go b/internal/controller/gittarget_publication.go new file mode 100644 index 00000000..fe0fc7bf --- /dev/null +++ b/internal/controller/gittarget_publication.go @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" +) + +// branchPublication is the branch worker's publication report: whether it can publish to the Git +// remote, and if not, since when and why. Every GitTarget on the branch reads the same report, +// because they share the worker: a sibling's success cannot clear it, and only a landed push does. +// It reads memory only, and costs no connection. +func (r *GitTargetReconciler) branchPublication( + target *configbutleraiv1alpha3.GitTarget, + providerNS string, +) git.PublicationStatus { + if r.WorkerManager == nil { + return git.PublicationStatus{} + } + worker, ok := r.WorkerManager.GetWorkerForTarget(target.Spec.GitProviderRef.Name, providerNS, target.Spec.Branch) + if !ok { + return git.PublicationStatus{} + } + return worker.Publication() +} + +// publicationCondition is the gate a publication report contributes: progress while a failed +// attempt waits for its retry, which needs nobody. A missing parent branch is left to the parent +// gate, which says which branch to create. The message changes only when the report does, so a +// retry that fails the same way again writes no status. +func publicationCondition(report git.PublicationStatus) conditionValue { + if !report.Failing || report.ParentUnavailable { + return conditionValue{Status: metav1.ConditionTrue, Reason: GitTargetReasonOK} + } + return conditionValue{Status: metav1.ConditionFalse, Reason: ReasonProgressing, Message: report.Message()} +} diff --git a/internal/controller/gittarget_publication_test.go b/internal/controller/gittarget_publication_test.go new file mode 100644 index 00000000..081b91a6 --- /dev/null +++ b/internal/controller/gittarget_publication_test.go @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/watch" +) + +var outageStart = time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC) + +func failingPublication(paused bool) git.PublicationStatus { + return git.PublicationStatus{ + Failing: true, Since: outageStart, Cause: "dial tcp: connection refused", IntakePaused: paused, + } +} + +// convergedObservation is a data plane with nothing pending. +func convergedObservation() dataPlaneObservation { + return dataPlaneObservation{axes: gitTargetAxes{ + Streams: conditionValue{Status: metav1.ConditionTrue}, + GitPath: conditionValue{Status: metav1.ConditionTrue}, + Render: conditionValue{Status: metav1.ConditionTrue}, + }} +} + +func publicationGates(observed dataPlaneObservation, publication git.PublicationStatus) kstatusTrio { + rd := newGitTargetReadiness() + gitTargetReadinessGates(rd, observed, healthyDependency(), healthyDependency(), healthyDependency(), + healthyDependency(), publicationCondition(publication)) + return rd.trio() +} + +// A branch that cannot publish is not Ready, and it is progress, not a stall: the retry is +// scheduled and needs nobody. The message says since when, why, and whether intake is paused. +func TestPublicationReadiness_AnOutageIsProgressNotAStall(t *testing.T) { + trio := publicationGates(convergedObservation(), failingPublication(true)) + assert.Equal(t, metav1.ConditionFalse, trio.Ready.Status) + assert.Equal(t, ReasonProgressing, trio.Ready.Reason) + assert.Contains(t, trio.Ready.Message, "dial tcp: connection refused") + assert.Contains(t, trio.Ready.Message, "2026-10-04T12:00:00Z") + assert.Contains(t, trio.Ready.Message, "intake of new changes is paused") + assert.Equal(t, metav1.ConditionTrue, trio.Reconciling.Status) + assert.Equal(t, metav1.ConditionFalse, trio.Stalled.Status) + + healthy := publicationGates(convergedObservation(), git.PublicationStatus{}) + assert.Equal(t, metav1.ConditionTrue, healthy.Ready.Status, "a branch publishing normally adds nothing") +} + +// The publication failure explains the streams a paused branch keeps waiting, so it is reported +// ahead of them. A refused Git path is a different problem that needs a person, and a publication +// that is merely retrying must not hide it. +func TestPublicationReadiness_TakesPrecedenceOverWaitingStreamsNotOverRefusals(t *testing.T) { + waiting := convergedObservation() + waiting.streams = watch.StreamSummary{Total: 1, Replaying: 1} + waiting.axes.Streams = conditionValue{Status: metav1.ConditionFalse, Reason: watch.StreamReasonReplaying, + Message: "0/1 streams running; replaying: configmaps"} + trio := publicationGates(waiting, failingPublication(true)) + assert.Contains(t, trio.Ready.Message, "Cannot publish to the Git remote") + + refused := convergedObservation() + refused.axes.GitPath = conditionValue{Status: metav1.ConditionFalse, Reason: GitTargetReasonUnsupportedContent, + Message: "the folder holds a kustomization the operator will not edit"} + trio = publicationGates(refused, failingPublication(false)) + assert.Equal(t, metav1.ConditionTrue, trio.Stalled.Status, "the refusal still needs somebody") + assert.Equal(t, GitTargetReasonUnsupportedContent, trio.Ready.Reason) +} + +// A missing parent branch is reported by the parent gate, which says which branch to create. The +// publication report of the same outage would only repeat it, less precisely. +func TestPublicationReadiness_LeavesAMissingParentToTheParentGate(t *testing.T) { + missing := failingPublication(false) + missing.Cause = "parent branch not found on the remote: \"release\"" + missing.ParentUnavailable = true + trio := publicationGates(convergedObservation(), missing) + assert.Equal(t, metav1.ConditionTrue, trio.Ready.Status) +} + +// A save the worker holds says why it is held, with the same words the GitTarget uses, and the +// status is written again when that reason changes, not only when the phase does. +func TestCommitRequestProgress_AHeldSaveSaysWhyItIsHeld(t *testing.T) { + held := failingPublication(false).Message() + reason, message := progressFor(git.PhaseWaitingForPush, held) + assert.Equal(t, string(git.PhaseWaitingForPush), reason) + assert.Contains(t, message, waitingForPushMessage) + assert.Contains(t, message, "dial tcp: connection refused") + + _, plain := progressFor(git.PhaseWaitingForPush, "") + assert.Equal(t, waitingForPushMessage, plain, "a push that is merely pending says nothing more") + + cr := &configbutleraiv1alpha3.CommitRequest{} + markCommitRequestProgressing(cr, attributionFromAdmission, git.PhaseWaitingForPush, "") + assert.True(t, progressRecorded(cr, git.PhaseWaitingForPush, "")) + assert.False(t, progressRecorded(cr, git.PhaseWaitingForPush, held), "the outage is news to the save") + markCommitRequestProgressing(cr, attributionFromAdmission, git.PhaseWaitingForPush, held) + assert.True(t, progressRecorded(cr, git.PhaseWaitingForPush, held)) + assert.False(t, progressRecorded(cr, git.PhaseWaitingForPush, ""), "and so is the recovery") +} diff --git a/internal/controller/gittarget_status_test.go b/internal/controller/gittarget_status_test.go index e5a1d656..e32e40cd 100644 --- a/internal/controller/gittarget_status_test.go +++ b/internal/controller/gittarget_status_test.go @@ -164,7 +164,8 @@ func TestGitTargetReadinessGates(t *testing.T) { st.setValue(GitTargetConditionRenderMatchesLive, observed.axes.Render) rd := newGitTargetReadiness() - gitTargetReadinessGates(rd, observed, conditionValue{}, tt.provider, tt.clusterProvider, tt.sourceReach) + gitTargetReadinessGates(rd, observed, conditionValue{}, tt.provider, tt.clusterProvider, tt.sourceReach, + conditionValue{}) st.applyReadiness(rd) ready := findCondition(target.Status.Conditions, GitTargetConditionReady) diff --git a/internal/controller/object_selector_rule_test.go b/internal/controller/object_selector_rule_test.go index e1a4f7b3..b043fa9c 100644 --- a/internal/controller/object_selector_rule_test.go +++ b/internal/controller/object_selector_rule_test.go @@ -66,15 +66,15 @@ func TestReconcile_WatchRuleWithAnInvalidObjectSelectorIsRefused(t *testing.T) { func TestReconcile_WatchRuleWithAConflictingObjectSelectorIsRefused(t *testing.T) { ctx := context.Background() f := newWRSNFixture(t, wrsnBaseObjects(false, "")) - f.wm.selectorConflict = "WatchRule tenant-acme/repo-config-rule is refused: it selects secrets" + f.wm.collectionOverlap = "WatchRule tenant-acme/repo-config-rule is refused: it selects secrets" _, err := f.reconcile(ctx) require.NoError(t, err) conditions := f.reloadRule(ctx, t).Status.Conditions - assertResourcesRefused(t, conditions, watch.ReasonObjectSelectorConflict) + assertResourcesRefused(t, conditions, watch.ReasonCollectionOverlap) resolved := apimeta.FindStatusCondition(conditions, ConditionTypeResourcesResolved) - assert.Equal(t, f.wm.selectorConflict, resolved.Message) + assert.Equal(t, f.wm.collectionOverlap, resolved.Message) } func TestReconcile_ClusterWatchRuleWithAnInvalidObjectSelectorIsRefused(t *testing.T) { @@ -100,10 +100,10 @@ func TestReconcile_ClusterWatchRuleWithAConflictingObjectSelectorIsRefused(t *te cwaClusterWatchRule(), cwaGitTarget(), cwaGitProvider(), cwaClusterProvider(cwaAdmitting()), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: cwaTargetNS}}, }, interceptor.Funcs{}) - f.wm.selectorConflict = "ClusterWatchRule mirror-everything is refused" + f.wm.collectionOverlap = "ClusterWatchRule mirror-everything is refused" _, err := f.reconcile(ctx) require.NoError(t, err) - assertResourcesRefused(t, f.reloadRule(ctx, t).Status.Conditions, watch.ReasonObjectSelectorConflict) + assertResourcesRefused(t, f.reloadRule(ctx, t).Status.Conditions, watch.ReasonCollectionOverlap) } diff --git a/internal/controller/stream_status.go b/internal/controller/stream_status.go index e903bddb..310e18a6 100644 --- a/internal/controller/stream_status.go +++ b/internal/controller/stream_status.go @@ -146,10 +146,10 @@ func gitTargetReadyReasonIsStalled(reason string) bool { // registered in SetupWithManager, so the rule is woken by an event long before the steady tick; // polling it faster would find nothing. // -// ObjectSelectorConflict is the exception. It clears when a SIBLING rule on the same GitTarget is +// CollectionOverlap is the exception. It clears when a SIBLING rule on the same GitTarget is // deleted or re-selects, and no watch edge wakes this rule on that: the only signal is the // best-effort stream-state event, which a full subscriber buffer drops. So that stall retries on -// RequeueSelectorConflictInterval rather than keeping a resolved conflict for the steady interval. +// RequeueCollectionOverlapInterval rather than keeping a resolved conflict for the steady interval. // // It is one function for both rule kinds. It was two identical methods, neither of which used its // receiver, and the pairing of a cadence with a verdict is exactly the decision that must not be @@ -164,14 +164,14 @@ func commitRule(ctx context.Context, st *reconcileStatus, rd *readiness) (ctrl.R switch { case rd.converging(): cadence = RequeueStreamSettleInterval - case selectorConflicted(*st.conditions): - cadence = RequeueSelectorConflictInterval + case collectionOverlapRefused(*st.conditions): + cadence = RequeueCollectionOverlapInterval } return ctrl.Result{RequeueAfter: st.requeueAfter(cadence)}, nil } -// selectorConflicted reports whether conditions refuse the rule for an objectSelector conflict. -func selectorConflicted(conditions []metav1.Condition) bool { +// collectionOverlapRefused reports whether conditions refuse the rule for a collection overlap. +func collectionOverlapRefused(conditions []metav1.Condition) bool { cond := apimeta.FindStatusCondition(conditions, ConditionTypeResourcesResolved) - return cond != nil && cond.Status == metav1.ConditionFalse && cond.Reason == watch.ReasonObjectSelectorConflict + return cond != nil && cond.Status == metav1.ConditionFalse && cond.Reason == watch.ReasonCollectionOverlap } diff --git a/internal/controller/stream_status_test.go b/internal/controller/stream_status_test.go index 75b445cc..ad181e31 100644 --- a/internal/controller/stream_status_test.go +++ b/internal/controller/stream_status_test.go @@ -140,13 +140,13 @@ func TestCommitRule_SelectorConflictRetriesOnABoundedCadence(t *testing.T) { c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(rule).WithStatusSubresource(rule).Build() st := beginStatus(c, nil, rule) - st.set(ConditionTypeResourcesResolved, metav1.ConditionFalse, watch.ReasonObjectSelectorConflict, "conflict") + st.set(ConditionTypeResourcesResolved, metav1.ConditionFalse, watch.ReasonCollectionOverlap, "conflict") rd := newRuleReadiness("rule", "") - rd.stalled(watch.ReasonObjectSelectorConflict, "conflict") + rd.stalled(watch.ReasonCollectionOverlap, "conflict") result, err := commitRule(context.Background(), st, rd) require.NoError(t, err) - assert.Equal(t, RequeueSelectorConflictInterval, result.RequeueAfter) + assert.Equal(t, RequeueCollectionOverlapInterval, result.RequeueAfter) stalled := apimeta.FindStatusCondition(rule.Status.Conditions, ConditionTypeStalled) require.NotNil(t, stalled) assert.Equal(t, metav1.ConditionTrue, stalled.Status) diff --git a/internal/controller/watchrule_controller.go b/internal/controller/watchrule_controller.go index 8d4578ba..c10db196 100644 --- a/internal/controller/watchrule_controller.go +++ b/internal/controller/watchrule_controller.go @@ -259,8 +259,8 @@ func (r *WatchRuleReconciler) setResourceResolutionCondition( st *reconcileStatus, watchRule *configbutleraiv1alpha3.WatchRule, ) { - if refused, message := r.WatchManager.ObjectSelectorConflictForWatchRule(*watchRule); refused { - st.set(ConditionTypeResourcesResolved, metav1.ConditionFalse, watch.ReasonObjectSelectorConflict, message) + if refused, message := r.WatchManager.CollectionOverlapForWatchRule(*watchRule); refused { + st.set(ConditionTypeResourcesResolved, metav1.ConditionFalse, watch.ReasonCollectionOverlap, message) return } resolved, message := r.WatchManager.ResolveWatchRuleResources(ctx, *watchRule) diff --git a/internal/controller/watchrule_controller_test.go b/internal/controller/watchrule_controller_test.go index 2887c36d..7e1a3902 100644 --- a/internal/controller/watchrule_controller_test.go +++ b/internal/controller/watchrule_controller_test.go @@ -258,6 +258,14 @@ var _ = Describe("WatchRule Controller", func() { g.Expect(ready).NotTo(BeNil()) g.Expect(ready.Status).To(Equal(metav1.ConditionFalse)) g.Expect(ready.Reason).To(Equal(ReasonProgressing)) + // Ready alone does not prove the write saw the settled GitTarget: with no streams + // running in envtest it is False/Progressing either way. GitTargetReady is the + // condition that mirrors the GitTarget, and the reconciler reads it through a cache + // that can still hold the GitTarget without conditions (Unknown), which flaked in CI. + mirrored := apimeta.FindStatusCondition(updatedRule.Status.Conditions, ConditionTypeGitTargetReady) + g.Expect(mirrored).NotTo(BeNil()) + g.Expect(mirrored.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(mirrored.Reason).To(Equal(ReasonProgressing)) }, "10s", "200ms").Should(Succeed()) By("Verifying WatchRule is reconciling until the GitTarget and streams are ready") diff --git a/internal/git/ado_multiack_test.go b/internal/git/ado_multiack_test.go index 56f13305..b2ef60f0 100644 --- a/internal/git/ado_multiack_test.go +++ b/internal/git/ado_multiack_test.go @@ -59,6 +59,10 @@ type adoSimulator struct { // round trips to the Git host can be measured rather than guessed. See // git_request_ledger_test.go and docs/design/push-notification-and-reconcile-trigger.md §1.6. ledger *gitRequestLedger + + // around, when set, serves each request in place of the backend and decides whether and how + // to call it: a test holds a request open, or lets the backend apply it and swallows the reply. + around atomic.Pointer[func(w http.ResponseWriter, r *http.Request, backend http.Handler)] } // gitHTTPBackend locates canonical git's CGI server, skipping the test when it is unavailable. @@ -157,6 +161,10 @@ func startGitHTTPServer(tb testing.TB, projectRoot, repoDir string, enforceMulti r.ContentLength = int64(len(raw)) } + if around := sim.around.Load(); around != nil { + (*around)(w, r, backend) + return + } backend.ServeHTTP(w, r) }) diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index 882c4619..2b2791dd 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -177,18 +177,33 @@ type BranchWorker struct { // repoMu serializes repository/worktree operations within this worker. repoMu sync.Mutex - // baseTrustedState and worktreeDirtyState carry the invariant "the worktree sits at the - // remote tip of the target branch, or the worker knows it does not", plus the separate - // question of whether the worktree is clean. They are read through baseTrusted() and - // worktreeDirty() and written through setBaseTrusted, invalidateBase and markWorktreeDirty. + // baseTrustedState carries the invariant "the checkout's root sits at the remote tip of the + // target branch, or the worker knows it does not". It is read through baseTrusted() and written + // through setBaseTrusted and invalidateBase. // - // They are atomics rather than repoMu-protected fields because the loss points are spread + // It is an atomic rather than a repoMu-protected field because the loss points are spread // across every error return on the write path, and a lock there would put a second ordering // constraint on code whose only job is to record that something went wrong. // // See the package's design note in docs/design/push-notification-and-reconcile-trigger.md §1.5. - baseTrustedState atomic.Bool - worktreeDirtyState atomic.Bool + baseTrustedState atomic.Bool + + // checkoutApplied is how many of the loop's retained writes the checkout holds: the checkout is + // the projection of the pending writes, its root plus one commit per write in + // pendingWrites[:applied]. + // checkoutUnknown means a write failed part-way and the worktree may hold anything. + // + // It is the one answer to "may the loop commit on, or push, what the checkout holds?", which + // is why it is a count and not a flag. A reset to the remote tip sets it to zero, which with + // writes retained says their commits are gone and must be replayed: pushing then would find the + // branch already at the remote tip, report success without sending anything, and settle work + // that exists nowhere. A commit raises it, a replay sets it to every pending write, and a push + // that publishes them returns it to zero. A failed write sets checkoutUnknown, and only a reset + // clears that: a successful push says where the remote is, and nothing about leftovers in the + // worktree. + // + // See docs/design/gittarget-branch-worker-pending-writes.md. + checkoutApplied atomic.Int64 // lastRefusalTouch records when this worker last pushed an empty commit for a GitTarget, keyed // by "namespace/name", so refusalTouchInterval can floor the rate. It is guarded by its own @@ -205,23 +220,6 @@ type BranchWorker struct { // Guarded by refusalTouchMu with the rate limit, because the two are read together. coveredRefusal map[refusalKey]string - // replayRequiredState records that a reset has discarded the local commits behind the retained - // writes while the replay that rebuilds them did not finish. - // - // It is a THIRD flag because it is a third question, and the other two answer it wrongly. - // baseTrusted is true — the worktree is exactly at the remote tip, which is the problem. - // worktreeDirty is false — the reset cleared it, and the worktree really is clean. What is - // stale is the retained writes: they still carry the commit hashes they had before the reset, - // and those commits are gone. - // - // Pushing in that state fails silently rather than loudly. The local branch equals the remote - // tip, so validatePushState answers "already up to date" and returns before it compares the - // cycle's root hash; the worker then counts the writes as published and resolves any - // CommitRequest riding one as Committed, naming a SHA that is not on the remote. - // - // See docs/design/push-notification-and-reconcile-trigger.md §1.5. - replayRequiredState atomic.Bool - // branchBufferMaxBytes caps the retained in-memory event data; tripped on // event arrival, an immediate finalize bypasses the commit window. branchBufferMaxBytes int64 @@ -292,8 +290,8 @@ type BranchWorker struct { // pendingResyncs coalesces queued resyncs by (GitTarget, scope). A resync is // state-based and idempotent — "make Git match this desired set for this // scope" — so a newer one wholly supersedes an older one still waiting. The - // map holds the current request per key; the FIFO holds one marker per key, - // which is what preserves ordering against live events. Queue depth is + // map holds the open marker per key, and the marker the current request; the + // FIFO holds the marker, which is what preserves ordering against live events. Queue depth is // therefore bounded by the number of distinct scopes, not by the request // rate, so a storm of resyncs for one target can no longer fill the queue // and starve every other GitTarget on this branch. @@ -303,7 +301,7 @@ type BranchWorker struct { // before writes it already contains, which would then overwrite it. The entry's tail flag // records that boundary. pendingResyncsMu sync.Mutex - pendingResyncs map[resyncKey]*pendingResync + pendingResyncs map[resyncKey]*resyncMarker // crOutcomes holds resolved CommitRequest outcomes for the controller to poll // via LookupCommitRequestOutcome. The event loop is the only writer (on its @@ -324,17 +322,22 @@ type BranchWorker struct { // Set by the WorkerManager before Start; nil for a worker built on its own. crOwners *commitRequestOwners - // clock is the time source of the parent-recovery schedule; nil is time.Now. See now(). + // clock is the time source of the retry schedule; nil is time.Now. See now(). clock func() time.Time // parentRecoveryOpen, parentRecoveryFound and parentProbeHold mirror the event loop's // recovery latch for readers outside it. See parent_recovery.go. parentRecoveryOpen atomic.Bool parentRecoveryFound atomic.Bool parentProbeHold atomic.Pointer[parentProbeHold] - // snapshotRequests counts, per GitTarget, the snapshots the worker has asked for. See - // SnapshotRequestSeq. - snapshotRequestsMu sync.Mutex - snapshotRequests map[itypes.ResourceReference]uint64 + // intake decides whether new writes, saves and resyncs may enter the FIFO. See intake.go. + intake intakeGate + // publicationReporter is told when the worker's publication report changes. Set by the + // WorkerManager before Start; nil in tests that do not assert on it. See publication.go. + publicationReporter func() + // publication is the loop's latest publication report, and backlog what it keeps, for readers + // outside the loop. See publication.go. + publication atomic.Pointer[PublicationStatus] + backlog publicationBacklog } // branchWorkerLogFirsts logs the first successful commit and push of a worker's @@ -398,7 +401,7 @@ func NewBranchWorker( ), contentWriter: writer, eventQueue: make(chan WorkItem, limits.QueueDepth), - pendingResyncs: make(map[resyncKey]*pendingResync), + pendingResyncs: make(map[resyncKey]*resyncMarker), branchBufferMaxBytes: limits.MaxBufferBytes, } } @@ -561,11 +564,6 @@ func (w *BranchWorker) Enqueue(event Event) bool { }) } -// EnqueueRequest adds a write request to this worker's queue. -func (w *BranchWorker) EnqueueRequest(request *WriteRequest) { - w.enqueueRequest(request) -} - // EnqueueWithdraw asks the loop to cancel a CommitRequest it has not acted on. It never blocks: a // full queue or a stopping worker drops it, and the controller sends it again on its next poll. A // worker whose loop has exited answers at once, because nothing is left that could act on it. @@ -604,6 +602,7 @@ func (w *BranchWorker) EnqueueAttach(req *AttachCommitRequest) { if req == nil { return } + charge := attachCharge(req.Message) // Increment before the send so inflightItems can never lag the loop's // receive; roll back if the queue is full and the item is dropped. @@ -616,14 +615,23 @@ func (w *BranchWorker) EnqueueAttach(req *AttachCommitRequest) { if w.stoppingLocked() { w.pendingResyncsMu.Unlock() w.inflightItems.Add(-1) - w.recordQueueDrop(queueDropAttach) + w.recordQueueRefusal(queueRefusalAttach) w.Log.V(1).Info("Worker is stopping, CommitRequest attach refused (the controller re-sends)") return } + if !w.admitPayload(charge) { + w.pendingResyncsMu.Unlock() + w.inflightItems.Add(-1) + w.recordQueueRefusal(queueRefusalAttach) + w.Log.V(1).Info("CommitRequest attach refused while the branch has paused intake "+ + "(the controller re-sends)", "request", req.Namespace+"/"+req.Name) + return + } id := req.id() if !w.crOwners.claim(id, w) { // The router routes to the owner, so only a request that moved workers between two polls // gets here. The owner answers for it; registering it twice could commit it twice. + w.intake.release(charge) w.pendingResyncsMu.Unlock() w.inflightItems.Add(-1) w.Log.Info("CommitRequest attach refused: another branch worker holds the request", @@ -633,7 +641,7 @@ func (w *BranchWorker) EnqueueAttach(req *AttachCommitRequest) { w.markResyncTailForTargetLocked(req.GitTargetNamespace, req.GitTargetName) w.countQueuedAttach(id) select { - case w.eventQueue <- WorkItem{Attach: req}: + case w.eventQueue <- WorkItem{Attach: req, charge: charge}: w.pendingResyncsMu.Unlock() w.Log.Info("CommitRequest attach enqueued", "request", req.Namespace+"/"+req.Name, @@ -646,32 +654,34 @@ func (w *BranchWorker) EnqueueAttach(req *AttachCommitRequest) { // loop has woken to notice it. default: w.uncountQueuedAttach(id) + w.intake.release(charge) w.pendingResyncsMu.Unlock() w.inflightItems.Add(-1) - w.recordQueueDrop(queueDropAttach) - w.Log.Error(nil, "Event queue full, CommitRequest attach dropped (controller will re-send)") + w.recordQueueRefusal(queueRefusalAttach) + w.Log.Error(nil, "Event queue full, CommitRequest attach refused (controller will re-send)") } } // EnqueueResync adds a resync request to this worker's queue. Like a finalize // signal it rides the same queue as resource events, so it is applied in order with // live events: a resync enqueued during the snapshot window lands before the buffered -// live events that follow it. If the queue is full the request is dropped and its -// caller is notified immediately via the result channel. +// live events that follow it. If the queue is full the request is refused and its +// caller is told immediately via the result channel, so it can gather the scope again. // // It reports whether the request actually entered the FIFO. A caller gating downstream state on -// the resync's ordering must not treat a drop as success: it would mark the target +// the resync's ordering must not treat a refusal as success: it would mark the target // reconciled-through-Hc with no reconcile ever queued. func (w *BranchWorker) EnqueueResync(request *ResyncRequest) bool { if request == nil { return false } key := resyncKeyFor(request) + charge := request.charge() // sized here, off the loop and outside the lock w.pendingResyncsMu.Lock() if w.stoppingLocked() { w.pendingResyncsMu.Unlock() - w.recordQueueDrop(queueDropResync) + w.recordQueueRefusal(queueRefusalResync) w.Log.V(1).Info("Worker is stopping, resync request refused", "gitTarget", request.GitTargetNamespace+"/"+request.GitTargetName) // Answered rather than dropped in silence: the caller is waiting on this channel, and the @@ -680,17 +690,20 @@ func (w *BranchWorker) EnqueueResync(request *ResyncRequest) bool { return false } if w.pendingResyncs == nil { - w.pendingResyncs = make(map[resyncKey]*pendingResync) + w.pendingResyncs = make(map[resyncKey]*resyncMarker) } w.markResyncTailForResyncLocked(request, key) if pending, queued := w.pendingResyncs[key]; queued && !pending.tailPassed { // A marker for this key is already in the FIFO and nothing for this scope // was queued behind it, so the marker's position is still the right place // for a newer snapshot. Swap in the newer request; the loop reads whatever - // is current when the marker comes up. + // is current when the marker comes up. Only the difference in size is new. + if !w.admitPayload(charge - pending.request.charge()) { + w.pendingResyncsMu.Unlock() + return w.refuseResyncAtIntake(request, charge) + } superseded := pending.request - request.RefreshRemote = request.RefreshRemote || superseded.RefreshRemote - pending.request = request + pending.request, pending.charge = request, charge w.pendingResyncsMu.Unlock() // The superseded request's caller is waiting on its reply channel. Answer // it so its drain ends, with a sentinel that says "a newer resync for the @@ -704,9 +717,8 @@ func (w *BranchWorker) EnqueueResync(request *ResyncRequest) bool { } else if queued { // A write inside this scope was queued behind the marker. Coalescing here // would run this snapshot ahead of those writes and let them overwrite it - // with older state. Release the key instead: the queued marker finds no - // entry and runs the payload it carries, at its own position, and this - // request takes a fresh marker at the tail below. + // with older state. Release the key instead: the queued marker keeps the + // request it holds, and this request takes a fresh marker at the tail below. delete(w.pendingResyncs, key) w.Log.V(1).Info("Resync request not coalesced: writes are queued behind the pending marker", "scope", request.Scope.String(), @@ -719,10 +731,15 @@ func (w *BranchWorker) EnqueueResync(request *ResyncRequest) bool { // while its request never ran and its drain waited for a reply that never // came. The send is non-blocking, so holding the lock across it cannot // deadlock against the loop's takePendingResync. - w.pendingResyncs[key] = &pendingResync{marker: request, request: request} + if !w.admitPayload(charge) { + w.pendingResyncsMu.Unlock() + return w.refuseResyncAtIntake(request, charge) + } + marker := &resyncMarker{request: request, charge: charge} + w.pendingResyncs[key] = marker w.inflightItems.Add(1) select { - case w.eventQueue <- WorkItem{Resync: request}: + case w.eventQueue <- WorkItem{Resync: marker}: w.pendingResyncsMu.Unlock() w.Log.V(1).Info("Resync request enqueued", "resources", len(request.Desired), @@ -732,9 +749,10 @@ func (w *BranchWorker) EnqueueResync(request *ResyncRequest) bool { default: w.inflightItems.Add(-1) delete(w.pendingResyncs, key) + w.intake.release(charge) w.pendingResyncsMu.Unlock() - w.recordQueueDrop(queueDropResync) - w.Log.Error(nil, "Event queue full, resync request dropped", + w.recordQueueRefusal(queueRefusalResync) + w.Log.Error(nil, "Event queue full, resync request refused; its caller gathers it again", "gitTarget", request.GitTargetNamespace+"/"+request.GitTargetName, "sourceCollection", sourceCollectionForLog(request.SourceCollection)) request.reply(ResyncResult{Err: ErrFinalizeQueueFull}) @@ -742,6 +760,17 @@ func (w *BranchWorker) EnqueueResync(request *ResyncRequest) bool { } } +// refuseResyncAtIntake answers a resync the intake gate refused. It reports false: the request did +// not enter the FIFO. +func (w *BranchWorker) refuseResyncAtIntake(request *ResyncRequest, charge int64) bool { + w.recordQueueRefusal(queueRefusalResync) + err := w.intakeRefusal(charge) + w.Log.V(1).Info("Resync refused while the branch has paused intake", + "gitTarget", request.GitTargetNamespace+"/"+request.GitTargetName, "reason", err.Error()) + request.reply(ResyncResult{Err: err}) + return false +} + // markResyncTailForWriteLocked records, on every pending resync whose GitTarget and // scope contain one of this write's events, that work for that scope is now queued // behind its marker. A pending resync with a nil scope covers the whole GitTarget, so @@ -813,35 +842,28 @@ func (w *BranchWorker) markResyncTailForTargetLocked(namespace, name string) { } } -// takePendingResync returns the current request for a marker's key, which may be -// newer than the one the marker carried, and clears the key so the next enqueue -// queues a fresh marker. -func (w *BranchWorker) takePendingResync(marker *ResyncRequest) *ResyncRequest { - key := resyncKeyFor(marker) +// takePendingResync returns the request a marker runs, which may be newer than the one it was queued +// with, and the charge it holds, and closes its key if it is still open, so the next enqueue queues a +// fresh marker. A key already released belongs to a later marker, which is left alone. +func (w *BranchWorker) takePendingResync(marker *resyncMarker) (*ResyncRequest, int64) { w.pendingResyncsMu.Lock() defer w.pendingResyncsMu.Unlock() - current, ok := w.pendingResyncs[key] - if !ok || current.marker != marker { - // No entry for this marker: it was queued before coalescing tracked it, the - // key was already taken, or coalescing released it because writes were - // queued behind this marker and a later request has since claimed the key. - // Run what the marker carried, at this position. Any entry present belongs - // to a marker still on the FIFO, so it is left alone. - return marker + key := resyncKeyFor(marker.request) + if w.pendingResyncs[key] == marker { + delete(w.pendingResyncs, key) } - delete(w.pendingResyncs, key) - return current.request + return marker.request, marker.charge } // enqueueRequest places a write request on the FIFO and reports whether it was -// accepted. A false return means the queue was full and the item was dropped, so a -// caller that gates durable state on the write (a watch cursor) must not treat the -// drop as success. +// accepted. A false return means the item was refused (a full queue, paused intake, or a +// stopping worker), so a caller that gates durable state on the write (a watch cursor) must not +// treat the refusal as success, and offers the write again. func (w *BranchWorker) enqueueRequest(request *WriteRequest) bool { if request == nil { return false } - item := WorkItem{Request: request} + item := WorkItem{Request: request, charge: eventCharge(request)} // sized off the loop // Increment before the send so inflightItems can never lag the loop's // receive; roll back if the queue is full and the item is dropped. w.inflightItems.Add(1) @@ -857,12 +879,20 @@ func (w *BranchWorker) enqueueRequest(request *WriteRequest) bool { if w.stoppingLocked() { w.pendingResyncsMu.Unlock() w.inflightItems.Add(-1) - w.recordQueueDrop(queueDropWrite) + w.recordQueueRefusal(queueRefusalWrite) w.Log.V(1).Info("Worker is stopping, request refused; the producer keeps it to redeliver", "events", len(request.Events), "gitTarget", request.GitTargetName) return false } + if !w.admitPayload(item.charge) { + w.pendingResyncsMu.Unlock() + w.inflightItems.Add(-1) + w.recordQueueRefusal(queueRefusalWrite) + w.Log.V(1).Info("Write refused while the branch has paused intake; "+ + "the producer keeps it to redeliver", "events", len(request.Events), "gitTarget", request.GitTargetName) + return false + } w.markResyncTailForWriteLocked(request) select { case w.eventQueue <- item: @@ -877,13 +907,14 @@ func (w *BranchWorker) enqueueRequest(request *WriteRequest) bool { // loop has woken to notice it. return true default: + w.intake.release(item.charge) w.pendingResyncsMu.Unlock() w.inflightItems.Add(-1) - w.recordQueueDrop(queueDropWrite) - // Name the producing collection on a drop. A saturated queue is diagnosed from what was - // dropped and by whom: the 595-in-16-seconds storm was one GitTarget, and the next + w.recordQueueRefusal(queueRefusalWrite) + // Name the producing collection on a refusal. A saturated queue is diagnosed from what was + // refused and by whom: the 595-in-16-seconds storm was one GitTarget, and the next // one may be one COLLECTION of one GitTarget. - w.Log.Error(nil, "Event queue full, request dropped", + w.Log.Error(nil, "Event queue full, request refused; its producer offers it again", "events", len(request.Events), "mode", request.CommitMode, "gitTarget", request.GitTargetName, @@ -1085,12 +1116,11 @@ type branchWorkerEventLoop struct { lastPushAt time.Time commitTimer *time.Timer - // pushTimer fires the next push: when the success cooldown ends, or when a failed - // publication's retry is due (publicationRetry). + // pushTimer fires the next push when the success cooldown ends. pushTimer *time.Timer - // publicationRetry is the schedule of a failed publication's next attempt. See - // publication_retry.go. - publicationRetry publicationRetry + // retry is the one schedule of the next attempt at work a failed attempt left owed. See + // retry.go. + retry retrySchedule // deferredHeals holds heal resyncs (periodic re-anchors, removed-type sweeps) parked while a // commit window is open, so a heal never force-finalizes (steals) that window — including a @@ -1146,17 +1176,19 @@ func newBranchWorkerEventLoop(w *BranchWorker, defaultIdleTimeout time.Duration) func (l *branchWorkerEventLoop) run() { defer l.stopTimers() - l.syncUnpushedWorkFlag() + l.publishLoopState(0) for { commitC, pushC, attachC, refusalC := l.timerChannels() select { case <-l.w.ctx.Done(): l.handleShutdown() - l.syncUnpushedWorkFlag() + l.publishLoopState(0) return case item := <-l.w.eventQueue: l.handleQueueItem(item) - l.releaseHandledItem() + // Released after the wake has published what the item left behind: see queueDepth. + l.w.inflightItems.Add(-1) + continue case <-commitC: l.commitTimer = nil l.closeOrArmWindow() @@ -1170,38 +1202,48 @@ func (l *branchWorkerEventLoop) run() { case <-refusalC: l.refusalTimer = nil l.flushPendingRefusalTouch() - case <-l.recoveryTimerC(): - l.recovery.timer = nil - l.runParentProbe() + case <-l.retryTimerC(): + l.retry.timer = nil + l.runRetry() } - // After every wake: bind any waiting CommitRequest to an open window, - // resolve any whose attach deadline has passed, and re-arm the deadline timer. - l.serviceCommitRequests() - // Drain any heal resync parked while a window was open, now that this wake may have - // finalized it (a silence timeout, a CommitRequest finalize). A no-op while a window - // is still open or nothing is parked. - l.applyDeferredHeals() - l.syncUnpushedWorkFlag() + l.endWake(0) } } -// syncUnpushedWorkFlag refreshes the worker's retained-work flag from the loop's authoritative -// state. Called once per loop iteration; the loop goroutine is the only writer of hasUnpushedWork. -// -// The flag is all the loop owes the depth gauge: queueDepth() reads it and inflightItems at scrape -// time, so nothing here publishes a value that could go stale while the loop is busy. -// releaseHandledItem releases one handled item from the in-flight count. +// handleQueueItem handles one item from the FIFO as one wake of the loop. +func (l *branchWorkerEventLoop) handleQueueItem(item WorkItem) { + l.endWake(l.dispatch(item)) +} + +// endWake is what every wake of the loop ends with, whatever woke it. released is the intake charge +// of the item the wake handled, or zero. // -// The order is the contract: queueDepth() reads the in-flight count and the retained-work flag -// together, so publishing the flag second lets both read empty while a window is open, and a scrape -// landing there reports a drained queue for a worker that has work. -func (l *branchWorkerEventLoop) releaseHandledItem() { - l.syncUnpushedWorkFlag() - l.w.inflightItems.Add(-1) +// - Bind any waiting CommitRequest to an open window, resolve any whose attach deadline has +// passed, and re-arm the deadline timer. +// - Drain any heal resync parked while a window was open, now that this wake may have finalized +// it (a silence timeout, a CommitRequest finalize). A no-op while a window is still open or +// nothing is parked. +// - Drive the pending writes: commit what the wake decided, schedule its push (advance). +// - Publish the result, moving the handled item's charge to what the loop holds. +func (l *branchWorkerEventLoop) endWake(released int64) { + l.serviceCommitRequests() + l.applyDeferredHeals() + l.advance() + l.publishLoopState(released) } -func (l *branchWorkerEventLoop) syncUnpushedWorkFlag() { +// publishLoopState refreshes the worker's retained-work flag and the intake gate from the loop's +// authoritative state; the loop goroutine is the only writer of both. released is the charge of the +// item the wake handled: see intakeGate.sync. +// +// The flag is all the loop owes the depth gauge: queueDepth() reads it and inflightItems at scrape +// time, so nothing here publishes a value that could go stale while the loop is busy. The order is +// the contract: the loop releases a handled item from the in-flight count only after this, because +// publishing the flag second would let both read empty while a window is open, and a scrape landing +// there would report a drained queue for a worker that has work. +func (l *branchWorkerEventLoop) publishLoopState(released int64) { l.w.hasUnpushedWork.Store(l.openWindow != nil || len(l.pendingWrites) > 0) + l.syncAdmission(released) } func (l *branchWorkerEventLoop) timerChannels() ( @@ -1230,46 +1272,35 @@ func (l *branchWorkerEventLoop) totalRetainedBytes() int64 { return l.windowBytes + l.pendingWritesBytes } -func (l *branchWorkerEventLoop) handleQueueItem(item WorkItem) { - if item.Attach != nil { +// dispatch hands one item to its handler, and returns the intake charge the item was admitted with. +func (l *branchWorkerEventLoop) dispatch(item WorkItem) int64 { + switch { + case item.Attach != nil: l.handleAttachCommitRequest(item.Attach) - return - } - - if item.Withdraw != nil { + case item.Withdraw != nil: l.handleWithdrawCommitRequest(item.Withdraw) - return - } - - if item.Resync != nil { - // Take the CURRENT request for this scope: a newer one may have replaced - // it while this marker waited in the FIFO. Running it here keeps the - // original queue position, so a resync still lands before the live events - // buffered behind it. - l.handleResyncRequest(l.w.takePendingResync(item.Resync)) - return - } - - if item.Refresh != nil { + case item.Resync != nil: + // Take the CURRENT request for this scope: a newer one may have replaced it while this + // marker waited in the FIFO. Running it here keeps the original queue position, so a resync + // still lands before the live events buffered behind it. + req, charge := l.w.takePendingResync(item.Resync) + l.handleResyncRequest(req) + return charge + case item.Refresh != nil: l.handleRefreshRequest(item.Refresh) - return - } - - if item.Request == nil { - return - } - if item.Request.CommitMode == CommitModeAtomic { + case item.Request == nil: + case item.Request.CommitMode == CommitModeAtomic: targetName, targetNamespace := atomicRefusalTarget(item.Request) if !l.w.normalWritesAllowed(targetName, targetNamespace) { l.w.Log.V(1).Info("Dropping atomic write while render fidelity is not established", "gitTarget", targetNamespace+"/"+targetName) - return + break } l.handleAtomicRequest(item.Request) - return + default: + l.handleLiveEvents(item.Request) } - - l.handleLiveEvents(item.Request) + return item.charge } // handleLiveEvents appends one write request's live events to the commit window, splitting and @@ -1295,7 +1326,6 @@ func (l *branchWorkerEventLoop) handleLiveEvents(request *WriteRequest) { "resource", event.Identifier.String(), "eventTarget", event.GitTargetNamespace+"/"+event.GitTargetName) l.finalizeOpenWindowWithReason(windowFinalizeReasonIdentityChange) - l.maybeSchedulePush() // The window just closed and a new one for this event has not opened yet: an idle // boundary. Drain any parked heal here so it gets a turn even under sustained, // author-alternating load that never lets the silence timer fire (Rec 1 non-starvation). @@ -1319,9 +1349,12 @@ func (l *branchWorkerEventLoop) handleLiveEvents(request *WriteRequest) { } opened = true } - l.openWindow.add(event) + // The window keeps one event per path, so a replaced one stops being charged. + if replaced, ok := l.openWindow.add(event); ok { + l.windowBytes -= eventRetainedCharge(&replaced) + } l.openWindow.lastWriteAt = time.Now() - l.windowBytes += l.w.estimateEventSize(event) + l.windowBytes += eventRetainedCharge(&event) if opened { // A waiting CommitRequest gets the new window BEFORE anything can close it: before the @@ -1343,7 +1376,6 @@ func (l *branchWorkerEventLoop) handleLiveEvents(request *WriteRequest) { // fails on a stuck remote will not free memory, but that's the // known stuck-push pathology documented in the design. l.finalizeOpenWindowWithReason(windowFinalizeReasonBufferLimit) - l.maybeSchedulePush() continue } l.closeOrArmWindow() @@ -1362,16 +1394,14 @@ func (l *branchWorkerEventLoop) closeOrArmWindow() { delay := time.Until(closeAt) if delay <= 0 { l.finalizeOpenWindowWithReason(reason) - l.maybeSchedulePush() return } l.resetCommitTimer(delay) } -// handleAtomicRequest applies one atomic write request. Atomic batches bypass the commit -// window, but not the retained push lifecycle: any open live work is finalized first so -// arrival order is preserved, then the atomic write joins pendingWrites and the normal -// cooldown-driven push path decides when to publish. +// handleAtomicRequest decides one atomic write request. Atomic batches bypass the commit window, but +// not the pending writes: any open live work is finalized first so arrival order is preserved, then +// the atomic write joins the pending writes behind it. func (l *branchWorkerEventLoop) handleAtomicRequest(request *WriteRequest) { l.finalizeOpenWindowWithReason(windowFinalizeReasonAtomicBeforeApply) // Finalizing the window opened an idle boundary: a heal parked behind that window @@ -1379,50 +1409,14 @@ func (l *branchWorkerEventLoop) handleAtomicRequest(request *WriteRequest) { // then atomic) rather than letting the atomic overtake it. l.applyDeferredHeals() - // Recover here too, and not only inside the finalize above: that call returns at its first - // line when no window is open, which is the ordinary case for an atomic write. Without this, - // an atomic arriving while work is retained and the worktree is dirty would commit a failed - // write's leftovers — commitPendingWrites cannot reset for us, because retained local commits - // are exactly what a reset would destroy. - if err := l.recoverRetainedWrites(); err != nil { - l.w.recordCommitFailure(commitFailureKindAtomic, commitFailureError) - l.w.Log.Error(err, "Failed to recover a dirty worktree; dropping atomic request", - "events", len(request.Events)) - l.noteParentUnavailable(err, atomicScope(request)) - return - } - pendingWrite, err := l.w.buildAtomicPendingWrite(l.w.ctx, request) if err != nil { l.w.recordCommitFailure(commitFailureKindAtomic, commitFailureError) l.w.Log.Error(err, "Failed to build atomic pending write", "events", len(request.Events)) return } - - // Retain the batch, not the original write: executePendingWrites stamps CommitSHA into the - // slice it is given, and the retained write is what the push counts from. - batch := []PendingWrite{*pendingWrite} - if err := l.w.commitPendingWrites(batch, len(l.pendingWrites) > 0); err != nil { - // A refused write plan is surfaced as a GitTarget status transition rather than - // logged as a write fault; nothing was committed either way, so the request is - // dropped in both cases. - name, namespace := atomicRefusalTarget(request) - if isRefusal, refused := l.w.reportPathRefusal( - err, name, namespace, request.sourceCollection()); isRefusal { - l.w.recordCommitFailure(commitFailureKindAtomic, commitFailureRefused) - l.touchBranchForRefusal(name, namespace, err.Error(), refused, - refusalObservationForEvents(request.Events, refused), request.sourceCollection()) - } else { - l.w.recordCommitFailure(commitFailureKindAtomic, commitFailureError) - l.w.Log.Error(err, "Atomic commit failed; dropping request", "events", len(request.Events)) - } - l.noteParentUnavailable(err, atomicScope(request)) - return - } - - l.pendingWrites = append(l.pendingWrites, batch[0]) - l.pendingWritesBytes += batch[0].ByteSize - l.maybeSchedulePush() + pendingWrite.origin.atomic = request + l.decide(*pendingWrite) } func (l *branchWorkerEventLoop) handleShutdown() { @@ -1432,8 +1426,8 @@ func (l *branchWorkerEventLoop) handleShutdown() { // clean shutdown rather than leaking its caller's reply channel. l.applyDeferredHeals() if len(l.pendingWrites) > 0 { - // Shutdown bypasses the cooldown — pending work needs to land before - // the worker exits, even if a push was just sent. + // Shutdown bypasses the cooldown and the retry: pending work needs to land before the + // worker exits, even if a push was just sent. The push commits what was just decided. l.pushPending() } l.failUnpushedCommitRequests() @@ -1474,6 +1468,11 @@ func (w *BranchWorker) drainQueue() { if item.Attach != nil { w.uncountQueuedAttach(item.Attach.id()) } + charge := item.charge + if item.Resync != nil { + _, charge = w.takePendingResync(item.Resync) + } + w.intake.release(charge) w.inflightItems.Add(-1) default: return @@ -1495,102 +1494,15 @@ func (l *branchWorkerEventLoop) resetCommitTimer(delay time.Duration) { l.commitTimer.Reset(delay) } -// recoverRetainedWrites resets and replays when retained work can no longer be trusted, for either -// of two reasons: a previous write left the worktree dirty, or a reset discarded the local commits -// behind the retained writes and the replay that should have rebuilt them did not finish. -// -// It runs before any commit the loop makes — finalizeOpenWindowWithReason, handleAtomicRequest, -// applyResync, and the two empty commits, recordCommitRequest and commitRefusalTouch — and before -// any push it makes. A path that reaches commitPendingWrites without -// calling this can commit a failed write's leftovers; -// TestEveryLoopCommitPathRecoversADirtyWorktree is what makes adding one fail loudly. -// -// It has to live on the LOOP, not inside commitPendingWrites, for two reasons an earlier draft of -// the design got wrong. commitPendingWrites already holds repoMu and -// refreshRemoteAndRebuildPendingWrites takes it, so calling one from the other deadlocks; and -// commitPendingWrites is handed the INCOMING batch, while the writes that need replaying are the -// retained ones the loop owns. Recovering from inside it would deadlock on the way to replaying -// the wrong slice. -// -// The case only arises with retained work. With nothing retained, commitPendingWrites' own guard -// resets for us, and it can do that safely because there is nothing to lose. -func (l *branchWorkerEventLoop) recoverRetainedWrites() error { - if len(l.pendingWrites) == 0 { - // Nothing is retained, so there is nothing a stale replay could strand. A replay only - // ever marks itself required while writes are retained, but clear it here too so the flag - // can never outlive the work it was about. - l.w.markReplayComplete() - return nil - } - // A parent change is read from the generations, not taken from a flag: a rebuild that fails - // leaves them apart, so the next attempt rebuilds again. - dirty, needsReplay, parentChanged := l.w.worktreeDirty(), l.w.replayRequired(), l.w.rootParentStale() - if !dirty && !needsReplay && !parentChanged { - return nil - } - // The rebuild fetches, and a parent known to be missing is not fetched again before its probe - // is due: every commit path calls this, so without the check each write arriving in the - // meantime would cost a connection. - if l.w.awaitingParentProbe() { - return errAwaitingParentProbe - } - - l.w.Log.Info("Rebuilding retained writes onto the remote tip", - "pendingWrites", len(l.pendingWrites), - "worktreeDirty", dirty, - "replayRequired", needsReplay, - "parentBranchChanged", parentChanged) - // fetchReasonRecovery, the same series the no-retained-writes case records in - // ensureBaseForCycle: this is one event, and which half of it an operator sees must not depend - // on whether a push happened to be in cooldown at the time. - return l.invalidateAndRefresh("retained writes cannot be trusted", fetchReasonRecovery) -} - -// invalidateAndRefresh drops base trust and, when writes are retained, acts on that invalidation -// at once by resetting to the remote tip and replaying them. -// -// Clearing the flag on its own is not enough, and the asymmetry is easy to miss: ensureBaseForCycle -// consults baseTrusted only when NOTHING is retained, because a reset would destroy the local -// commits retained writes already produced. A target holding work therefore ignores a bare -// invalidation completely and plans its next cycle on the stale base anyway. +// finalizeOpenWindowWithReason closes the live event window into one decided write. The attached +// CommitRequest message overrides the live template. // -// Nothing is lost by resetting here: a replay re-PLANS from the retained writes rather than from -// the worktree, which is exactly what makes discarding the worktree safe. -// -// This is the second effect the inbound push receiver needs as well (§8.1): the handler itself -// performs no round trip, the worker does, at the moment it was going to talk to the remote anyway. -func (l *branchWorkerEventLoop) invalidateAndRefresh(reason, fetchReason string) error { - l.w.invalidateBase(reason) - if len(l.pendingWrites) == 0 { - // Nothing to replay, so fetch and reset directly. Leaving it to ensureBaseForCycle would - // work, but that call records `publication`, so the fetch would be attributed to the very - // series this design asserts at zero on a healthy target: a snapshot resync would read as - // a regression to fetching on every publication. This does NOT fetch twice — the reset - // leaves the base trusted, so ensureBaseForCycle skips its own. - return l.w.syncWithRemote(l.w.ctx, fetchReason) - } - if err := l.w.refreshRemoteAndRebuildPendingWrites(l.w.ctx, l.pendingWrites, fetchReason); err != nil { - return fmt.Errorf("refresh after %s: %w", reason, err) - } - return nil -} - -// finalizeOpenWindow closes the live event window using the generated -// grouped-commit message. It returns true when a commit-shaped pending write -// was produced and retained. -func (l *branchWorkerEventLoop) finalizeOpenWindow() bool { - return l.finalizeOpenWindowWithReason(windowFinalizeReasonUnspecified) -} - -// finalizeOpenWindowWithReason closes the live event window into one retained pending write and -// creates the local commit. The attached CommitRequest message overrides the live template. -// -// On failure the window is DROPPED rather than retried: the repo is unreachable or the events are -// unrecoverable, and retrying the same broken state every cycle helps nobody. An attached -// CommitRequest is then resolved Failed. -func (l *branchWorkerEventLoop) finalizeOpenWindowWithReason(reason windowFinalizeReason) bool { +// The closed window is a decided write (decide): a remote that cannot be reached leaves it pending +// for the publication retry, and its CommitRequest held. Only a failure of the write itself +// drops it, and resolves that request Failed (settleFailed). +func (l *branchWorkerEventLoop) finalizeOpenWindowWithReason(reason windowFinalizeReason) { if l.openWindow == nil { - return false + return } l.stopCommitTimer() @@ -1610,7 +1522,7 @@ func (l *branchWorkerEventLoop) finalizeOpenWindowWithReason(reason windowFinali l.w.Log.V(1).Info("Discarding open window while render fidelity is not established", "reason", string(reason), "gitTarget", targetNamespace+"/"+targetName) l.dropOpenWindow(pendingCR, err) - return false + return } l.w.Log.Info("Finalizing open commit window", @@ -1623,14 +1535,6 @@ func (l *branchWorkerEventLoop) finalizeOpenWindowWithReason(reason windowFinali "messageOverride", effectiveMessage != "", "attachedCR", pendingCR != nil) - if err := l.recoverRetainedWrites(); err != nil { - l.w.recordCommitFailure(commitFailureKindWindow, commitFailureError) - l.w.Log.Error(err, "Failed to recover a dirty worktree; dropping open window", - "reason", string(reason), "windowTarget", windowTarget) - l.dropFailedWindow(err, pendingCR, err, windowScopes(targetNamespace, targetName, events)) - return false - } - pendingWrite, err := l.w.buildGroupedPendingWrite(l.w.ctx, events) if err != nil { l.w.recordCommitFailure(commitFailureKindWindow, commitFailureError) @@ -1640,68 +1544,18 @@ func (l *branchWorkerEventLoop) finalizeOpenWindowWithReason(reason windowFinali "windowTarget", windowTarget, "events", len(events)) l.dropOpenWindow(pendingCR, fmt.Errorf("build pending write: %w", err)) - return false + return } pendingWrite.CommitMessage = effectiveMessage pendingCR, proceed := l.gateBuiltWindow(pendingCR, pendingWrite.Target(), reason) if !proceed { - return false + return } l.carryRequestOnWrite(pendingWrite, pendingCR) - // Commit on a single-element batch so executePendingWrites threads the resulting - // commit hash back onto batch[0]; the retained write then carries the real SHA - // into the push (and the rebase-replay refreshes it). - batch := []PendingWrite{*pendingWrite} - hasPendingCommits := len(l.pendingWrites) > 0 - if err := l.w.commitPendingWrites(batch, hasPendingCommits); err != nil { - // A refused write plan (acceptance gate or write-boundary precondition) committed - // nothing and needs a human to fix the Git path, so it is surfaced as - // GitPathAccepted=False instead of being logged as a transient write fault. The - // window is dropped either way — the events are already lost to the failed flush, - // and the next resync re-derives them. - if isRefusal, refused := l.w.reportPathRefusal( - err, targetName, targetNamespace, sourceCollectionForEvents(events)); isRefusal { - l.w.recordCommitFailure(commitFailureKindWindow, commitFailureRefused) - l.touchBranchForRefusal(targetName, targetNamespace, err.Error(), refused, - refusalObservationForEvents(events, refused), sourceCollectionForEvents(events)) - } else { - l.w.recordCommitFailure(commitFailureKindWindow, commitFailureError) - l.w.Log.Error(err, "Commit failed; dropping open window", - "reason", string(reason), - "windowAuthor", windowAuthor, - "windowTarget", windowTarget, - "events", len(events)) - } - l.dropFailedWindow(err, pendingCR, fmt.Errorf("commit failed: %w", err), - windowScopes(targetNamespace, targetName, events)) - return false - } - - l.pendingWrites = append(l.pendingWrites, batch[0]) - l.pendingWritesBytes += batch[0].ByteSize l.openWindow = nil l.windowBytes = 0 - - if pendingCR != nil { - l.w.setCommitRequestPhase(*pendingCR, PhaseWaitingForPush) - // Resolution moves to the push success path either way (§6.5), including for a no-diff - // window. It is no longer window-pending — it now rides the retained write. - // - // A no-diff window used to resolve AlreadyPresent right here, on the strength of the - // local plan finding nothing to change. That was only safe while every cycle fetched - // first. It is not safe now: the plan may have run against a tree the remote has moved - // past, and the replay after the rejection can produce a real commit for a request that - // has already told its caller there was nothing to do. "Already present" is a claim - // about the REMOTE, so only the remote can settle it, and the push is where it speaks. - // - // It is MARKED rather than forgotten. Dropping it here left the worker unable to tell a - // re-sent attach for this very request from a new one, which resolved it NoOpenWindow - // during the push cooldown — the one window where it is neither pending nor resolved. - if pcr := l.pendingCRs[*pendingCR]; pcr != nil { - pcr.committed = true - } - } + l.decide(*pendingWrite) l.w.Log.Info("Open commit window finalized", "reason", string(reason), @@ -1709,7 +1563,6 @@ func (l *branchWorkerEventLoop) finalizeOpenWindowWithReason(reason windowFinali "windowTarget", windowTarget, "events", len(events), "pendingWrites", len(l.pendingWrites)) - return true } // carryRequestOnWrite carries the attached CommitRequest onto the write so its result follows the @@ -1750,17 +1603,8 @@ func (l *branchWorkerEventLoop) gateBuiltWindow( return pendingCR, true } -// dropOpenWindow discards a window whose finalize failed, resolving any attached -// CommitRequest as Failed so the controller does not poll forever. -// dropFailedWindow drops a window whose write failed with err, and when the parent branch is what -// failed it, remembers the scopes the dropped writes belong to so they are recovered. -func (l *branchWorkerEventLoop) dropFailedWindow( - err error, pendingCR *commitRequestID, cause error, scopes []recoveryScope, -) { - l.noteParentUnavailable(err, scopes...) - l.dropOpenWindow(pendingCR, cause) -} - +// dropOpenWindow discards a window whose finalize failed before it was decided, resolving any +// attached CommitRequest as Failed so the controller does not poll forever. func (l *branchWorkerEventLoop) dropOpenWindow(pendingCR *commitRequestID, cause error) { l.openWindow = nil l.windowBytes = 0 @@ -1769,18 +1613,13 @@ func (l *branchWorkerEventLoop) dropOpenWindow(pendingCR *commitRequestID, cause } } -// maybeSchedulePush is the post-commit hook: it pushes immediately when the -// cooldown has elapsed (or has never fired), and otherwise schedules a -// one-shot pushTimer to fire when the cooldown expires. While the cooldown -// is active, additional commits accumulate locally; when the timer fires, -// all pending commits go to the remote in a single push. After a failed -// publication the retry's deadline governs instead, so new commits do not -// each spend a push on a remote that just refused one. +// maybeSchedulePush is advance's push step: it pushes immediately when the cooldown has elapsed (or +// has never fired), and otherwise schedules a one-shot pushTimer to fire when the cooldown expires. +// While the cooldown is active, additional commits accumulate locally; when the timer fires, all +// pending commits go to the remote in a single push. After a failed attempt the retry's deadline +// governs instead, so new commits do not each spend a push on a remote that just refused one. func (l *branchWorkerEventLoop) maybeSchedulePush() { - if len(l.pendingWrites) == 0 { - return - } - if l.awaitingPublicationRetry() { + if len(l.pendingWrites) == 0 || l.awaitingRetry() { return } if l.lastPushAt.IsZero() { @@ -1797,15 +1636,14 @@ func (l *branchWorkerEventLoop) maybeSchedulePush() { } } -// pushPending publishes any retained pending writes that already exist as local -// commits. On success, pendingWrites is cleared and lastPushAt advances. On -// failure (transient or after exhausting replay retries), pendingWrites stays -// in place, safe but unpublished, and the push timer is armed for the next -// attempt (publication_retry.go), so the work lands without another commit. +// pushPending materializes the pending writes and publishes them. On success, pendingWrites is cleared, +// lastPushAt advances, and the outage, if there was one, is over. On failure (transient or after +// exhausting replay retries), pendingWrites stays in place, safe but unpublished, and the next +// attempt is scheduled (retry.go), so the work lands without another commit. Pending writes that +// all settle without a push are advance's to close. func (l *branchWorkerEventLoop) pushPending() { if len(l.pendingWrites) == 0 { l.stopPushTimer() - l.notePublicationSettled() return } if l.w.awaitingParentProbe() { @@ -1814,28 +1652,32 @@ func (l *branchWorkerEventLoop) pushPending() { return } - // A reset may have discarded the local commits behind these writes while the replay that - // rebuilds them did not finish. Pushing now would find the branch already at the remote tip, - // report success without sending anything, and settle work that exists nowhere. Rebuild first, - // and keep the writes rather than publish a lie if that fails. - if err := l.recoverRetainedWrites(); err != nil { + // Commit any decided write that is not committed yet, and rebuild when a reset discarded the + // local commits behind the others: pushing then would find the branch already at the remote + // tip, report success without sending anything, and settle work that exists nowhere. Keep the + // writes rather than publish a lie if that fails. + if err := l.materialize(); err != nil { + l.publicationFailed(err, "Cannot publish until the retained writes are committed; keeping them") + return + } + if len(l.pendingWrites) == 0 { + // Every decided write failed to commit, or committed nothing, so there is nothing to publish. l.stopPushTimer() - l.noteParentUnavailable(err) - l.notePublicationFailed() - l.w.Log.Error(err, "Cannot publish until the retained writes are rebuilt; keeping them", - "pendingWrites", len(l.pendingWrites), "retryAt", l.publicationRetry.nextAttempt) return } - if err := l.w.pushPendingCommits(l.pendingWrites); err != nil { + err := l.w.pushPendingCommits(l.pendingWrites) + // A rejected push replays the writes onto the moved remote, and the new tree can refuse some of + // them. Those were not pushed and never will be, whatever the push did with the rest: they stop + // being pending now, and are settled once the pending writes describe the push's outcome, + // because settling one can decide more work. + refused := l.takeReplayRefusals() + if err != nil { // Leave pendingWrites in place; do NOT advance lastPushAt — the // design specifies lastPushAt only advances on a successful push. A // CommitRequest riding a retained write stays unresolved while we retry. - l.stopPushTimer() - l.noteParentUnavailable(err) - l.notePublicationFailed() - l.w.Log.Error(err, "Push failed; pending writes retained for retry", - "pendingWrites", len(l.pendingWrites), "retryAt", l.publicationRetry.nextAttempt) + l.publicationFailed(err, "Push failed; pending writes retained for retry") + l.settleReplayRefusals(refused) return } @@ -1847,8 +1689,16 @@ func (l *branchWorkerEventLoop) pushPending() { l.pendingWritesBytes = 0 l.lastPushAt = time.Now() l.stopPushTimer() - l.notePublicationSettled() - l.noteRecoveryPublished() + l.endOutage() + l.settleReplayRefusals(refused) +} + +// publicationFailed keeps the writes for the next attempt, which it schedules: see retry.go. +func (l *branchWorkerEventLoop) publicationFailed(err error, message string) { + l.stopPushTimer() + l.noteParentUnavailable(err) + l.scheduleRetry(err) + l.w.Log.Error(err, message, "pendingWrites", len(l.pendingWrites), "retryAt", l.retry.due) } // resolvePushedCommitRequests settles every CommitRequest carried by a just-pushed write, now @@ -1919,9 +1769,9 @@ func (l *branchWorkerEventLoop) stopTimers() { l.stopPushTimer() l.stopAttachTimer() l.stopRefusalTimer() - if l.recovery.timer != nil { - l.recovery.timer.Stop() - l.recovery.timer = nil + if l.retry.timer != nil { + l.retry.timer.Stop() + l.retry.timer = nil } } @@ -1945,15 +1795,24 @@ func (w *BranchWorker) baseTrusted() bool { return w.baseTrustedState.Load() && w.baseParentGen.Load() == w.parentSnapshot().gen } -// worktreeDirty reports whether the worktree may hold a partial write. +// checkoutUnknown is the checkoutApplied value of a worktree a failed write may have left anything +// in. Only a reset clears it. // -// This is NOT the same question as baseTrusted, and collapsing the two is a bug rather than a +// It is NOT the same question as baseTrusted, and collapsing the two is a bug rather than a // simplification. A write can fail part-way through executePendingWrites and leave staged changes // behind while an earlier write is still retained; if the push that follows were allowed to // declare everything well, the next cycle would plan on top of those leftovers and commit them // under an unrelated author. So a successful push may make the base trusted, and must never make -// the worktree clean. Only a reset does that. -func (w *BranchWorker) worktreeDirty() bool { return w.worktreeDirtyState.Load() } +// the worktree clean. +const checkoutUnknown = -1 + +// worktreeDirty reports whether the worktree may hold a partial write. +func (w *BranchWorker) worktreeDirty() bool { return w.checkoutApplied.Load() == checkoutUnknown } + +// checkoutHolds reports whether the checkout is the projection of n retained writes: its root plus +// one commit for each. With n > 0 and anything else, the writes must be replayed before they are +// committed on or pushed. +func (w *BranchWorker) checkoutHolds(n int) bool { return w.checkoutApplied.Load() == int64(n) } // setBaseTrusted records whether the base can be trusted. // @@ -1978,32 +1837,26 @@ func (w *BranchWorker) invalidateBase(reason string) { // markWorktreeDirty records that the worktree may hold a partial write. Only a reset clears it. func (w *BranchWorker) markWorktreeDirty(reason string) { - if !w.worktreeDirtyState.Swap(true) { + if w.checkoutApplied.Swap(checkoutUnknown) != checkoutUnknown { w.Log.V(1).Info("Worktree may hold a partial write", "reason", reason, "branch", w.Branch) } } -// markWorktreeClean records that a reset has just discarded whatever the worktree held. It is -// called only from updateBranchMetadataFromPullReport, because that is the one place a reset is -// known to have happened. -func (w *BranchWorker) markWorktreeClean() { - if w.worktreeDirtyState.Swap(false) { - w.Log.V(1).Info("Worktree reset to the fetched tip", "branch", w.Branch) - } -} - -// replayRequired reports that a reset discarded the local commits behind the retained writes and -// the replay that rebuilds them has not completed. Until it clears, those writes may not be -// pushed: the push would succeed against a branch that already equals the remote tip and settle -// work that exists nowhere. -func (w *BranchWorker) replayRequired() bool { return w.replayRequiredState.Load() } +// markCheckoutReset records that the checkout now holds none of the retained writes: a reset to +// the remote tip landed, or is about to. It is called BEFORE a reset that a replay will follow, as +// well as after every reset, because a reset moves the branch ref and then rewrites the worktree, +// so it can fail with the ref already moved and the commits behind the retained writes already +// unreachable. Marking too eagerly costs one replay. +func (w *BranchWorker) markCheckoutReset() { w.checkoutApplied.Store(0) } -// markReplayRequired is called the moment a reset lands inside a replay, BEFORE the rebuild is -// attempted, so an abort anywhere in the rebuild leaves the flag set. -func (w *BranchWorker) markReplayRequired() { - if !w.replayRequiredState.Swap(true) { - w.Log.V(1).Info("Retained writes need rebuilding: their local commits were reset away", - "branch", w.Branch) +// markCheckoutPublished records that a push published every retained write: unless a failed write +// left the worktree dirty, the checkout is its root again. +func (w *BranchWorker) markCheckoutPublished() { + for { + applied := w.checkoutApplied.Load() + if applied == checkoutUnknown || w.checkoutApplied.CompareAndSwap(applied, 0) { + return + } } } @@ -2011,25 +1864,17 @@ func (w *BranchWorker) markReplayRequired() { // hold pendingResyncsMu and must still be holding it when it sends: see stoppingState. func (w *BranchWorker) stoppingLocked() bool { return w.stoppingState } -// markReplayComplete is called only when a rebuild has replayed every retained write. -func (w *BranchWorker) markReplayComplete() { w.replayRequiredState.Store(false) } - // ensureBaseForCycle performs the head-of-cycle fetch, which is now conditional. // // The push session reads the remote's ref advertisement on a connection the cycle was making // anyway, and a cycle that commits nothing still reaches it, so a trusted base needs no fetch to // plan against. That is the whole saving. See docs/design/push-notification-and-reconcile-trigger.md §1.3 and §1.5. // -// Only the first commit of a cycle may fetch at all: a reset would destroy the local commits the -// retained writes already produced. When those exist AND the worktree is dirty, recovery is the -// event loop's job instead — see recoverRetainedWrites, which resets and replays rather than -// resetting alone. -func (w *BranchWorker) ensureBaseForCycle( - provider *configv1alpha3.GitProvider, - repoPath string, - hasPendingCommits bool, -) error { - if hasPendingCommits { +// Only a checkout holding no retained writes may fetch here at all: a reset would destroy the +// local commits those writes produced. With writes retained, the event loop's materialize has +// already made the checkout their projection, resetting and replaying when it had to. +func (w *BranchWorker) ensureBaseForCycle(provider *configv1alpha3.GitProvider, repoPath string) error { + if w.checkoutApplied.Load() > 0 { return nil } w.restampParentGenIfIndependent() @@ -2065,13 +1910,11 @@ func (w *BranchWorker) ensureBaseForCycle( return nil } -// commitPendingWrites creates local commits for the provided pending writes without pushing them. -// -// hasPendingCommits reports whether commits from earlier in the current push cycle are retained. -// It gates the head-of-cycle base check: only the first commit of a cycle may fetch and reset, -// because a reset would destroy exactly those retained commits. See ensureBaseForCycle, which -// decides whether that fetch is needed at all. -func (w *BranchWorker) commitPendingWrites(pendingWrites []PendingWrite, hasPendingCommits bool) error { +// commitPendingWrites creates local commits for the provided pending writes without pushing them, +// on top of whatever retained writes the checkout already holds (checkoutApplied), and counts +// each write the loop retains into it. Only the first commit with nothing committed before it may +// fetch and reset; see ensureBaseForCycle, which decides whether that fetch is needed at all. +func (w *BranchWorker) commitPendingWrites(pendingWrites []PendingWrite) error { w.repoMu.Lock() defer w.repoMu.Unlock() @@ -2085,7 +1928,7 @@ func (w *BranchWorker) commitPendingWrites(pendingWrites []PendingWrite, hasPend } repoPath := w.repoPath() - if err := w.ensureBaseForCycle(provider, repoPath, hasPendingCommits); err != nil { + if err := w.ensureBaseForCycle(provider, repoPath); err != nil { return err } @@ -2103,7 +1946,7 @@ func (w *BranchWorker) commitPendingWrites(pendingWrites []PendingWrite, hasPend return err } - if !hasPendingCommits { + if w.checkoutApplied.Load() == 0 { w.pushCycleRootBranch = baseBranch w.pushCycleRootHash = baseHash w.pushCycleRootGen = w.baseParentGen.Load() @@ -2111,8 +1954,17 @@ func (w *BranchWorker) commitPendingWrites(pendingWrites []PendingWrite, hasPend commitsCreated, err := w.executePendingWrites(w.ctx, repo, pendingWrites) if err != nil { - w.invalidateBase("execute pending writes failed") - return fmt.Errorf("execute pending writes: %w", err) + // A failed write undone locally left the checkout where it was, base included; one that + // could not be undone has left a worktree nobody has verified. + if w.worktreeDirty() { + w.invalidateBase("execute pending writes failed") + } + return writeFailedError{fmt.Errorf("execute pending writes: %w", err)} + } + for i := range pendingWrites { + if pendingWrites[i].retained() { + w.checkoutApplied.Add(1) + } } if commitsCreated == 0 { return nil @@ -2149,6 +2001,10 @@ func (w *BranchWorker) pushPendingCommits(pendingWrites []PendingWrite) error { } w.recordPushOutcome(outcome, started) if err == nil { + // The published commits are the remote tip now, so they are the root the next write + // commits on, and the checkout holds no pending write on top of it. A worktree a + // failed write left dirty stays dirty: the push says nothing about its leftovers. + w.markCheckoutPublished() // The commits are on the remote now, which is the only place they can honestly be counted, // and pendingWrites reflect whatever the final replay produced. w.publishCommitsForPush(pendingWrites) @@ -2158,13 +2014,20 @@ func (w *BranchWorker) pushPendingCommits(pendingWrites []PendingWrite) error { // runPushCycle is the push itself: try, and on a rejection caused by a moved remote, sync, rebuild // the pending writes on the new head, and try again. The caller owns repoMu and the measurement. +// +// The whole cycle runs under one deadline, gitPublishTimeout, on top of each call's own bound +// (network_bound.go): three contention attempts, each with a fetch and a replay, must not each +// start a fresh budget while the loop, and every target on the branch, waits. func (w *BranchWorker) runPushCycle(pendingWrites []PendingWrite) error { - provider, err := w.getGitProvider(w.ctx) + ctx, cancel := context.WithTimeout(w.ctx, gitPublishTimeout) + defer cancel() + + provider, err := w.getGitProvider(ctx) if err != nil { return fmt.Errorf("get GitProvider: %w", err) } - auth, err := getAuthFromSecret(w.ctx, w.Client, provider, w.sshHostKeys, w.credentialPolicy) + auth, err := getAuthFromSecret(ctx, w.Client, provider, w.sshHostKeys, w.credentialPolicy) if err != nil { return fmt.Errorf("resolve auth: %w", err) } @@ -2196,50 +2059,19 @@ func (w *BranchWorker) runPushCycle(pendingWrites []PendingWrite) error { w.Log.Info("Parent branch changed since this cycle was planned; rebuilding before the push", "branch", w.Branch, "parentBranch", w.ParentBranch()) w.recordFetch(fetchReasonRecovery) - if err := w.replayOntoRemote(repo, pendingWrites, auth); err != nil { + if err := w.replayOntoRemote(ctx, repo, pendingWrites, auth); err != nil { return err } continue } - outcome, err := pushAtomicFn(w.ctx, repo, rootHash, rootBranch, auth) + outcome, err := pushAtomicFn(ctx, repo, rootHash, rootBranch, auth) if err == nil { - // A push that returns without an error is an OBSERVATION of the remote, on the - // connection it was opening anyway: the server took the ref update, the advertisement - // already showed the branch at our head, or it did not carry the branch at all. All - // three say where the branch is, so all three take trust — including the push that - // CREATED the branch, which the old branchExists guard (written by fetches alone) - // wrongly excluded. - // - // This must NOT clear worktreeDirty. An earlier write can have failed part-way - // through executePendingWrites and left staged changes behind while this write was - // retained; a successful push says where the remote is, and nothing about that. - w.setBaseTrusted(true) - // The half a fetch-only record could not carry: on an active branch the push is the - // event that MOVES the revision. - w.recordPushObservation(outcome, rootBranch, rootHash) - w.endPushCycle(outcome.Kind) - w.firsts.push.Do(func() { - w.Log.Info("First push to remote completed", - "branch", w.Branch, - // String(), never the raw URL: spec.url takes userinfo, and this line is at - // default verbosity. - "repository", w.repo.String(), - "commits", len(pendingWrites)) - }) + w.notePushSucceeded(outcome, rootBranch, rootHash, len(pendingWrites)) return nil } lastErr = err - // A rejection is a moved remote; any other push error may have died mid-upload. Either - // way the remote state is no longer something we can claim to know. - w.invalidateBase("push failed or was rejected") - - if !w.remoteMovedDuringPush(repo, err, rootBranch, rootHash, auth) { - return err - } - w.recordPushRetry(pushRetryRemoteMoved) - w.recordFetch(fetchReasonContention) - if err := w.replayOntoRemote(repo, pendingWrites, auth); err != nil { + if retry, err := w.afterFailedPush(ctx, repo, pendingWrites, err, rootBranch, rootHash, auth); !retry { return err } } @@ -2250,35 +2082,111 @@ func (w *BranchWorker) runPushCycle(pendingWrites []PendingWrite) error { return fmt.Errorf("push failed after %d attempts: %w", maxRetries, lastErr) } +// afterFailedPush decides what one failed push attempt means: the remote moved, and the writes are +// replayed onto it for another attempt (retry); the push landed although its answer was lost, and +// the cycle is done (nil); or anything else, which ends the cycle with the push's error. The caller +// owns repoMu. +func (w *BranchWorker) afterFailedPush( + ctx context.Context, + repo *gogit.Repository, + pendingWrites []PendingWrite, + pushErr error, + rootBranch plumbing.ReferenceName, + rootHash plumbing.Hash, + auth []gitclient.Option, +) (bool, error) { + // A rejection is a moved remote; any other push error may have died mid-upload. Either + // way the remote state is no longer something we can claim to know. + w.invalidateBase("push failed or was rejected") + + moved, landed := w.remoteMovedDuringPush(ctx, repo, pushErr, rootBranch, rootHash, auth) + if landed != nil { + // The push reached the remote and only its answer was lost (a deadline, a dropped + // connection): the branch is at the commits we sent. Replaying them would plan the same + // writes onto a tree that already holds them, and a save's empty commit would land twice. + w.Log.Info("A push whose reply was lost had landed; the remote holds our commits", + "branch", w.Branch, "head", landed.Head.String(), "err", pushErr.Error()) + w.notePushSucceeded(*landed, rootBranch, rootHash, len(pendingWrites)) + return false, nil + } + if !moved { + return false, pushErr + } + w.recordPushRetry(pushRetryRemoteMoved) + w.recordFetch(fetchReasonContention) + if err := w.replayOntoRemote(ctx, repo, pendingWrites, auth); err != nil { + return false, err + } + return true, nil +} + +// notePushSucceeded records a push the remote took, or already had. The caller owns repoMu. +func (w *BranchWorker) notePushSucceeded( + outcome PushOutcome, rootBranch plumbing.ReferenceName, rootHash plumbing.Hash, commits int, +) { + // A push that returns without an error is an OBSERVATION of the remote, on the + // connection it was opening anyway: the server took the ref update, the advertisement + // already showed the branch at our head, or it did not carry the branch at all. All + // three say where the branch is, so all three take trust — including the push that + // CREATED the branch, which the old branchExists guard (written by fetches alone) + // wrongly excluded. + // + // This must NOT clear worktreeDirty. An earlier write can have failed part-way + // through executePendingWrites and left staged changes behind while this write was + // retained; a successful push says where the remote is, and nothing about that. + w.setBaseTrusted(true) + // The half a fetch-only record could not carry: on an active branch the push is the + // event that MOVES the revision. + w.recordPushObservation(outcome, rootBranch, rootHash) + w.endPushCycle(outcome.Kind) + w.firsts.push.Do(func() { + w.Log.Info("First push to remote completed", + "branch", w.Branch, + // String(), never the raw URL: spec.url takes userinfo, and this line is at + // default verbosity. + "repository", w.repo.String(), + "commits", commits) + }) +} + // replayOntoRemote syncs to the remote, rebuilds the pending writes on what it found, and makes // that the cycle's root. The caller owns repoMu and has recorded the fetch. func (w *BranchWorker) replayOntoRemote( + ctx context.Context, repo *gogit.Repository, pendingWrites []PendingWrite, auth []gitclient.Option, ) error { - // Marked BEFORE the reset, not after it. A reset moves the branch ref and then rewrites - // the worktree, so it can fail with the ref already moved and the commits behind these - // writes already unreachable. Marking afterwards misses exactly that case, and the cost of - // marking too eagerly is one fetch on the next cycle if the reset turns out to have - // changed nothing. - w.markReplayRequired() - pullReport, err := w.syncToRemote(w.ctx, repo, auth) + // Marked BEFORE the reset: see markCheckoutReset. + w.markCheckoutReset() + pullReport, err := w.syncToRemote(ctx, repo, auth) if err != nil { - w.invalidateBase("sync during replay failed") - return fmt.Errorf("sync remote during replay: %w", err) + w.invalidateBase("sync before replay failed") + return fmt.Errorf("sync remote before replay: %w", err) } w.updateBranchMetadataFromPullReport(pullReport) - rootBranch, rootHash, err := w.rebuildPendingWrites(repo, pendingWrites) + // The replay re-PLANS against the rebased worktree, so it can decide deletions the first apply + // never made, under whatever policy was in force when the write was planned. Re-check the policy + // before that happens, so an operator who tightened it in the meantime is obeyed. + rootBranch, rootHash, err := w.ensureWriteBranch(repo) + if err == nil { + err = w.tightenPendingPruneModes(ctx, pendingWrites) + } + held := 0 + if err == nil { + // A write the new tree refuses is skipped and stamped, not a failure of the replay: see + // replayPendingWrites. The checkout holds the others, and the loop settles the skipped one. + held, err = w.replayPendingWrites(ctx, repo, pendingWrites) + } if err != nil { - w.invalidateBase("replay rebuild failed") - return err + w.invalidateBase("rebuild before replay failed") + return fmt.Errorf("rebuild pending writes: %w", err) } w.pushCycleRootBranch = rootBranch w.pushCycleRootHash = rootHash w.pushCycleRootGen = w.baseParentGen.Load() - w.markReplayComplete() + w.checkoutApplied.Store(int64(held)) return nil } @@ -2329,52 +2237,72 @@ func (w *BranchWorker) rootParentStale() bool { // remoteMovedDuringPush reports whether a failed push was contention — the remote branch no // longer sitting at the hash our commits were based on — which is the only push failure a replay -// can fix. +// can fix. When it finds the remote at the commits the push sent, it reports the push as landed +// instead: the failure lost only the answer. // // It prefers the rejection's own advertisement and reaches the network only when the push -// produced none: a dropped connection, an auth failure, a server-side refusal. That fallback is a -// whole SmartFetch, which is why it is the exception rather than the path. +// produced none: a dropped connection, a deadline, an auth failure, a server-side refusal. That +// fallback is a whole SmartFetch, which is why it is the exception rather than the path. It is +// also the only path that can find a landed push: a rejection's advertisement was compared +// against our head already, and an advertisement at our head is never a rejection. // // A fetch that itself fails is reported as "not moved". The caller surfaces the original push // error either way, and replaying onto a tree nobody has read would be guessing at contention // rather than observing it. func (w *BranchWorker) remoteMovedDuringPush( + ctx context.Context, repo *gogit.Repository, pushErr error, rootBranch plumbing.ReferenceName, rootHash plumbing.Hash, auth []gitclient.Option, -) bool { +) (bool, *PushOutcome) { // A push that creates the write branch is rooted on its parent, so a rejection naming the // write branch itself is not about the root: somebody created the branch since we based our // work. That is contention all the same, and the replay builds on their commit. var moved *RemoteMovedError if errors.As(pushErr, &moved) && moved.Branch != rootBranch && moved.Branch == plumbing.NewBranchReferenceName(w.Branch) { - return true + return true, nil } // An empty repository gained refs since the cycle was planned on it. The replay's fetch // resolves the new default branch, or refuses, and the rebuild re-plans from the retained // writes instead of reusing the orphan commits. var notEmpty *RepositoryNotEmptyError if errors.As(pushErr, ¬Empty) { - return true + return true, nil } remoteHash, known := advertisedRootHash(pushErr, rootBranch) - if !known && rootBranch != plumbing.NewBranchReferenceName(w.Branch) { - return w.newBranchMovedDuringPush(rootBranch, rootHash, auth) - } - if !known { - var fetchErr error - w.recordFetch(fetchReasonPushFailureProbe) - remoteHash, fetchErr = fetchRemoteBranchHashFn(w.ctx, repo, rootBranch, auth) - if fetchErr != nil { - w.invalidateBase("remote-state fetch failed") - return false - } + if known { + return remoteHash != rootHash, nil } - return remoteHash != rootHash + if rootBranch != plumbing.NewBranchReferenceName(w.Branch) { + return w.newBranchMovedDuringPush(ctx, repo, rootBranch, rootHash, auth) + } + w.recordFetch(fetchReasonPushFailureProbe) + remoteHash, fetchErr := fetchRemoteBranchHashFn(ctx, repo, rootBranch, auth) + if fetchErr != nil { + w.invalidateBase("remote-state fetch failed") + return false, nil + } + if landed := landedPush(repo, remoteHash, rootHash); landed != nil { + return false, landed + } + return remoteHash != rootHash, nil +} + +// landedPush reports a push that reached the remote although its caller saw it fail: the branch +// is now at our local head, and that head is not the root the cycle started from (a cycle with +// nothing to send has nothing to have landed). Only the exact head counts. A remote that moved on +// past our commits would also hold them, but proving that needs ancestry a depth-1 fetch does not +// carry, so that case replays, as any other contention does. +func landedPush(repo *gogit.Repository, remoteHash, rootHash plumbing.Hash) *PushOutcome { + head := localHead(repo) + if head.IsZero() || head == rootHash || remoteHash != head { + return nil + } + return &PushOutcome{Kind: PushAccepted, Head: head} } // newBranchMovedDuringPush is the fallback for a push that was creating the write branch and failed @@ -2382,22 +2310,30 @@ func (w *BranchWorker) remoteMovedDuringPush( // Old was zero because somebody created the branch after our advertisement. Probing the root alone // would find the parent unmoved and conclude nothing happened, and the work would never replay. So // it reads both refs from one advertisement, and a write branch that is now there is contention, -// exactly like the typed rejection naming it. It runs only on this exceptional path. +// exactly like the typed rejection naming it — unless it is there at our own head, which is our +// push having landed. It runs only on this exceptional path. func (w *BranchWorker) newBranchMovedDuringPush( - rootBranch plumbing.ReferenceName, rootHash plumbing.Hash, auth []gitclient.Option, -) bool { - advertisement, err := advertiseRemoteBranchFn(w.repo.URL, plumbing.NewBranchReferenceName(w.Branch), + ctx context.Context, + repo *gogit.Repository, + rootBranch plumbing.ReferenceName, + rootHash plumbing.Hash, + auth []gitclient.Option, +) (bool, *PushOutcome) { + advertisement, err := advertiseRemoteBranchFn(ctx, w.repo.URL, plumbing.NewBranchReferenceName(w.Branch), rootBranch.Short(), auth) if err != nil { w.invalidateBase("remote-state probe failed") - return false + return false, nil } if !advertisement.branch.IsZero() { + if landed := landedPush(repo, advertisement.branch, rootHash); landed != nil { + return false, landed + } w.Log.Info("The write branch was created by somebody else during our push; replaying onto it", "branch", w.Branch) - return true + return true, nil } - return advertisement.parent != rootHash + return advertisement.parent != rootHash, nil } // advertisedRootHash reads the remote's own answer out of a failed push, when the push got one. @@ -2419,29 +2355,6 @@ func advertisedRootHash(err error, rootBranch plumbing.ReferenceName) (plumbing. return moved.Advertised, true } -func (w *BranchWorker) rebuildPendingWrites( - repo *gogit.Repository, - pendingWrites []PendingWrite, -) (plumbing.ReferenceName, plumbing.Hash, error) { - baseBranch, baseHash, err := w.ensureWriteBranch(repo) - if err != nil { - return "", plumbing.ZeroHash, err - } - - // A replay re-PLANS against the rebased worktree, so it can decide deletions the first apply - // never made — under whatever policy was in force when the write was planned. Re-check the - // policy before that happens, so an operator who tightened it in the meantime is obeyed. - if err := w.tightenPendingPruneModes(w.ctx, pendingWrites); err != nil { - return "", plumbing.ZeroHash, err - } - - if _, err := w.executePendingWrites(w.ctx, repo, pendingWrites); err != nil { - return "", plumbing.ZeroHash, fmt.Errorf("execute replay pending writes: %w", err) - } - - return baseBranch, baseHash, nil -} - // refreshRemoteAndRebuildPendingWrites moves the local checkout to the current remote tip, then // replays retained pending writes on top of it without pushing, so whatever judges the tree next // judges the newest remote one instead of a stale local clone. @@ -2485,28 +2398,8 @@ func (w *BranchWorker) refreshRemoteAndRebuildPendingWrites( return fmt.Errorf("open repository: %w", err) } - // Marked BEFORE the reset: see the note at the other reset site. A reset that fails with the - // branch ref already moved leaves these writes' commits unreachable while worktreeDirty stays - // false, which is precisely the state no other flag describes. - w.markReplayRequired() w.recordFetch(reason) - pullReport, err := w.syncToRemote(ctx, repo, auth) - if err != nil { - w.invalidateBase("sync before replay failed") - return fmt.Errorf("sync remote before replay: %w", err) - } - w.updateBranchMetadataFromPullReport(pullReport) - - rootBranch, rootHash, err := w.rebuildPendingWrites(repo, pendingWrites) - if err != nil { - w.invalidateBase("rebuild before replay failed") - return fmt.Errorf("rebuild pending writes: %w", err) - } - w.pushCycleRootBranch = rootBranch - w.pushCycleRootHash = rootHash - w.pushCycleRootGen = w.baseParentGen.Load() - w.markReplayComplete() - return nil + return w.replayOntoRemote(ctx, repo, pendingWrites, auth) } // tightenPendingPruneModes lowers every retained write's captured prune mode before replay. @@ -2552,8 +2445,16 @@ func (w *BranchWorker) tightenPendingPruneModes(ctx context.Context, pendingWrit return nil } -// estimateEventSize approximates the serialized YAML size for an event's object. -func (w *BranchWorker) estimateEventSize(ev Event) int64 { +func (w *BranchWorker) estimateEventsSize(events []Event) int64 { + var total int64 + for i := range events { + total += eventPayloadSize(&events[i]) + } + return total +} + +// estimateObjectSize approximates the serialized YAML size for an event's object. +func estimateObjectSize(ev *Event) int64 { if ev.Object == nil { return 0 } @@ -2563,14 +2464,6 @@ func (w *BranchWorker) estimateEventSize(ev Event) int64 { return 0 } -func (w *BranchWorker) estimateEventsSize(events []Event) int64 { - var total int64 - for _, event := range events { - total += w.estimateEventSize(event) - } - return total -} - func (w *BranchWorker) getGitTarget( ctx context.Context, targetName string, @@ -2823,10 +2716,10 @@ const ( // Queue-drop kinds. The set covers every item that can be refused by a full queue. const ( - queueDropWrite = "write" - queueDropAttach = "attach" - queueDropResync = "resync" - queueDropRefresh = "refresh" + queueRefusalWrite = "write" + queueRefusalAttach = "attach" + queueRefusalResync = "resync" + queueRefusalRefresh = "refresh" ) // commitLabels is the {author_kind, message_source} pair one commit is counted under. @@ -2851,7 +2744,8 @@ func (w *BranchWorker) publishCommitsForPush(pendingWrites []PendingWrite) { } counts := map[commitLabels]int64{} for _, pendingWrite := range pendingWrites { - if !pendingWrite.createdCommit() { + // A write the last replay refused was not pushed: the loop settles it as a refusal. + if !pendingWrite.createdCommit() || pendingWrite.replayRefusal != nil { continue } counts[commitLabels{ @@ -2933,7 +2827,8 @@ func (w *BranchWorker) providerAttrs(extra ...attribute.KeyValue) []attribute.Ke } // recordCommitFailure counts one window or request that died between routing and pushing: its -// events are lost until the next resync re-derives them. +// events are lost until the next resync re-derives them. An unreachable remote is not one: the +// decided write stays pending for the publication retry. // // `refused` and `error` need different people. A refusal is a Git path a human has to fix and will // not clear on its own; an error may be transient. @@ -3012,19 +2907,20 @@ func (w *BranchWorker) recordWindowClosed(window *openWindow, reason windowFinal } } -// recordQueueDrop counts one item the queue was too full to accept. +// recordQueueRefusal counts one item refused at enqueue: the queue was full, intake was paused, or the +// worker was stopping. // -// Every increment is work thrown away: a live write is recovered only by the next resync, and an -// attach by the controller's next poll. -func (w *BranchWorker) recordQueueDrop(kind string) { - if telemetry.GitQueueDropsTotal == nil { +// The producer keeps a refused item and offers it again: the watch delivers a live write again from +// its unadvanced cursor, and the controller re-sends an attach on its next poll. +func (w *BranchWorker) recordQueueRefusal(kind string) { + if telemetry.GitQueueRefusalsTotal == nil { return } ctx := w.ctx if ctx == nil { ctx = context.Background() } - telemetry.GitQueueDropsTotal.Add(ctx, 1, + telemetry.GitQueueRefusalsTotal.Add(ctx, 1, metric.WithAttributes(w.providerAttrs(attribute.String("kind", kind))...)) } @@ -3166,7 +3062,7 @@ func (w *BranchWorker) publishObservation(observed RemoteObservation) { } // syncWithRemote fetches latest changes from remote and resets onto them. It is the -// no-retained-writes half of a refresh; resync_flush.go and invalidateAndRefresh are its callers. +// no-retained-writes half of materialize. // // reason is a parameter for the same purpose it is on refreshRemoteAndRebuildPendingWrites: the // caller knows why it is reading the remote, and a hard-wired constant here would file somebody @@ -3280,7 +3176,7 @@ func (w *BranchWorker) updateBranchMetadataFromPullReport(report *PullReport) { // refuses a branch created after the advertisement. So the cost of being wrong is one // rejection, not a bad write. w.setBaseTrusted(true) - w.markWorktreeClean() + w.markCheckoutReset() if report.ExistsOnRemote { w.clearNewBranchParent() } diff --git a/internal/git/branch_worker_base_trust_test.go b/internal/git/branch_worker_base_trust_test.go index bcb9bd21..37800fd9 100644 --- a/internal/git/branch_worker_base_trust_test.go +++ b/internal/git/branch_worker_base_trust_test.go @@ -109,7 +109,7 @@ func TestBaseTrust_AResetAlwaysClearsTheDirtyFlag(t *testing.T) { // grant trust. func TestBaseTrust_FetchingTheRemoteHashDoesNotGrantTrust(t *testing.T) { f := newLedgerFixture(t, "trust-hash-fetch", true) - f.commit(false, "mine") + f.commit("mine") f.contend("OUTSIDE.md", "from-another-writer\n") repo, err := gogit.PlainOpen(f.worker.repoPath()) @@ -131,7 +131,7 @@ func TestBaseTrust_FetchingTheRemoteHashDoesNotGrantTrust(t *testing.T) { // died mid-upload leaves the remote in a state nobody observed. func TestBaseTrust_LostOnEveryPushFailure(t *testing.T) { f := newLedgerFixture(t, "trust-push-fail", true) - f.commit(false, "mine") + f.commit("mine") originalPush := pushAtomicFn pushAtomicFn = func( @@ -163,7 +163,7 @@ func TestBaseTrust_SuccessfulPushDoesNotLaunderADirtyWorktree(t *testing.T) { f := newLedgerFixture(t, "trust-dirty-survives-push", true) // 1. Write A commits and is retained. - f.commit(false, "write-a") + f.commit("write-a") // 2. Write B fails part-way through executePendingWrites. Its window is dropped; A stays. f.worker.markWorktreeDirty("write B failed part-way") @@ -194,7 +194,7 @@ func TestBaseTrust_LostWhenSomebodySaysTheRemoteMoved(t *testing.T) { } defer func() { syncToRemoteFn = original }() - f.commit(false, "retained") + f.commit("retained") f.worker.baseTrustedState.Store(true) err := f.worker.refreshRemoteAndRebuildPendingWrites(f.worker.ctx, f.pending, fetchReasonForcedRecheck) diff --git a/internal/git/branch_worker_credread_test.go b/internal/git/branch_worker_credread_test.go index 2c3effbd..0880fb3c 100644 --- a/internal/git/branch_worker_credread_test.go +++ b/internal/git/branch_worker_credread_test.go @@ -25,7 +25,7 @@ import ( // TestCommitPendingWrites_ResolvesCredentialsOncePerPushCycle proves the branch worker does not // re-read the Git credentials Secret for every commit in a push cycle. Only the first commit -// (hasPendingCommits=false) touches the remote through PrepareBranch and needs auth; later commits +// (the checkout holds no retained write yet) touches the remote through PrepareBranch and needs auth; later commits // build on the local repo and must not issue another Secret GET — with the Secret cache disabled // that would be a wasted API round-trip per commit. See docs/rbac.md §5. func TestCommitPendingWrites_ResolvesCredentialsOncePerPushCycle(t *testing.T) { @@ -88,14 +88,14 @@ func TestCommitPendingWrites_ResolvesCredentialsOncePerPushCycle(t *testing.T) { // First commit of the cycle: fetches the remote tip, so it resolves credentials exactly once. firstWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{createTestEvent(t, "pod-a")}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*firstWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*firstWrite})) require.Equal(t, int64(1), credentialReads.Load(), "first commit resolves credentials once for the remote fetch") - // Second commit in the same cycle (hasPendingCommits=true): local-only, no credential read. + // Second commit in the same cycle (the first is retained): local-only, no credential read. secondWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{createTestEvent(t, "pod-b")}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*secondWrite}, true)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*secondWrite})) assert.Equal(t, int64(1), credentialReads.Load(), "a later commit in the same push cycle must not re-read the credentials Secret") } diff --git a/internal/git/branch_worker_empty_repo_test.go b/internal/git/branch_worker_empty_repo_test.go index 36bc17fd..383e0d4a 100644 --- a/internal/git/branch_worker_empty_repo_test.go +++ b/internal/git/branch_worker_empty_repo_test.go @@ -228,13 +228,15 @@ func TestBranchWorker_ADanglingRemoteHeadIsMissingNotUnborn(t *testing.T) { assert.True(t, f.ref("feature").IsZero(), "no orphan") assert.True(t, f.loop.lastPushAt.IsZero(), "nothing was pushed") - // §4.3 test 3: repairing HEAD is enough. + // §4.3 test 3: repairing HEAD is enough, and the held write lands first. gitIn(t, f.repoDir, "symbolic-ref", "HEAD", "refs/heads/main") pastProbeDeadline(f.worker) liveWrite(f.loop, "cm2") tip := f.ref("feature") require.False(t, tip.IsZero()) - assert.Equal(t, []plumbing.Hash{f.ref("main")}, f.parentOf(tip)) + held := f.parentOf(tip) + require.Len(t, held, 1) + assert.Equal(t, []plumbing.Hash{f.ref("main")}, f.parentOf(held[0])) } // §4.3 test 2. A detached HEAD is resolved the way go-git resolves it, which this pins: master when diff --git a/internal/git/branch_worker_fetch_metric_test.go b/internal/git/branch_worker_fetch_metric_test.go index 4c50a61c..25fa2505 100644 --- a/internal/git/branch_worker_fetch_metric_test.go +++ b/internal/git/branch_worker_fetch_metric_test.go @@ -72,9 +72,9 @@ func TestGitFetchesTotal_SteadyStatePublicationDoesNotFetch(t *testing.T) { // And still once per CYCLE rather than per commit, which is the property the guard must not // have broken on its way to becoming conditional. - f.commit(false, "fourth") - f.commit(true, "fifth") - f.commit(true, "sixth") + f.commit("fourth") + f.commit("fifth") + f.commit("sixth") f.push() assert.Equal(t, int64(1), fetchCount(t, reader, f.worker, fetchReasonPublication)) @@ -115,7 +115,7 @@ func TestGitFetchesTotal_ContentionIsCountedSeparately(t *testing.T) { f := newLedgerFixture(t, "metric-contention", true) f.publish("prime") - f.commit(false, "mine") + f.commit("mine") f.contend("OUTSIDE.md", "from-another-writer\n") f.push() @@ -140,7 +140,7 @@ func TestGitFetchesTotal_PushFailureProbeIsNotContention(t *testing.T) { f := newLedgerFixture(t, "metric-probe", true) f.publish("prime") - f.commit(false, "mine") + f.commit("mine") // A push that dies before the remote says anything, on every attempt. original := pushAtomicFn @@ -168,7 +168,7 @@ func TestGitFetchesTotal_ProbeThenResetWhenTheRemoteDidMove(t *testing.T) { f := newLedgerFixture(t, "metric-probe-moved", true) f.publish("prime") - f.commit(false, "mine") + f.commit("mine") original := pushAtomicFn pushes := 0 @@ -212,7 +212,7 @@ func TestGitFetchesTotal_ForcedRecheckAndBootstrap(t *testing.T) { assert.Equal(t, int64(1), fetchCount(t, reader, f.worker, fetchReasonForcedRecheck), "a forced recheck with nothing retained fetches through syncWithRemote") - f.commit(false, "retained") + f.commit("retained") require.NoError(t, f.worker.refreshRemoteAndRebuildPendingWrites(f.worker.ctx, f.pending, fetchReasonForcedRecheck)) assert.Equal(t, int64(2), fetchCount(t, reader, f.worker, fetchReasonForcedRecheck), "and with retained writes it fetches through refreshRemoteAndRebuildPendingWrites") @@ -235,7 +235,7 @@ func TestGitFetchesTotal_IdleTargetNeverFetches(t *testing.T) { for range idleLedgerWindows { loop.finalizeOpenWindow() loop.pushPending() - loop.applyDeferredHeals() + loop.endWake(0) } for _, reason := range []string{ @@ -311,6 +311,7 @@ func TestGitFetchesTotal_ResyncWithRetainedWritesStillReadsTheRemote(t *testing. Result: make(chan ResyncResult, 1), } loop.applyResync(req) + loop.endWake(0) require.NoError(t, (<-req.Result).Err) assert.Greater(t, fetchCount(t, reader, f.worker, fetchReasonForcedRecheck), before, @@ -347,7 +348,7 @@ func TestGitFetchesTotal_RecoveryIsOneSeriesWhetherOrNotWritesAreRetained(t *tes forcedBefore := fetchCount(t, reader, f.worker, fetchReasonForcedRecheck) f.worker.markWorktreeDirty("a write failed part-way") - require.NoError(t, loop.recoverRetainedWrites()) + require.NoError(t, loop.materialize()) assert.Equal(t, int64(1), fetchCount(t, reader, f.worker, fetchReasonRecovery), "a recovery is a recovery whether or not a push happened to be in cooldown") @@ -360,8 +361,8 @@ func TestGitFetchesTotal_RecoveryIsOneSeriesWhetherOrNotWritesAreRetained(t *tes // resync guarantee, and it is worth pinning separately because the fetch is not where you would // look for it. // -// prepareBaseForResync owns the fetch on this path. With nothing retained, invalidateAndRefresh -// drops base trust and then calls syncWithRemote directly rather than leaving the reset to the +// The resync's own materialize pass owns the fetch on this path. With nothing retained, it +// calls syncWithRemote directly, which drops base trust first, rather than leaving the reset to the // commit that follows, which is what makes the reason `forced_recheck` instead of `publication`: // the fetch belongs to the snapshot that asked for it, not to a live publication. Ledger row 10 // measures the same operation as requests on the wire; this pins which series it lands in. @@ -390,6 +391,7 @@ func TestGitFetchesTotal_ResyncWithNoRetainedWritesAlsoReadsTheRemote(t *testing Result: make(chan ResyncResult, 1), } loop.applyResync(req) + loop.endWake(0) require.NoError(t, (<-req.Result).Err) assert.Equal(t, forcedBefore+1, fetchCount(t, reader, f.worker, fetchReasonForcedRecheck), diff --git a/internal/git/branch_worker_metrics_test.go b/internal/git/branch_worker_metrics_test.go index 011ac4a0..651eabd8 100644 --- a/internal/git/branch_worker_metrics_test.go +++ b/internal/git/branch_worker_metrics_test.go @@ -21,7 +21,7 @@ import ( const ( commitsTotalMetric = "gitopsreverser_git_commits_total" queueDepthMetric = "gitopsreverser_git_queue_depth" - queueDropsMetric = "gitopsreverser_git_queue_drops_total" + queueRefusalsMetric = "gitopsreverser_git_queue_refusals_total" pushesTotalMetric = "gitopsreverser_git_pushes_total" pushRetriesMetric = "gitopsreverser_git_push_retries_total" pushDurationMetric = "gitopsreverser_git_push_duration_seconds" @@ -211,8 +211,8 @@ func TestEnqueueRequest_FullQueueCountsTheDrop(t *testing.T) { "the second request must be refused by the full queue") labels := workerLabels() - labels["kind"] = queueDropWrite - drops, ok := telemetry.CollectInt64Sum(reader, queueDropsMetric, labels) + labels["kind"] = queueRefusalWrite + drops, ok := telemetry.CollectInt64Sum(reader, queueRefusalsMetric, labels) require.True(t, ok, "a dropped write must be counted") assert.Equal(t, int64(1), drops) assert.Equal(t, int64(1), w.inflightItems.Load(), "the refused item must not stay in the inflight count") @@ -230,7 +230,7 @@ func TestHandleShutdown_DrainsBufferedItemsToZeroDepth(t *testing.T) { loop := newBranchWorkerEventLoop(w, time.Second) loop.handleShutdown() - loop.syncUnpushedWorkFlag() + loop.publishLoopState(0) assert.Equal(t, int64(0), w.inflightItems.Load(), "buffered items must be drained from the inflight count") assert.Equal(t, int64(0), w.queueDepth(), "a drained, exiting worker must read depth 0") @@ -456,7 +456,7 @@ func TestRecordPushOutcome_CountsTheCycleAndItsDuration(t *testing.T) { // A commit that fails takes its whole window with it, and that has to be countable. // // This was the largest remaining hole in the pipeline census. The failure happens AFTER routing and -// BEFORE pushing, so neither git_queue_drops_total nor git_pushes_total sees it: the mirror falls +// BEFORE pushing, so neither git_queue_refusals_total nor git_pushes_total sees it: the mirror falls // behind for every object in the window until the next resync re-derives them, and the only trace // was a log line, or for a refusal a GitTarget condition nobody alerts on. func TestRecordCommitFailure_SeparatesRefusalFromWriteFault(t *testing.T) { diff --git a/internal/git/branch_worker_new_branch_test.go b/internal/git/branch_worker_new_branch_test.go index c40357a5..47f24770 100644 --- a/internal/git/branch_worker_new_branch_test.go +++ b/internal/git/branch_worker_new_branch_test.go @@ -151,7 +151,7 @@ func (f *newBranchFixture) noopResync(loop *branchWorkerEventLoop) { GitTargetNamespace: "default", Result: make(chan ResyncResult, 1), } - loop.handleQueueItem(WorkItem{Resync: req}) + loop.handleQueueItem(resyncItem(req)) result := <-req.Result require.NoError(f.t, result.Err) require.Zero(f.t, result.Stats.Created+result.Stats.Updated+result.Stats.Deleted, "the resync must be a no-op") @@ -295,7 +295,7 @@ func TestBranchWorker_ResyncOnAbsentBranchDiffsAgainstTheMovedParent(t *testing. GitTargetNamespace: "default", Result: make(chan ResyncResult, 1), } - loop.handleQueueItem(WorkItem{Resync: req}) + loop.handleQueueItem(resyncItem(req)) result := <-req.Result require.NoError(t, result.Err) assert.Equal(t, 1, result.Stats.Deleted, "the document main gained is not in the cluster") diff --git a/internal/git/branch_worker_parent_branch_test.go b/internal/git/branch_worker_parent_branch_test.go index e5e0617c..562e3f1d 100644 --- a/internal/git/branch_worker_parent_branch_test.go +++ b/internal/git/branch_worker_parent_branch_test.go @@ -122,8 +122,8 @@ func TestBranchWorker_NewBranchStartsFromTheConfiguredParent(t *testing.T) { } // TestBranchWorker_MissingConfiguredParentWritesNothingUntilItExists: a configured parent the -// remote does not carry refuses the write, starts no orphan branch, and is reported; once it is -// pushed the next write creates the branch on it. +// remote does not carry holds the write back, starts no orphan branch, and is reported; once it is +// pushed the next write creates the branch on it, carrying the held write first. func TestBranchWorker_MissingConfiguredParentWritesNothingUntilItExists(t *testing.T) { f := newNewBranchFixture(t, nil) f.worker.SetParentBranch("release") @@ -132,7 +132,8 @@ func TestBranchWorker_MissingConfiguredParentWritesNothingUntilItExists(t *testi liveWrite(loop, "cm1") - assert.Empty(t, loop.pendingWrites, "nothing was committed") + require.Len(t, loop.pendingWrites, 1, "the write is kept") + assert.False(t, loop.pendingWrites[0].committedOnce, "but nothing was committed") assert.True(t, loop.lastPushAt.IsZero(), "and nothing was pushed") _, onRemote := f.featureOnRemote() assert.False(t, onRemote, "no orphan branch") @@ -146,7 +147,7 @@ func TestBranchWorker_MissingConfiguredParentWritesNothingUntilItExists(t *testi tip, onRemote := f.featureOnRemote() require.True(t, onRemote, "it recovers once the parent exists") - assert.Equal(t, r1, f.commitParent(tip)) + assert.Equal(t, r1, f.commitParent(f.commitParent(tip)), "the held cm1, then cm2, on the parent's tip") observed, _ = f.worker.LastRemoteObservation() assert.Empty(t, observed.ParentState, "the write branch exists, so no parent is reported") } @@ -380,7 +381,8 @@ func TestBranchWorker_ObservesTheParentOfAnAbsentBranch(t *testing.T) { assert.Equal(t, ParentMissing, o.ParentState) r1 := f.pushToRelease("RELEASE.md", "release\n") - require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + // The worker holds the write, so the bootstrap path would read its own clone: fetch directly. + require.NoError(t, f.worker.syncWithRemote(f.worker.ctx, fetchReasonForcedRecheck)) o = observed(f) assert.Equal(t, "release", o.ParentBranch) assert.Equal(t, r1.String(), o.ParentCommit) diff --git a/internal/git/branch_worker_queue_depth_test.go b/internal/git/branch_worker_queue_depth_test.go index f4ba4e0b..0cb90781 100644 --- a/internal/git/branch_worker_queue_depth_test.go +++ b/internal/git/branch_worker_queue_depth_test.go @@ -24,10 +24,10 @@ import ( // queueDepth() sums the in-flight count and the retained-work flag, so an item that opened a commit // window is covered by the first until the second is published. Releasing the count first leaves a // window where neither covers it, and the loop does real work inside that window -// (serviceCommitRequests, applyDeferredHeals) before the flag is synced. A scrape landing there -// reports a drained queue for a worker holding an uncommitted event, which can satisfy a drain gate -// early. -func TestReleaseHandledItem_PublishesRetainedWorkBeforeReleasingInflight(t *testing.T) { +// (serviceCommitRequests, applyDeferredHeals, advance) before the flag is published. A scrape +// landing there reports a drained queue for a worker holding an uncommitted event, which can satisfy +// a drain gate early. The loop releases the count only after the wake that handled the item. +func TestHandledItem_PublishesRetainedWorkBeforeReleasingInflight(t *testing.T) { w := newMetricsTestWorker() w.inflightItems.Store(1) @@ -36,7 +36,8 @@ func TestReleaseHandledItem_PublishesRetainedWorkBeforeReleasingInflight(t *test loop.openWindow = &openWindow{} require.Positive(t, w.queueDepth(), "the in-flight count covers the item before the release") - loop.releaseHandledItem() + loop.endWake(0) + w.inflightItems.Add(-1) // as run does, after the wake assert.Positive(t, w.queueDepth(), "the retained-work flag must cover it after the release") } diff --git a/internal/git/branch_worker_split_test.go b/internal/git/branch_worker_split_test.go index 3bcb70cf..ca5d341c 100644 --- a/internal/git/branch_worker_split_test.go +++ b/internal/git/branch_worker_split_test.go @@ -205,7 +205,7 @@ func TestCommitGroups_DoesNotPush(t *testing.T) { } pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, events) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) // Remote must be untouched after commitPendingWrites; only pushPendingCommits // publishes work. @@ -247,7 +247,7 @@ func TestCommitGroups_AccumulatesAcrossCalls(t *testing.T) { []Event{configMapEvent("first", "alice", "team-a")}, ) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*firstPendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*firstPendingWrite})) rootAfterFirst := worker.pushCycleRootHash secondPendingWrite, err := worker.buildGroupedPendingWrite( @@ -255,7 +255,7 @@ func TestCommitGroups_AccumulatesAcrossCalls(t *testing.T) { []Event{configMapEvent("second", "bob", "team-b")}, ) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*secondPendingWrite}, true)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*secondPendingWrite})) assert.Equal(t, rootAfterFirst, worker.pushCycleRootHash, "hasUnpushedCommits=true must preserve the rootHash from the first commit") @@ -293,11 +293,11 @@ func TestPushPendingCommits_FlushesAccumulated(t *testing.T) { pendingWrite1, err := worker.buildGroupedPendingWrite(worker.ctx, events1) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite1}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite1})) pendingWrite2, err := worker.buildGroupedPendingWrite(worker.ctx, events2) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite2}, true)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite2})) pending1, err := worker.buildGroupedPendingWrite(worker.ctx, events1) require.NoError(t, err) @@ -335,7 +335,7 @@ func TestPushPendingCommits_ReplaysOnConflict(t *testing.T) { events := []Event{configMapEvent("from-operator", "alice", "team-a")} retainedPendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, events) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*retainedPendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*retainedPendingWrite})) pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, events) require.NoError(t, err) @@ -378,7 +378,7 @@ func TestRefreshRemoteAndRebuildPendingWrites_ReplaysWithoutPushing(t *testing.T []Event{configMapEvent("from-operator", "alice", "team-a")}, ) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) pendingWrites := []PendingWrite{*pendingWrite} otherPath := filepath.Join(t.TempDir(), "other") @@ -422,10 +422,10 @@ func TestPushPendingCommits_ReplayPreservesPendingWriteCommitOrder(t *testing.T) } pendingWrites := make([]PendingWrite, 0, len(events)) - for i, batch := range events { + for _, batch := range events { pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, batch) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, i > 0)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) pendingWrites = append(pendingWrites, *pendingWrite) } @@ -523,7 +523,7 @@ func TestBranchWorker_Replay_UsesResolvedMetadata_GitTargetDeletedMidBurst(t *te require.NoError(t, err) targetMetadata := pendingWrite.findTargetMetadata("secret-target", "default") require.NotNil(t, targetMetadata.EncryptionConfig, "resolved encryption must be retained on the pending write") - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.Client.Delete(worker.ctx, &configv1alpha3.GitTarget{ ObjectMeta: metav1.ObjectMeta{Name: "secret-target", Namespace: "default"}, @@ -572,7 +572,7 @@ func TestBranchWorker_TransientPushFailure_RetriesSameLocalCommits(t *testing.T) []Event{configMapEvent("transient", "alice", "team-a")}, ) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) localRepo, err := git.PlainOpen(worker.repoPath()) require.NoError(t, err) @@ -651,7 +651,7 @@ func TestBranchWorker_PushFollowedByFetchFailure_TreatsAsTransient(t *testing.T) []Event{configMapEvent("fetch-failure", "alice", "team-a")}, ) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) localRepo, err := git.PlainOpen(worker.repoPath()) require.NoError(t, err) @@ -726,7 +726,7 @@ func TestBranchWorker_AtomicAndGroupedInterleaved_PreservesArrivalOrder(t *testi []Event{configMapEvent("grouped-first", "alice", "team-a")}, ) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*groupedFirst}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*groupedFirst})) atomicRequest := &WriteRequest{ Events: []Event{configMapEvent("atomic-second", "reconciler", "team-a")}, @@ -735,14 +735,14 @@ func TestBranchWorker_AtomicAndGroupedInterleaved_PreservesArrivalOrder(t *testi } atomicSecond, err := worker.buildAtomicPendingWrite(worker.ctx, atomicRequest) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*atomicSecond}, true)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*atomicSecond})) groupedThird, err := worker.buildGroupedPendingWrite( worker.ctx, []Event{configMapEvent("grouped-third", "bob", "team-a")}, ) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*groupedThird}, true)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*groupedThird})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*groupedFirst, *atomicSecond, *groupedThird})) @@ -778,7 +778,7 @@ func TestBranchWorker_Replay_DropsUnitsThatBecomeNoOpAgainstNewRemoteTree(t *tes event := configMapEvent("already-applied", "alice", "team-a") pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) localRepoPath := worker.repoPath() filePath := generateFilePath(event.Identifier, itypes.SensitiveResourcePolicy{}) @@ -1091,7 +1091,7 @@ func TestEventLoop_DeferredEventCommitsAndAtomicDuringCooldownPushTogether(t *te }}) require.Len(t, loop.pendingWrites, 2, "deferred event commits during cooldown are retained as local commits, not lost") - loop.syncUnpushedWorkFlag() + loop.publishLoopState(0) loop.handleQueueItem(WorkItem{Request: &WriteRequest{ Events: []Event{configMapEvent("snapshot-only", "reconciler", "")}, @@ -1124,7 +1124,7 @@ func TestEventLoop_AtomicPushFailure_DoesNotAdvanceCooldownOrLosePendingWrite(t pendingWrite, err := worker.buildAtomicPendingWrite(worker.ctx, request) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) loop := newBranchWorkerEventLoop(worker, time.Second) loop.pendingWrites = []PendingWrite{*pendingWrite} @@ -1170,7 +1170,7 @@ func TestResync_WorkerAppliesMarkAndSweepAndCommits(t *testing.T) { GitTargetNamespace: "default", Result: make(chan ResyncResult, 1), } - loop.handleQueueItem(WorkItem{Resync: req}) + loop.handleQueueItem(resyncItem(req)) result := <-req.Result require.NoError(t, result.Err) @@ -1206,7 +1206,7 @@ func TestResync_WorkerNoopDoesNotRetainOrPush(t *testing.T) { GitTargetNamespace: "default", Result: make(chan ResyncResult, 1), } - loop.handleQueueItem(WorkItem{Resync: req}) + loop.handleQueueItem(resyncItem(req)) result := <-req.Result require.NoError(t, result.Err) @@ -1239,7 +1239,7 @@ func TestResync_WorkerEmptyDesiredSweepsManagedResource(t *testing.T) { GitTargetNamespace: "default", Result: make(chan ResyncResult, 1), } - loop.handleQueueItem(WorkItem{Resync: req}) + loop.handleQueueItem(resyncItem(req)) result := <-req.Result require.NoError(t, result.Err) diff --git a/internal/git/branch_worker_test.go b/internal/git/branch_worker_test.go index 347e490e..23505a15 100644 --- a/internal/git/branch_worker_test.go +++ b/internal/git/branch_worker_test.go @@ -635,7 +635,7 @@ func TestBranchWorker_CommitAndPushRequest_PreparesRepositoryBeforeFirstWrite(t pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, request.Events) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) localRepoPath := worker.repoPath() @@ -753,7 +753,7 @@ func TestBranchWorker_CommitAndPushRequest_NewBranchStartsFromLatestMain(t *test pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, request.Events) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) remoteMainRef, err := serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) @@ -886,7 +886,7 @@ func TestBranchWorker_CommitAndPushRequest_UsesProviderCommitterAndTargetMessage pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, request.Events) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) remoteHeadRef, err := serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) @@ -1008,7 +1008,7 @@ func TestBranchWorker_CommitAndPushRequest_UsesBatchTemplateForAtomicRequest(t * pendingWrite, err := worker.buildAtomicPendingWrite(worker.ctx, request) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) remoteHeadRef, err := serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) @@ -1114,7 +1114,7 @@ func TestBranchWorker_CommitAndPushRequest_SignsCommitWhenConfigured(t *testing. pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, request.Events) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) remoteHeadRef, err := serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) diff --git a/internal/git/commit_executor.go b/internal/git/commit_executor.go index 18699509..84d12b74 100644 --- a/internal/git/commit_executor.go +++ b/internal/git/commit_executor.go @@ -4,6 +4,7 @@ package git import ( "context" + "errors" "fmt" "path/filepath" "time" @@ -11,33 +12,95 @@ import ( gogit "github.com/go-git/go-git/v6" "github.com/go-git/go-git/v6/plumbing" "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" ) +// executePendingWrites commits each write in order on top of the checkout, and stops at the first +// that fails, undoing the whole batch. It is the first commit of a decided write; a replay of +// writes committed before goes through replayPendingWrites instead. func (w *BranchWorker) executePendingWrites( ctx context.Context, repo *gogit.Repository, pendingWrites []PendingWrite, ) (int, error) { + commitsCreated, _, err := w.runPendingWrites(ctx, repo, pendingWrites, false) + return commitsCreated, err +} + +// replayPendingWrites commits retained writes again onto a fresh root, and returns how many of them +// the checkout holds afterwards. +// +// A write is accepted against the tree it was first committed on, and the tree it is replayed onto +// can refuse it: somebody else changed its folder in the meantime. That refusal is a final answer +// about that one write, and the writes after it have no reason to wait on it, so the replay skips +// it and goes on. The skipped write is stamped with its refusal (replayRefusal), and the loop +// settles it the way it settles a refusal at first commit. Any other failure says nothing final +// about the write it hit, so it abandons the whole replay, as before, and nothing is stamped. +func (w *BranchWorker) replayPendingWrites( + ctx context.Context, + repo *gogit.Repository, + pendingWrites []PendingWrite, +) (int, error) { + // A stamp describes the checkout the last completed replay produced, so a replay that starts + // clears them all, and only one that completes leaves any behind. + for i := range pendingWrites { + pendingWrites[i].replayRefusal = nil + } + _, refusals, err := w.runPendingWrites(ctx, repo, pendingWrites, true) + if err != nil { + return 0, err + } + for i, refusal := range refusals { + pendingWrites[i].replayRefusal = refusal + } + return len(pendingWrites) - len(refusals), nil +} + +// runPendingWrites is the loop both execute the writes with. With skipRefused, a write the +// acceptance gate refuses is undone on its own and recorded in the returned map by index, instead +// of failing the batch. +func (w *BranchWorker) runPendingWrites( + ctx context.Context, + repo *gogit.Repository, + pendingWrites []PendingWrite, + skipRefused bool, +) (int, map[int]error, error) { worktree, err := repo.Worktree() if err != nil { - return 0, fmt.Errorf("failed to get worktree: %w", err) + return 0, nil, fmt.Errorf("failed to get worktree: %w", err) } + // Where the batch starts, so a write that fails part-way can be undone locally: see + // restoreWorktree. An unborn branch has no commit to return to, and falls back to a reset. + start := headOr(repo, plumbing.ZeroHash) + commitsCreated := 0 + refusals := map[int]error{} // Index over the slice so the per-write commit hash is written back onto the // caller's PendingWrite (§6.5): a CommitRequest riding a write resolves to this // SHA on push, and a rebase-replay (which re-runs this loop on the retained // writes) refreshes it to the post-rebase hash. for i := range pendingWrites { + // Where this write starts: a refused write is undone back to here, keeping the writes + // before it. + before := headOr(repo, start) created, hash, source, err := w.executePendingWrite(ctx, repo, worktree, pendingWrites[i]) + if err != nil && skipRefused && undoRefusedWrite(repo, before, err) { + refusals[i] = err + continue + } if err != nil { // A write can fail after it has staged part of its change, so the worktree may now - // hold content nobody asked for. Only a reset clears this: a later successful push - // says where the remote is and says nothing about these leftovers, and a cycle that - // planned on top of them would commit them under an unrelated author. - w.markWorktreeDirty("execute pending write failed") - return commitsCreated, err + // hold content nobody asked for, and a cycle that planned on top of it would commit it + // under an unrelated author. Returning to where the batch started clears it without a + // round trip; only if that fails is the worktree left for a reset from the remote. + if restoreErr := restoreWorktree(repo, start); restoreErr != nil { + w.Log.V(1).Info("Could not undo a failed write locally", "error", restoreErr.Error()) + w.markWorktreeDirty("execute pending write failed") + } + return commitsCreated, nil, err } pendingWrites[i].CommitSHA = hash pendingWrites[i].emptyCommitted = pendingWrites[i].mayCommitEmpty() && isEmptyCommit(repo, hash) @@ -48,7 +111,59 @@ func (w *BranchWorker) executePendingWrites( commitsCreated += created } - return commitsCreated, nil + return commitsCreated, refusals, nil +} + +// headOr is the commit HEAD points at, or fallback when it cannot be read. +func headOr(repo *gogit.Repository, fallback plumbing.Hash) plumbing.Hash { + if head, err := repo.Head(); err == nil { + return head.Hash() + } + return fallback +} + +// undoRefusedWrite reports whether err is the acceptance gate refusing a write, and that write was +// undone back to before, the commit it started on. A refusal that cannot be undone fails the batch +// like any other error. +func undoRefusedWrite(repo *gogit.Repository, before plumbing.Hash, err error) bool { + var refused *manifestanalyzer.AcceptanceRefusedError + return errors.As(err, &refused) && restoreWorktree(repo, before) == nil +} + +// windowTarget is the GitTarget a window's write belongs to: the target its events were routed +// to, as the window recorded it when it opened. +func (p PendingWrite) windowTarget() (string, string) { + for i := range p.Events { + if p.Events[i].GitTargetName != "" { + return p.Events[i].GitTargetName, p.Events[i].GitTargetNamespace + } + } + return p.GitTargetName, p.GitTargetNamespace +} + +// restoreWorktree undoes a batch that failed part-way: the branch goes back to the commit the batch +// started on, discarding any commits it made, and the worktree back to that commit's tree, leftovers +// included (discardWorktreeLeftovers). The checkout then holds exactly what it held before the +// batch, which is what checkoutApplied still says, so nothing has to be fetched. +func restoreWorktree(repo *gogit.Repository, start plumbing.Hash) error { + if start.IsZero() { + return errors.New("the branch had no commit to return to") + } + worktree, err := repo.Worktree() + if err != nil { + return fmt.Errorf("get worktree: %w", err) + } + if err := worktree.Reset(&gogit.ResetOptions{Commit: start, Mode: gogit.HardReset}); err != nil { + return fmt.Errorf("reset to %s: %w", start, err) + } + return discardWorktreeLeftovers(repo) +} + +// retained reports whether the loop keeps this write once it is committed: every write does except +// a resync that changed nothing, which is neither retained nor pushed (see Committed). The checkout +// counts the same writes (checkoutApplied), so the two can never disagree. +func (p PendingWrite) retained() bool { + return p.Kind != PendingWriteResync || p.Committed == nil || *p.Committed } func (p PendingWrite) path() string { diff --git a/internal/git/commit_request_attach.go b/internal/git/commit_request_attach.go index 97b65be0..6baf22e6 100644 --- a/internal/git/commit_request_attach.go +++ b/internal/git/commit_request_attach.go @@ -58,7 +58,7 @@ const ( PhaseWaitingForWindow CommitRequestPhase = "WaitingForWindow" // PhaseCollectingWindow is an attached request collecting writes until the window's timers close it. PhaseCollectingWindow CommitRequestPhase = "CollectingWindow" - // PhaseWaitingForPush is a request committed locally and not yet confirmed by the remote. + // PhaseWaitingForPush is a request whose write is decided and not yet confirmed by the remote. PhaseWaitingForPush CommitRequestPhase = "WaitingForPush" ) @@ -88,6 +88,9 @@ type FinalizeResult struct { // Phase is where an UNRESOLVED request stands on the worker; empty once resolved, and empty // before the worker has registered it. Phase CommitRequestPhase + // Held is why a request the worker holds has not reached the remote yet: the worker's + // publication report while it cannot publish, empty otherwise. See PublicationStatus. + Held string } // AttachCommitRequest is the "attach this CommitRequest to the author's window, with its @@ -170,19 +173,17 @@ type pendingCommitRequest struct { // commitEmpty records the message in an empty commit when the request ends with nothing to // commit. Never for a WindowMismatch: that would claim more than happened. commitEmpty bool - // attached is true once this request's message is bound to the open window. - attached bool - // committed is true once the window this request attached to has been finalized into a local - // commit. The request now rides that retained write and only the push can settle it, which is - // why it is `committed` and not `published`: the work exists locally and is nowhere else yet. + // attached is true once this request is bound to a write: its message to the open window, or + // its own empty-commit record. From then on only that write's outcome settles it, and it stays + // set while the write is pending, waiting for the push. // - // The flag exists because the request must stay IDENTIFIABLE while it waits. The controller + // It stays set because the request must stay IDENTIFIABLE while it waits. The controller // re-sends its attach every couple of seconds until it reads an outcome, and forgetting the // request at finalize made that re-send look like a brand-new one: it would register again, // expire against its fresh deadline, and report NoOpenWindow for work that was sitting in - // pendingWrites waiting for the push cooldown — or, worse, attach to the next same-author window + // pending, waiting for the push cooldown, or, worse, attach to the next same-author window // and stamp this request's message onto a commit somebody else authored. - committed bool + attached bool // sawForeignWindow is set when a window was open during this request's wait that it could // not attach to, because the window belonged to a different author or GitTarget. // diff --git a/internal/git/commit_request_attach_loop.go b/internal/git/commit_request_attach_loop.go index eba176b9..ee312315 100644 --- a/internal/git/commit_request_attach_loop.go +++ b/internal/git/commit_request_attach_loop.go @@ -239,7 +239,6 @@ func (l *branchWorkerEventLoop) handleAttachCommitRequest(req *AttachCommitReque // including one another request attached, and wait for a write to open the next. The // registration above is what makes this happen once per request. l.finalizeOpenWindowWithReason(windowFinalizeReasonAttachNext) - l.maybeSchedulePush() return } if l.openWindow.pendingCR == nil { @@ -268,7 +267,7 @@ func (l *branchWorkerEventLoop) serviceCommitRequests() { // waiting reports whether a request is still looking for a window it may attach to at now. func (p *pendingCommitRequest) waiting(now time.Time) bool { - return !p.attached && !p.committed && p.attachDeadline.After(now) + return !p.attached && p.attachDeadline.After(now) } // noteForeignWindow records, on every waiting request, that a window it cannot attach to is open. @@ -329,13 +328,15 @@ func (l *branchWorkerEventLoop) attachWaitingCommitRequests() { // attachToOpenWindow binds a request's message to the open window and replaces the window's timers // with the request's own: the request's maxDuration runs from this attach, and so does its first -// idle interval. A zero maxDuration closes the window right here. +// idle interval. A zero maxDuration closes the window right here. The message is the window's to +// keep from now on, so it moves into the window's charge. func (l *branchWorkerEventLoop) attachToOpenWindow(pcr *pendingCommitRequest) { now := time.Now() l.openWindow.pendingMessage = pcr.message id := pcr.id l.openWindow.pendingCR = &id pcr.attached = true + l.windowBytes += attachCharge(pcr.message) timers := windowTimers{noIdle: pcr.idleTimeout == nil, maxAt: now.Add(pcr.maxDuration)} if pcr.idleTimeout != nil { timers.idle = *pcr.idleTimeout @@ -357,7 +358,7 @@ func (l *branchWorkerEventLoop) attachToOpenWindow(pcr *pendingCommitRequest) { func (l *branchWorkerEventLoop) expireWaitingCommitRequests() { now := time.Now() for id, pcr := range l.pendingCRs { - if pcr.attached || pcr.committed || pcr.attachDeadline.After(now) { + if pcr.attached || pcr.attachDeadline.After(now) { continue } // Which of the two refusals this is depends on whether anything was open that this @@ -387,10 +388,12 @@ func (l *branchWorkerEventLoop) expireWaitingCommitRequests() { } } -// recordCommitRequest commits a request's message with an untouched tree, for a request that ran -// out of time to attach and asked for whenNothingToCommit: CommitEmpty. Every reason no record was -// made is returned, including a write gate (an empty commit is a write too), so the request fails -// instead of reporting a save it did not make. +// recordCommitRequest decides an empty commit recording a request's message, for a request that ran +// out of time to attach and asked for whenNothingToCommit: CommitEmpty. Every reason no record can +// be made is returned, including a write gate (an empty commit is a write too), so the request fails +// instead of reporting a save it did not make. Once decided, the request rides the record write +// like an attached request rides its window: a remote that cannot be reached holds it, and only a +// failure of the commit itself fails it. func (l *branchWorkerEventLoop) recordCommitRequest(pcr *pendingCommitRequest) (bool, error) { pendingWrite, err := l.w.buildRequestRecordWrite(l.w.ctx, pcr) if err != nil { @@ -398,24 +401,9 @@ func (l *branchWorkerEventLoop) recordCommitRequest(pcr *pendingCommitRequest) ( "request", pcr.id.Namespace+"/"+pcr.id.Name) return false, fmt.Errorf("record the message in an empty commit: %w", err) } - // An empty commit is only empty on a clean worktree: a failed write's staged leftovers would - // otherwise ride along in it. - if err := l.recoverRetainedWrites(); err != nil { - l.w.Log.Error(err, "Cannot recover the worktree for the empty commit recording a CommitRequest", - "request", pcr.id.Namespace+"/"+pcr.id.Name) - return false, fmt.Errorf("record the message in an empty commit: %w", err) - } - batch := []PendingWrite{*pendingWrite} - if err := l.w.commitPendingWrites(batch, len(l.pendingWrites) > 0); err != nil { - l.w.Log.Error(err, "The empty commit recording a CommitRequest failed", - "request", pcr.id.Namespace+"/"+pcr.id.Name) - return false, fmt.Errorf("record the message in an empty commit: %w", err) - } - l.pendingWrites = append(l.pendingWrites, batch...) - l.pendingWritesBytes += batch[0].ByteSize - pcr.committed = true - l.w.setCommitRequestPhase(pcr.id, PhaseWaitingForPush) - l.maybeSchedulePush() + // A record whose commit fails for good resolves the request itself, so either way the request + // is no longer the caller's to resolve. + l.decide(*pendingWrite) return true, nil } @@ -468,7 +456,7 @@ func (l *branchWorkerEventLoop) handleWithdrawCommitRequest(req *AttachCommitReq if l.w.hasCommitRequestOutcome(id) { return // already resolved, or already withdrawn. } - if pcr, ok := l.pendingCRs[id]; ok && (pcr.attached || pcr.committed) { + if pcr, ok := l.pendingCRs[id]; ok && pcr.attached { l.w.Log.Info("CommitRequest withdraw ignored: the worker already holds it", "request", id.Namespace+"/"+id.Name) return @@ -498,7 +486,7 @@ func (l *branchWorkerEventLoop) resolveCommitRequest(id commitRequestID, result func (l *branchWorkerEventLoop) rearmAttachTimer() { var earliest time.Time for _, pcr := range l.pendingCRs { - if pcr.attached || pcr.committed { + if pcr.attached { continue } if earliest.IsZero() || pcr.attachDeadline.Before(earliest) { diff --git a/internal/git/commit_request_attach_test.go b/internal/git/commit_request_attach_test.go index 14511838..72be19df 100644 --- a/internal/git/commit_request_attach_test.go +++ b/internal/git/commit_request_attach_test.go @@ -34,7 +34,7 @@ const ( // does after dequeuing an attach work item. func serviceAttach(loop *branchWorkerEventLoop, req *AttachCommitRequest) { loop.handleAttachCommitRequest(req) - loop.serviceCommitRequests() + loop.endWake(0) } // attachReq builds a CurrentOrNext request that waits up to d for a window and, once attached, @@ -65,6 +65,7 @@ func forceDue(loop *branchWorkerEventLoop) { if loop.openWindow != nil && loop.openWindow.pendingCR != nil && *loop.openWindow.pendingCR == id { loop.openWindow.timers.maxAt = past loop.closeOrArmWindow() + loop.endWake(0) } } @@ -220,14 +221,14 @@ func TestAttach_CollectGraceJoinsLaterWindow(t *testing.T) { Events: []Event{configMapTargetEvent("late", "alice", "team-a")}, CommitMode: CommitModePerEvent, }}) - loop.serviceCommitRequests() + loop.endWake(0) require.NotNil(t, loop.openWindow) require.NotNil(t, loop.openWindow.pendingCR, "the opened window must carry the attached request") assert.Equal(t, "bundle save", loop.openWindow.pendingMessage) // Grace elapses → the collected window is finalized as one commit. forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) res, ok := outcome(t, worker) require.True(t, ok) @@ -286,7 +287,7 @@ func TestAttach_AnExpiredWaitNeverTakesALaterWindow(t *testing.T) { Events: []Event{configMapTargetEvent("late", "alice", "team-a")}, CommitMode: CommitModePerEvent, }}) - loop.serviceCommitRequests() + loop.endWake(0) res, ok := outcome(t, worker) require.True(t, ok, "the overdue request must resolve in this pass") @@ -316,13 +317,13 @@ func TestAttach_WritesAfterTheClaimJoinOneCommit(t *testing.T) { Events: []Event{configMapTargetEvent(name, "alice", "team-a")}, CommitMode: CommitModePerEvent, }}) - loop.serviceCommitRequests() + loop.endWake(0) _, resolved := outcome(t, worker) require.False(t, resolved, "a write inside the delay must not close the window: %s", name) } forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) res, ok := outcome(t, worker) require.True(t, ok) require.NoError(t, res.Err) @@ -366,7 +367,7 @@ func TestAttach_ForeignWindowIsNotStolen(t *testing.T) { require.Nil(t, loop.openWindow.pendingCR, "bob's attach must not claim alice's window") forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) res, ok := outcome(t, worker) require.True(t, ok) @@ -392,7 +393,7 @@ func TestAttach_NoWindowAtAllIsNotAMismatch(t *testing.T) { require.Nil(t, loop.openWindow, "precondition: nothing is open") forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) res, ok := outcome(t, worker) require.True(t, ok) @@ -426,7 +427,7 @@ func TestAttach_ForeignWindowClosingBeforeExpiryIsStillAMismatch(t *testing.T) { require.Nil(t, loop.openWindow, "precondition: nothing is open when bob's grace runs out") forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) res, ok := outcome(t, worker) require.True(t, ok) @@ -492,7 +493,7 @@ func TestAttach_ForeignWindowOpeningAfterExpiryIsNotAMismatch(t *testing.T) { CommitMode: CommitModePerEvent, }}) require.NotNil(t, loop.openWindow) - loop.serviceCommitRequests() + loop.endWake(0) res, ok := outcome(t, worker) require.True(t, ok) @@ -522,9 +523,11 @@ func TestAttach_IdempotentReSendKeepsFirstDeadline(t *testing.T) { assert.Equal(t, firstDeadline, loop.pendingCRs[id].attachDeadline, "the first deadline must be kept") } -// TestAttach_FinalizeFailureResolvesFailed verifies that when the attached -// window's commit fails (unreachable remote) the request resolves with an error. -func TestAttach_FinalizeFailureResolvesFailed(t *testing.T) { +// TestAttach_AnUnreachableRemoteHoldsTheRequest pins gap 4's contract for a save. A window whose +// commit cannot be made because the remote cannot be reached is a decided write, not a failed +// one: it stays pending for the publication retry, and the request riding it is held in +// WaitingForPush, because its write can still land. It used to be dropped and the request failed. +func TestAttach_AnUnreachableRemoteHoldsTheRequest(t *testing.T) { ctx := context.Background() scheme := runtime.NewScheme() require.NoError(t, clientgoscheme.AddToScheme(scheme)) @@ -562,10 +565,13 @@ func TestAttach_FinalizeFailureResolvesFailed(t *testing.T) { serviceAttach(loop, attachReq("alice", 0)) - res, ok := outcome(t, worker) - require.True(t, ok) - require.Error(t, res.Err, "an unreachable remote must resolve the request with an error") - assert.Nil(t, loop.openWindow, "a failed finalize still drops the broken window") + _, resolved := outcome(t, worker) + assert.False(t, resolved, "a request whose write can still land is not failed") + assert.Equal(t, PhaseWaitingForPush, worker.LookupCommitRequestPhase("default", crName, "uid-"+crName)) + assert.Nil(t, loop.openWindow, "the window closed on its own timers") + require.Len(t, loop.pendingWrites, 1, "the decided window stays pending") + assert.False(t, loop.pendingWrites[0].committedOnce, "nothing could commit it yet") + assert.True(t, loop.retry.pending(), "the publication retry owns the next attempt") } // TestFinalizeOpenWindow_ReturnsCommittedFlag verifies the boolean contract of @@ -701,6 +707,7 @@ func TestAttach_ResyncCutOffCarriesMessageAndResolvesOnPush(t *testing.T) { Scope: &scope, Result: resultCh, }) + loop.endWake(0) require.NoError(t, (<-resultCh).Err) res, ok := outcome(t, worker) @@ -1024,7 +1031,7 @@ func TestAttach_ACommittedRequestNeverClaimsAnotherWindow(t *testing.T) { CommitMode: CommitModePerEvent, }}) require.NotNil(t, loop.openWindow) - loop.serviceCommitRequests() + loop.endWake(0) assert.Nil(t, loop.openWindow.pendingCR, "a request whose window is already committed must not claim a second window") diff --git a/internal/git/commit_request_empty_test.go b/internal/git/commit_request_empty_test.go index 5aa25791..9e78f3e0 100644 --- a/internal/git/commit_request_empty_test.go +++ b/internal/git/commit_request_empty_test.go @@ -58,7 +58,7 @@ func TestCommitEmpty_NoWindowBeforeTheTimeoutRecordsTheMessage(t *testing.T) { serviceAttach(loop, commitEmptyReq("alice", "save: nothing changed")) forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) loop.pushPending() res, ok := outcome(t, worker) @@ -126,7 +126,7 @@ func TestCommitEmpty_AWindowMismatchNeverFallsBackToAnEmptyCommit(t *testing.T) serviceAttach(loop, commitEmptyReq("bob", "bob's save")) writeTo(loop, "alices-edit") // only alice's window is ever open forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) res, ok := outcome(t, worker) require.True(t, ok) @@ -148,7 +148,7 @@ func TestCommitEmpty_ResolveIsTheDefaultAndCommitsNothing(t *testing.T) { req.Message = "save" serviceAttach(loop, req) forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) res, ok := outcome(t, worker) require.True(t, ok) @@ -194,7 +194,7 @@ func TestCommitEmpty_AFailedPushTreatsEveryCommitShapeAlike(t *testing.T) { commit: func(loop *branchWorkerEventLoop) { serviceAttach(loop, commitEmptyReq("alice", message)) forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) }, outcome: FinalizeNoOpenWindow, empty: true, @@ -275,7 +275,7 @@ func TestCommitEmpty_AFailedRecordFailsTheRequest(t *testing.T) { // message to be recorded, and reporting a benign NoWindow would hide that it was not. serviceAttach(loop, commitEmptyReq("alice", "save")) forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) res, ok := outcome(t, worker) require.True(t, ok) @@ -323,7 +323,7 @@ func TestCommitEmpty_ARecordThatFailedToPushLandsInOrderAfterTheRemoteMoved(t *t // The record, then a failed push. serviceAttach(loop, commitEmptyReq("alice", message)) forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) require.Len(t, loop.pendingWrites, 1) staleRecordSHA := loop.pendingWrites[0].CommitSHA loop.pushPending() @@ -409,7 +409,7 @@ func TestCommitEmpty_ARecordOnARepositoryWithNoHistoryKeepsItsCause(t *testing.T serviceAttach(loop, commitEmptyReq("alice", "save: the first commit")) forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) loop.pushPending() res, ok := outcome(t, worker) @@ -443,7 +443,7 @@ func TestCommitEmpty_ABufferFlushStillServesTheWaitingSaves(t *testing.T) { serviceAttach(loop, req) writeTo(loop, "oversized") assert.Nil(t, loop.openWindow, "the buffer limit flushed the window") - loop.serviceCommitRequests() + loop.endWake(0) loop.pushPending() res, ok := outcome(t, worker) @@ -467,7 +467,7 @@ func TestCommitEmpty_ABufferFlushStillServesTheWaitingSaves(t *testing.T) { writeTo(loop, "oversized") // alice's write, flushed in the step that opened it require.Len(t, loop.pendingWrites, 1, "only alice's commit") forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) res, ok := outcome(t, worker) require.True(t, ok) diff --git a/internal/git/commit_request_withdraw_test.go b/internal/git/commit_request_withdraw_test.go index 22609e8e..44175c3a 100644 --- a/internal/git/commit_request_withdraw_test.go +++ b/internal/git/commit_request_withdraw_test.go @@ -144,7 +144,7 @@ func TestWithdraw_AnOutcomeIsKeptWhileAnAttachForItIsQueued(t *testing.T) { require.True(t, kept, "an outcome with an attach still queued is not collected") loop.handleQueueItem(<-worker.eventQueue) - loop.serviceCommitRequests() + loop.endWake(0) assert.Empty(t, loop.pendingCRs, "the late attach does not register the withdrawn request") res, _ := outcome(t, worker) require.ErrorIs(t, res.Err, ErrCommitRequestWithdrawn) diff --git a/internal/git/commit_window_metrics_test.go b/internal/git/commit_window_metrics_test.go index 413e0afa..1794297d 100644 --- a/internal/git/commit_window_metrics_test.go +++ b/internal/git/commit_window_metrics_test.go @@ -77,6 +77,7 @@ func TestCommitWindowMetrics_EachCloseReasonIsCountedUnderItsOwnName(t *testing. writeTo(loop, "a") loop.openWindow.timers.maxAt = time.Now().Add(-time.Millisecond) loop.closeOrArmWindow() + loop.endWake(0) // identity_change: another author's write arrives while alice's window is open. writeTo(loop, "b") @@ -200,7 +201,7 @@ func TestCommitWindowMetrics_ACommitEmptyRecordIsNotAWindow(t *testing.T) { serviceAttach(loop, commitEmptyReq("alice", "save: nothing changed")) forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) require.Len(t, loop.pendingWrites, 1, "the record was made") _, counted := telemetry.CollectInt64Sum(reader, commitWindowsMetric, map[string]string{}) diff --git a/internal/git/commit_window_timers_test.go b/internal/git/commit_window_timers_test.go index 14db07ba..77ef5323 100644 --- a/internal/git/commit_window_timers_test.go +++ b/internal/git/commit_window_timers_test.go @@ -82,6 +82,7 @@ func TestWindowTimers_ARequestIdleTimeoutLongerThanTheTargetsKeepsTheWindowOpen( time.Sleep(20 * time.Millisecond) loop.closeOrArmWindow() // what the commit timer does when it fires + loop.endWake(0) assert.NotNil(t, loop.openWindow, "the request's longer idleTimeout replaces the target's") } @@ -100,6 +101,7 @@ func TestWindowTimers_ARequestIdleTimeoutShorterThanTheTargetsClosesSooner(t *te time.Sleep(20 * time.Millisecond) loop.closeOrArmWindow() + loop.endWake(0) assert.Nil(t, loop.openWindow, "the request's shorter idleTimeout replaces the target's hour") } @@ -114,6 +116,7 @@ func TestWindowTimers_ARequestWithOnlyMaxDurationIgnoresTheTargetsIdleTimer(t *t writeTo(loop, "first") time.Sleep(20 * time.Millisecond) loop.closeOrArmWindow() + loop.endWake(0) assert.NotNil(t, loop.openWindow, "no idle close: maxDuration alone ends the collection") } @@ -262,7 +265,7 @@ func TestWindowTimers_AWaitingRequestWithMaxDurationZeroCommitsExactlyTheNextWri assert.Equal(t, PhaseWaitingForWindow, worker.LookupCommitRequestPhase("default", crName, "uid-"+crName)) writeTo(loop, "the-change") - loop.serviceCommitRequests() + loop.endWake(0) assert.Nil(t, loop.openWindow, "the write opened a window, the request attached, and maxDuration: 0s closed it") require.Len(t, loop.pendingWrites, 1) assert.Equal(t, req.Message, loop.pendingWrites[0].CommitMessage) diff --git a/internal/git/decided_write_test.go b/internal/git/decided_write_test.go new file mode 100644 index 00000000..a68c1f90 --- /dev/null +++ b/internal/git/decided_write_test.go @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// A closed window is a decided write. A remote that cannot be reached when the window closes used +// to drop the window and fail the save riding it, and only a missing parent asked for a snapshot to +// re-derive what was lost. These pin that the decision now survives the outage, keeps its author +// and message, and lands through the publication retry, without a fetch per window meanwhile. + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + gogit "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + gitclient "github.com/go-git/go-git/v6/plumbing/client" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/client" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// failSyncs makes every sync to the remote fail as an unreachable remote would, until restored, +// and counts the attempts. +func failSyncs(t *testing.T) (*atomic.Int32, func()) { + t.Helper() + original := syncToRemoteFn + var attempts atomic.Int32 + syncToRemoteFn = func( + _ context.Context, _ *gogit.Repository, _ plumbing.ReferenceName, _ string, _ []gitclient.Option, + ) (*PullReport, error) { + attempts.Add(1) + return nil, errors.New("dial tcp: connection refused") + } + restore := func() { syncToRemoteFn = original } + t.Cleanup(restore) + return &attempts, restore +} + +func TestDecidedWrite_SurvivesAFailedRebuildAndLandsThroughTheRetry(t *testing.T) { + f := newLedgerFixture(t, "decided-write-survives-rebuild", true) + f.createLedgerTarget("team-a", nil) + f.publish("prime") + + loop := newBranchWorkerEventLoop(f.worker, time.Hour) + loop.lastPushAt = time.Now() // the first write waits out the push cooldown + defer loop.stopTimers() + + // One write is committed and retained, and then the checkout has to be rebuilt before anything + // can commit on it, while the remote is unreachable. + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("retained", "alice", ledgerTargetName)}, + CommitMode: CommitModePerEvent, + }}) + require.True(t, loop.finalizeOpenWindow()) + f.worker.markWorktreeDirty("a write failed part-way and could not be undone") + syncs, restoreSyncs := failSyncs(t) + + // A save's window closes during the outage. + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("saved-during-outage", "alice", ledgerTargetName)}, + CommitMode: CommitModePerEvent, + }}) + req := attachReq("alice", 0) + req.GitTargetName = ledgerTargetName + req.Message = "save during the outage" + serviceAttach(loop, req) + + _, resolved := outcome(t, f.worker) + require.False(t, resolved, "the save is held, not failed: its write can still land") + assert.Equal(t, PhaseWaitingForPush, f.worker.LookupCommitRequestPhase("default", crName, "uid-"+crName)) + require.Len(t, loop.pendingWrites, 2, "the window is decided and kept pending") + assert.False(t, loop.pendingWrites[1].committedOnce) + require.True(t, loop.retry.pending(), "the failure armed the publication retry") + require.Equal(t, int32(1), syncs.Load(), "the window tried the rebuild once") + + // Another window during the outage is decided without spending a connection on the rebuild. + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("also-during-outage", "bob", ledgerTargetName)}, + CommitMode: CommitModePerEvent, + }}) + require.True(t, loop.finalizeOpenWindow()) + assert.Equal(t, int32(1), syncs.Load(), "a decision waits for the pending retry instead of fetching") + require.Len(t, loop.pendingWrites, 3) + + // The remote comes back and the retry fires. + restoreSyncs() + fireRetry(loop) + + res, resolved := outcome(t, f.worker) + require.True(t, resolved) + require.NoError(t, res.Err) + assert.Equal(t, FinalizeCommitted, res.Outcome) + assert.Contains(t, gitOut(t, f.repoDir, "log", "-1", "--format=%B", res.Commit), "save during the outage", + "the save's own message reached Git") + assert.Contains(t, gitOut(t, f.repoDir, "log", "-1", "--format=%an", res.Commit), "alice", + "the window's own author reached Git") + names := remoteFileNames(t, f.repoDir) + for _, name := range []string{"retained", "saved-during-outage", "also-during-outage"} { + assert.Contains(t, names, name) + } + assert.Empty(t, loop.pendingWrites) + assert.False(t, loop.retry.pending()) +} + +// A delete decided during an outage is committed on a tree the worker fetched later, so it is planned +// under the prune policy in force then, as a replay is. Found in review of #413: the deferred write +// skipped the re-read, and a target tightened to Never still lost the file. +func TestDecidedWrite_ADeferredDeleteObeysATightenedPrunePolicy(t *testing.T) { + f := newLedgerFixture(t, "decided-write-tightened-prune", true) + f.createLedgerTarget("team-a", nil) + loop := newBranchWorkerEventLoop(f.worker, time.Hour) + defer loop.stopTimers() + write := func(event Event) { + loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) + require.True(t, loop.finalizeOpenWindow()) + } + + write(configMapTargetEvent("keep-me", "alice", ledgerTargetName)) + loop.pushPending() + require.Contains(t, remoteFileNames(t, f.repoDir), "keep-me") + + // A retained write and a checkout to rebuild, while the remote is unreachable: the delete is + // decided and waits. + write(configMapTargetEvent("retained", "alice", ledgerTargetName)) + f.worker.markWorktreeDirty("a write failed part-way and could not be undone") + _, restoreSyncs := failSyncs(t) + deleted := configMapTargetEvent("keep-me", "alice", ledgerTargetName) + deleted.Operation = "DELETE" + write(deleted) + require.Len(t, loop.pendingWrites, 2) + require.False(t, loop.pendingWrites[1].committedOnce) + + // The operator stops pruning before the remote comes back. + target := &configv1alpha3.GitTarget{} + key := client.ObjectKey{Name: ledgerTargetName, Namespace: "default"} + require.NoError(t, f.worker.Client.Get(f.worker.ctx, key, target)) + target.Spec.Prune = &configv1alpha3.PrunePolicy{Mode: configv1alpha3.PruneNever} + require.NoError(t, f.worker.Client.Update(f.worker.ctx, target)) + + restoreSyncs() + fireRetry(loop) + + require.Empty(t, loop.pendingWrites) + assert.Contains(t, remoteFileNames(t, f.repoDir), "keep-me", + "the deferred delete is planned under the policy in force when it is committed") +} + +// Writes are kept through an outage, so the pending writes are bounded at admission instead: past the +// retained-byte budget, while a failed attempt waits for its retry, new writes, saves and resyncs are +// refused the way a full queue refuses them, and their producers keep them. A healthy branch never +// closes, and admission reopens once the pending writes are published. +func TestDecidedWrite_AdmissionClosesAtTheBudgetDuringAnOutage(t *testing.T) { + f := newLedgerFixture(t, "decided-write-admission", true) + f.createLedgerTarget("team-a", nil) + f.publish("prime") + f.worker.branchBufferMaxBytes = 1 // any retained write fills it + loop := newBranchWorkerEventLoop(f.worker, time.Hour) + loop.lastPushAt = time.Now() + defer loop.stopTimers() + write := func(name string) { + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent(name, "alice", ledgerTargetName)}, + CommitMode: CommitModePerEvent, + }}) + loop.finalizeOpenWindow() // the budget usually closed it on arrival already + loop.publishLoopState(0) // what every loop iteration does + } + + write("over-budget-but-healthy") + assert.True(t, f.worker.Enqueue(configMapTargetEvent("admitted", "alice", ledgerTargetName)), + "a healthy branch drains at the next push, so it never closes") + <-f.worker.eventQueue + f.worker.inflightItems.Add(-1) + + f.worker.markWorktreeDirty("a write failed part-way and could not be undone") + _, restoreSyncs := failSyncs(t) + write("during-the-outage") + require.True(t, loop.retry.pending()) + + assert.False(t, f.worker.Enqueue(configMapTargetEvent("refused", "alice", ledgerTargetName)), + "the watch keeps its cursor and delivers it again") + result := make(chan ResyncResult, 1) + assert.False(t, f.worker.EnqueueResync(&ResyncRequest{ + GitTargetName: ledgerTargetName, GitTargetNamespace: "default", Result: result, + })) + require.ErrorIs(t, (<-result).Err, ErrFinalizeQueueFull) + + restoreSyncs() + fireRetry(loop) + loop.publishLoopState(0) + require.Empty(t, loop.pendingWrites) + assert.True(t, f.worker.Enqueue(configMapTargetEvent("after", "alice", ledgerTargetName)), + "published, so admission reopens") +} + +// A write with no payload still costs memory to keep: its message, its metadata and the save it +// carries. An empty save's record used to be charged nothing, so a branch whose remote was down +// kept admitting saves against a budget it never reached. +func TestDecidedWrite_EmptySavesCountAgainstTheBudgetDuringAnOutage(t *testing.T) { + f := newLedgerFixture(t, "decided-write-empty-saves", true) + f.createLedgerTarget("team-a", nil) + f.publish("prime") + f.worker.branchBufferMaxBytes = 1 // any retained write fills it + f.worker.markWorktreeDirty("a write failed part-way and could not be undone") + failSyncs(t) + loop := newBranchWorkerEventLoop(f.worker, time.Hour) + defer loop.stopTimers() + + refused := 0 + for i := range 10 { + req := attachReq("alice", 0) + req.Name = fmt.Sprintf("save-%d", i) + req.UID = "uid-" + req.Name + req.GitTargetName = ledgerTargetName + req.CommitEmpty = true + req.Message = "save while Git is down" + f.worker.EnqueueAttach(req) + select { + case item := <-f.worker.eventQueue: + loop.handleQueueItem(item) + loop.endWake(0) + default: + refused++ // the controller sends it again + } + } + + require.True(t, loop.retry.pending()) + assert.NotNil(t, f.worker.IntakePaused(), "the retained empty save fills the budget") + assert.Len(t, loop.pendingWrites, 1, "only the save admitted before the budget filled is retained") + assert.Equal(t, 9, refused, "every later save is refused at admission") + assert.Positive(t, loop.pendingWritesBytes, "an empty record is charged for what it keeps") + + loop.removeAt(0) + assert.Zero(t, loop.pendingWritesBytes, "no longer pending refunds exactly what deciding charged") +} + +// A resync arriving while a failed attempt waits for its retry does not dial the remote early: its +// caller is answered at once with the failure the retry is waiting out, and the resync stays in the +// log, applied with the rest when the retry is due. Nor does a write decided behind it, which +// would otherwise reach the resync's fetch through the same pass. +func TestDecidedWrite_AResyncDuringBackoffWaitsForTheRetry(t *testing.T) { + f := newLedgerFixture(t, "decided-write-resync-backoff", true) + f.createLedgerTarget("team-a", nil) + f.publish("prime") + calls, restoreSyncs := failSyncs(t) + loop := newBranchWorkerEventLoop(f.worker, time.Hour) + defer loop.stopTimers() + resync := func() error { + req := &ResyncRequest{GitTargetName: ledgerTargetName, GitTargetNamespace: "default", + Result: make(chan ResyncResult, 1)} + loop.handleResyncRequest(req) + loop.endWake(0) + return (<-req.Result).Err + } + + require.Error(t, resync(), "the first resync spends the attempt and finds the remote down") + require.True(t, loop.awaitingRetry()) + require.Equal(t, int32(1), calls.Load()) + + require.ErrorContains(t, resync(), "connection refused", "the caller hears the known failure at once") + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("behind-the-resyncs", "alice", ledgerTargetName)}, + CommitMode: CommitModePerEvent, + }}) + loop.finalizeOpenWindow() + assert.Equal(t, int32(1), calls.Load(), "nothing dials the remote before the retry is due") + require.Len(t, loop.pendingWrites, 3, "both resyncs and the write stay pending") + + restoreSyncs() + fireRetry(loop) + assert.Empty(t, loop.pendingWrites, "the retry applies and publishes everything it kept") +} + +// Two decided writes to one object replay as the decisions they are, even when the moved remote +// already holds the second one's content. Each write is judged against the tree the one before it +// left, so the first reverts what the remote holds and the second applies it again: the final tree +// is right, and the history carries both writes with their own authors and saves. Collapsing them +// would need each write judged against the writes after it, which a save riding the first would +// then lose its commit to. Edits inside one window do collapse, before anything is decided. +func TestDecidedWrite_SameObjectEditsReplayAsDecidedOntoARemoteHoldingTheLast(t *testing.T) { + f := newLedgerFixture(t, "decided-write-same-object", true) + f.createLedgerTarget("team-a", nil) + f.publish("prime") + loop := newBranchWorkerEventLoop(f.worker, 0) + t.Cleanup(loop.stopTimers) + loop.lastPushAt = time.Now() // the cooldown holds both writes back + + edit := func(value string) []byte { + event := configMapTargetEvent("settings", "alice", ledgerTargetName) + event.Object.Object["data"] = map[string]any{"value": value} + loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) + path := filepath.Join(f.worker.repoPath(), "team-a", + generateFilePath(event.Identifier, itypes.SensitiveResourcePolicy{})) + content, err := os.ReadFile(path) + require.NoError(t, err) + return content + } + first := edit("1") + save := attachReq("alice", time.Hour) + save.GitTargetName = ledgerTargetName + save.MaxDuration = 0 // the save's window closes on the write it attaches to + save.Message = "set value to 2" + loop.handleQueueItem(WorkItem{Attach: save}) + second := edit("2") + require.Len(t, loop.pendingWrites, 2, "two decided writes wait for the push") + + gitPath := filepath.ToSlash(filepath.Join("team-a", generateFilePath( + configMapTargetEvent("settings", "alice", ledgerTargetName).Identifier, itypes.SensitiveResourcePolicy{}))) + external := simulateClientCommitOnDisk(t, f.repoDir, "main", gitPath, string(second)) + + loop.pushPending() + loop.endWake(0) + + require.Empty(t, loop.pendingWrites, "the replayed writes are published") + repo, err := gogit.PlainOpen(f.repoDir) + require.NoError(t, err) + head, err := repo.Reference(plumbing.NewBranchReferenceName("main"), true) + require.NoError(t, err) + replayed := commitsAfterHash(t, repo, head.Hash(), external) + require.Len(t, replayed, 2, "both decisions are commits: a revert, then the reapply") + assert.Equal(t, string(first), fileAt(t, replayed[0], gitPath), "the first write reverts the remote's value") + assert.Equal(t, string(second), fileAt(t, replayed[1], gitPath), "the second applies it again") + assert.Equal(t, "set value to 2", replayed[1].Message) + + outcome, ok := f.worker.LookupCommitRequestOutcome(save.Namespace, save.Name, save.UID) + require.True(t, ok) + assert.Equal(t, FinalizeCommitted, outcome.Outcome, "the save resolves on its own write's commit") + assert.Equal(t, replayed[1].Hash.String(), outcome.Commit) +} + +// fileAt is the content of path in commit's tree. +func fileAt(t *testing.T, commit *object.Commit, path string) string { + t.Helper() + file, err := commit.File(path) + require.NoError(t, err) + content, err := file.Contents() + require.NoError(t, err) + return content +} + +// A write that stops being pending leaves memory with it. Shortening the pending writes in place kept the vacated +// slot of its backing array, so a resync that committed nothing kept its whole snapshot reachable +// while the budget said the branch held nothing. +func TestDecidedWrite_NoLongerPendingReleasesThePayload(t *testing.T) { + loop := newBranchWorkerEventLoop(newMetricsTestWorker(), time.Hour) + loop.decide(PendingWrite{Kind: PendingWriteResync, Desired: []manifestanalyzer.DesiredResource{{}}}) + + loop.removeAt(0) + + require.Empty(t, loop.pendingWrites) + assert.Zero(t, loop.pendingWritesBytes) + vacated := loop.pendingWrites[:1][0] + assert.Nil(t, vacated.Desired, "the vacated slot no longer holds the removed write's snapshot") +} diff --git a/internal/git/deleted_remote_branch_test.go b/internal/git/deleted_remote_branch_test.go index a5370640..de85cadb 100644 --- a/internal/git/deleted_remote_branch_test.go +++ b/internal/git/deleted_remote_branch_test.go @@ -55,7 +55,7 @@ func TestPush_DeletedRemoteBranchIsTreatedAsMovement(t *testing.T) { giveRemoteADefaultBranchBesidesMain(t, f.repoDir) f.publish("prime") - f.commit(false, "after-the-delete") + f.commit("after-the-delete") gitInRepo(t, f.repoDir, "update-ref", "-d", "refs/heads/main") // Push once, without the replay, to read the error the remote produces. @@ -128,10 +128,10 @@ func TestPush_DeletedRemoteBranchDoesNotStrandRetainedWrites(t *testing.T) { // Drop trust so the next cycle fetches, which leaves refs/remotes/origin/main exactly at the // remote tip the cycle then roots on. That equality is the stranding condition. f.worker.invalidateBase("test: force the cycle to fetch first") - f.commit(false, "written-after-the-fetch") + f.commit("written-after-the-fetch") f.push() f.worker.invalidateBase("test: force the cycle to fetch first") - f.commit(false, "written-after-the-delete") + f.commit("written-after-the-delete") gitInRepo(t, f.repoDir, "update-ref", "-d", "refs/heads/main") diff --git a/internal/git/dirty_worktree_recovery_test.go b/internal/git/dirty_worktree_recovery_test.go index a2867f5a..727b439b 100644 --- a/internal/git/dirty_worktree_recovery_test.go +++ b/internal/git/dirty_worktree_recovery_test.go @@ -7,8 +7,9 @@ package git // The head-of-cycle fetch used to launder the worktree as a side effect (§5). Now that a trusted, // clean base skips it, the cleanup has to be deliberate — and commitPendingWrites cannot perform // it once work is retained, because a reset is exactly what would destroy the local commits those -// writes already produced. recoverRetainedWrites resets and REPLAYS instead, which is why it lives -// on the event loop, and why every loop path that reaches commitPendingWrites has to call it. +// writes already produced. The loop's materialize resets and REPLAYS instead, and its commit +// refuses a checkout that is not the projection of the retained writes, so a path that skipped +// materialize fails rather than committing leftovers. // // See docs/design/push-notification-and-reconcile-trigger.md §1.5. @@ -122,6 +123,7 @@ func TestAtomicWrite_DoesNotCommitAFailedWritesLeftovers(t *testing.T) { GitTargetName: "team-a", GitTargetNamespace: "default", }) + loop.endWake(0) require.Len(t, loop.pendingWrites, 2, "C committed: A is retained alongside it") assert.False(t, headTreeContains(t, repoPath, leftoverPath), @@ -134,10 +136,9 @@ func TestAtomicWrite_DoesNotCommitAFailedWritesLeftovers(t *testing.T) { // TestEveryLoopCommitPathRecoversADirtyWorktree enumerates the loop's commit entry points against // one dirty fixture. // -// It is written as a table on purpose. recoverRetainedWrites has to be called by every path that -// reaches commitPendingWrites, and nothing in the type system says so — a new entry point added -// later would silently commit leftovers. Adding its row here is the cheapest way to make that -// omission fail loudly. +// It is written as a table on purpose. The commit guard turns a path that skips materialize into a +// failed write rather than a laundered one, but only this table proves each path RECOVERS: a new +// entry point that forgot to materialize would fail every write while the worktree is dirty. func TestEveryLoopCommitPathRecoversADirtyWorktree(t *testing.T) { cases := []struct { name string @@ -164,6 +165,7 @@ func TestEveryLoopCommitPathRecoversADirtyWorktree(t *testing.T) { GitTargetName: "team-a", GitTargetNamespace: "default", }) + loop.endWake(0) }, }, { @@ -179,6 +181,7 @@ func TestEveryLoopCommitPathRecoversADirtyWorktree(t *testing.T) { Scope: &scope, Result: resultCh, }) + loop.endWake(0) require.NoError(t, (<-resultCh).Err) }, }, @@ -188,7 +191,7 @@ func TestEveryLoopCommitPathRecoversADirtyWorktree(t *testing.T) { t.Helper() serviceAttach(loop, commitEmptyReq("alice", "save: nothing changed")) forceDue(loop) - loop.serviceCommitRequests() + loop.endWake(0) require.Len(t, loop.pendingWrites, 2, "the record was made") }, }, @@ -198,6 +201,7 @@ func TestEveryLoopCommitPathRecoversADirtyWorktree(t *testing.T) { t.Helper() target := itypes.NewResourceReference("team-a", "default") loop.commitRefusalTouch(refusalKeyFor(target, "configmaps"), "refused", "observation") + loop.endWake(0) require.Len(t, loop.pendingWrites, 2, "the empty commit was made") }, }, @@ -234,3 +238,43 @@ func TestEveryLoopCommitPathRecoversADirtyWorktree(t *testing.T) { }) } } + +// TestDirtyWorktree_AFailedWriteIsUndoneWithoutAFetch pins the local undo. A batch whose second +// write fails after the first has committed and staged its file used to leave the worktree dirty, +// and the next commit then fetched and reset from the remote to clear it. The checkout is the +// projection of the retained writes, so the commit the batch started on is all the cleanup needs: +// the failed batch leaves no commit, no file and no doubt behind, and the next commit costs no +// round trip. +func TestDirtyWorktree_AFailedWriteIsUndoneWithoutAFetch(t *testing.T) { + f := newLedgerFixture(t, "failed-write-undone-locally", true) + f.publish("prime") + + repo, err := gogit.PlainOpen(f.worker.repoPath()) + require.NoError(t, err) + start := localHead(repo) + + committed, err := f.worker.buildGroupedPendingWrite(f.worker.ctx, + []Event{configMapEvent("committed-then-undone", "alice", "team-a")}) + require.NoError(t, err) + secret := configMapEvent("unencrypted", "alice", "team-a") + secret.Identifier.Resource = "secrets" + secret.Object.SetKind("Secret") + failing, err := f.worker.buildGroupedPendingWrite(f.worker.ctx, []Event{secret}) + require.NoError(t, err) + + err = f.worker.commitPendingWrites([]PendingWrite{*committed, *failing}) + require.ErrorContains(t, err, "secret encryption is required") + + assert.False(t, f.worker.worktreeDirty(), "the failed batch was undone locally") + assert.True(t, f.worker.baseTrusted(), "the checkout is back where the trusted base left it") + assert.Equal(t, start, localHead(repo), "the first write's commit is undone with the batch") + worktree, err := repo.Worktree() + require.NoError(t, err) + status, err := worktree.Status() + require.NoError(t, err) + assert.True(t, status.IsClean(), "no leftover of the failed batch survives: %s", status) + + before := f.mark() + f.commit("after-the-failure") + assert.Zero(t, f.mark().since(before).connections(), "the next commit plans on the checkout as it is") +} diff --git a/internal/git/git.go b/internal/git/git.go index 7fc97eca..67443f72 100644 --- a/internal/git/git.go +++ b/internal/git/git.go @@ -42,15 +42,19 @@ func CheckRepo(ctx context.Context, repoURL string, auth []gitclient.Option) (*R logger := log.FromContext(ctx) logger.V(1).Info("Checking repository connectivity and metadata", "url", repoURL) - // Use remote.List() for lightweight connectivity check + // Use a ref listing for a lightweight connectivity check, bounded like every call to a Git + // server (network_bound.go): a controller's reconcile has no deadline of its own. remote := git.NewRemote(nil, &config.RemoteConfig{ Name: "origin", URLs: []string{repoURL}, }) - refs, err := remote.List(&git.ListOptions{ - ClientOptions: auth, + callCtx, cancel := boundGitCall(ctx) + defer cancel() + refs, err := remote.ListContext(callCtx, &git.ListOptions{ + ClientOptions: boundToContext(callCtx, auth), }) + err = boundedCallError(callCtx, err) if err != nil { // Check if this is an empty repository error if errors.Is(err, transport.ErrEmptyRemoteRepository) { diff --git a/internal/git/git_atomic_push.go b/internal/git/git_atomic_push.go index a921eb30..ff303375 100644 --- a/internal/git/git_atomic_push.go +++ b/internal/git/git_atomic_push.go @@ -49,7 +49,7 @@ func getPushSession( return nil, fmt.Errorf("failed to parse remote URL: %w", err) } - session, err := gitclient.New(auth...).Handshake(ctx, &transport.Request{ + session, err := gitclient.New(boundToContext(ctx, auth)...).Handshake(ctx, &transport.Request{ URL: endpoint, Command: transport.ReceivePackService, }) @@ -391,6 +391,22 @@ func PushAtomic( logger := log.FromContext(ctx) + // One push session is one call to the server: bounded as a whole (network_bound.go), from the + // handshake through the advertisement it validates against to the upload. + ctx, cancel := boundGitCall(ctx) + defer cancel() + outcome, err := pushAtomicSession(ctx, repo, rootHash, rootBranch, auth, logger) + return outcome, boundedCallError(ctx, err) +} + +func pushAtomicSession( + ctx context.Context, + repo *git.Repository, + rootHash plumbing.Hash, + rootBranch plumbing.ReferenceName, + auth []gitclient.Option, + logger logr.Logger, +) (PushOutcome, error) { session, err := getPushSession(ctx, repo, auth) if err != nil { return PushOutcome{}, err diff --git a/internal/git/git_operations_test.go b/internal/git/git_operations_test.go index 280cffa8..f6d030af 100644 --- a/internal/git/git_operations_test.go +++ b/internal/git/git_operations_test.go @@ -339,7 +339,7 @@ func TestBranchWorker_FirstCommitOnEmptyRepo(t *testing.T) { require.NoError(t, err) pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) // Verify repository state @@ -443,7 +443,7 @@ func TestBranchWorker_BranchCreationAndPush(t *testing.T) { require.NoError(t, err) pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) // Verify local branch exists @@ -468,7 +468,7 @@ func TestBranchWorker_ConflictResolution(t *testing.T) { event := createTestEvent(t, "some-resource") pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) // Simulate another client creating conflicting commit in remote after our local commit exists. simulateClientCommitOnDisk(t, "file://"+serverPath, "main", "README.md", "This is our conflicting readme") @@ -527,7 +527,7 @@ func TestBranchWorker_ConcurrentOperations(t *testing.T) { results <- err return } - if err := worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false); err != nil { + if err := worker.commitPendingWrites([]PendingWrite{*pendingWrite}); err != nil { results <- err return } @@ -689,7 +689,7 @@ func TestPullBranch_MergeToDefaultScenario(t *testing.T) { event := createTestEvent(t, "resource1") pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) pullReport, err = PrepareBranch(context.Background(), remoteURL, localPath, "feature", nil) @@ -714,7 +714,7 @@ func TestPullBranch_MergeToDefaultScenario(t *testing.T) { event = createTestEvent(t, "resource2") pendingWrite, err = worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) pullReport, err = PrepareBranch(context.Background(), remoteURL, localPath, "feature", nil) @@ -775,7 +775,7 @@ func TestPullBranch_DanglingHead(t *testing.T) { event := createTestEvent(t, "resilient-change") pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) // 7. Verify Persistence @@ -832,7 +832,7 @@ func TestPullBranch_DanglingHead_NewOrphan(t *testing.T) { event := createTestEvent(t, "orphan-resource") pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) // 5. Verify Topology on Server @@ -883,7 +883,7 @@ func TestPullBranch_UnexpectedMergeScenario(t *testing.T) { event := createTestEvent(t, "resource1") pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) simulateSimpleMerge(t, remoteURL, "feature", "main") @@ -892,7 +892,7 @@ func TestPullBranch_UnexpectedMergeScenario(t *testing.T) { event = createTestEvent(t, "resource2") pendingWrite, err = worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) pullReport, err = PrepareBranch(context.Background(), remoteURL, localPath, "feature", nil) @@ -989,7 +989,7 @@ spec: require.NoError(t, err) pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) repo, err = git.PlainOpen(worker.repoPath()) require.NoError(t, err) @@ -1031,7 +1031,7 @@ func TestPullBranch_WhipedRepo(t *testing.T) { event := createTestEvent(t, "resource1") pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) pullReport, err = PrepareBranch(context.Background(), remoteURL, localPath, "feature", nil) diff --git a/internal/git/git_push_advertisement_test.go b/internal/git/git_push_advertisement_test.go index ef5a050e..1a25cf6d 100644 --- a/internal/git/git_push_advertisement_test.go +++ b/internal/git/git_push_advertisement_test.go @@ -28,7 +28,7 @@ import ( func TestPushAtomic_RejectionCarriesAdvertisedHash(t *testing.T) { f := newLedgerFixture(t, "adv-rejection", true) - f.commit(false, "mine") + f.commit("mine") f.contend("OUTSIDE.md", "from-another-writer\n") contendingHash := plumbing.NewHash(revParseMain(t, f.repoDir)) @@ -52,7 +52,7 @@ func TestPushAtomic_RejectionCarriesAdvertisedHash(t *testing.T) { // carried an advertisement must not spend a SmartFetch learning the same number again. func TestRunPushCycle_MovedRemote_DoesNotRefetchTheHash(t *testing.T) { f := newLedgerFixture(t, "adv-no-refetch", true) - f.commit(false, "mine") + f.commit("mine") f.contend("OUTSIDE.md", "from-another-writer\n") fetchCalls := 0 @@ -80,7 +80,7 @@ func TestRunPushCycle_MovedRemote_DoesNotRefetchTheHash(t *testing.T) { // transient fault. func TestRunPushCycle_PushWithoutAdvertisement_FallsBackToFetch(t *testing.T) { f := newLedgerFixture(t, "adv-fallback", true) - f.commit(false, "mine") + f.commit("mine") // A connection that died: no advertisement, so no typed error. originalPush := pushAtomicFn diff --git a/internal/git/git_roundtrip_ledger_test.go b/internal/git/git_roundtrip_ledger_test.go index e8a419b8..04df94eb 100644 --- a/internal/git/git_roundtrip_ledger_test.go +++ b/internal/git/git_roundtrip_ledger_test.go @@ -128,10 +128,9 @@ func newLedgerFixtureOnBranch(t *testing.T, slug string, seeded bool, branch str // operation, so setup traffic is never attributed to the thing under test. func (f *ledgerFixture) mark() gitRequestSnapshot { return f.sim.ledger.snapshot() } -// commit runs one local commit for the named ConfigMaps and retains its pending write. -// hasPendingCommits is the flag commitPendingWrites gates its head-of-cycle fetch on: false means -// "first commit of a cycle". -func (f *ledgerFixture) commit(hasPendingCommits bool, names ...string) { +// commit runs one local commit for the named ConfigMaps and retains its pending write. Only the +// first commit with nothing committed before it may fetch: commitPendingWrites reads that from the checkout. +func (f *ledgerFixture) commit(names ...string) { f.t.Helper() events := make([]Event, 0, len(names)) for _, name := range names { @@ -139,7 +138,7 @@ func (f *ledgerFixture) commit(hasPendingCommits bool, names ...string) { } pendingWrite, err := f.worker.buildGroupedPendingWrite(f.worker.ctx, events) require.NoError(f.t, err) - require.NoError(f.t, f.worker.commitPendingWrites([]PendingWrite{*pendingWrite}, hasPendingCommits)) + require.NoError(f.t, f.worker.commitPendingWrites([]PendingWrite{*pendingWrite})) f.pending = append(f.pending, *pendingWrite) } @@ -153,7 +152,7 @@ func (f *ledgerFixture) push() { // publish is the whole steady-state cycle: one commit, then the push. func (f *ledgerFixture) publish(names ...string) { f.t.Helper() - f.commit(false, names...) + f.commit(names...) f.push() } @@ -235,9 +234,9 @@ func ledgerOperations() []ledgerOp { prime: func(f *ledgerFixture) { f.publish("prime") }, run: func(f *ledgerFixture) { // Only the first commit of a cycle may fetch: that is the claim this row checks. - f.commit(false, "first") - f.commit(true, "second") - f.commit(true, "third") + f.commit("first") + f.commit("second") + f.commit("third") f.push() }, }, @@ -250,7 +249,7 @@ func ledgerOperations() []ledgerOp { // The identical object plans to no change, so commitPendingWrites creates no // commit — and the write is still retained and still reaches PushAtomic, which is // the property §2 rests on. - f.commit(false, "same") + f.commit("same") require.True(f.t, f.pending[0].CommitSHA.IsZero(), "the no-diff write must be retained with no commit of its own") f.push() @@ -262,7 +261,7 @@ func ledgerOperations() []ledgerOp { seeded: true, prime: func(f *ledgerFixture) { f.publish("prime") }, run: func(f *ledgerFixture) { - f.commit(false, "mine") + f.commit("mine") f.contend("OUTSIDE.md", "from-another-writer\n") f.push() }, @@ -273,7 +272,7 @@ func ledgerOperations() []ledgerOp { seeded: true, prime: func(f *ledgerFixture) { f.publish("prime") }, run: func(f *ledgerFixture) { - f.commit(false, "mine") + f.commit("mine") f.contend("OUTSIDE-1.md", "first-other-writer\n") // Move the remote again just before the SECOND push attempt, so the replay is // rejected too. Hooking the push is the only way to land a write inside the retry @@ -301,7 +300,7 @@ func ledgerOperations() []ledgerOp { seeded: true, prime: func(f *ledgerFixture) { f.publish("prime") - f.commit(false, "retained") + f.commit("retained") }, run: func(f *ledgerFixture) { require.NoError(f.t, @@ -332,7 +331,7 @@ func ledgerOperations() []ledgerOp { // handleQueueItem commits AND pushes: applyResync ends in maybeSchedulePush, // and a worker that has not pushed before has no cooldown to wait out. So this // one call is the whole resync cycle, which is what the row is meant to cost. - loop.handleQueueItem(WorkItem{Resync: req}) + loop.handleQueueItem(resyncItem(req)) result := <-req.Result require.NoError(f.t, result.Err) require.Equal(f.t, 1, result.Stats.Created, "the resync must have written something") @@ -386,7 +385,7 @@ func ledgerOperations() []ledgerOp { deleted.Object = nil pendingWrite, err := f.worker.buildGroupedPendingWrite(f.worker.ctx, []Event{deleted}) require.NoError(f.t, err) - require.NoError(f.t, f.worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(f.t, f.worker.commitPendingWrites([]PendingWrite{*pendingWrite})) f.pending = append(f.pending, *pendingWrite) f.push() }, @@ -419,9 +418,11 @@ func ledgerOperations() []ledgerOp { pendingWrite, err := f.worker.buildGroupedPendingWrite(f.worker.ctx, []Event{configMapEvent("held", "alice", "team-a")}) require.NoError(f.t, err) - err = f.worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false) - require.ErrorIs(f.t, err, ErrParentBranchNotFound) - f.loop.noteParentUnavailable(err) + // The fetch that finds the parent missing; the write stays pending for it. + f.loop.decide(*pendingWrite) + f.loop.endWake(0) + require.Len(f.t, f.loop.pendingWrites, 1) + require.True(f.t, f.loop.recovery.active) }, run: func(f *ledgerFixture) { f.worker.clock = func() time.Time { return time.Now().Add(time.Hour) } diff --git a/internal/git/git_smart_fetch.go b/internal/git/git_smart_fetch.go index 97b2d79e..757c5ba1 100644 --- a/internal/git/git_smart_fetch.go +++ b/internal/git/git_smart_fetch.go @@ -109,7 +109,7 @@ func SmartFetchFrom( } // 1. Audit: List refs - refs, err := listRemoteRefs(remote, auth) + refs, err := listRemoteRefs(ctx, remote, auth) if err != nil { return "", err } @@ -141,16 +141,8 @@ func SmartFetchFrom( // 4. Execute: Fetch if len(refSpecs) > 0 { - err = repo.Fetch(&git.FetchOptions{ - RemoteName: remoteName, - ClientOptions: auth, - RefSpecs: refSpecs, - Depth: 1, - Force: true, - Prune: true, - }) - if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { - return "", fmt.Errorf("smart fetch failed: %w", err) + if err := fetchBounded(ctx, repo, remoteName, refSpecs, auth); err != nil { + return "", err } } @@ -193,17 +185,43 @@ func configuredParent(refs []*plumbing.Reference, parent string) (string, string return "", "" } -func listRemoteRefs(remote *git.Remote, auth []gitclient.Option) ([]*plumbing.Reference, error) { - refs, err := remote.List(&git.ListOptions{ClientOptions: auth}) +// listRemoteRefs reads the remote's advertisement, bounded by gitCallTimeout: see network_bound.go. +func listRemoteRefs( + ctx context.Context, remote *git.Remote, auth []gitclient.Option, +) ([]*plumbing.Reference, error) { + ctx, cancel := boundGitCall(ctx) + defer cancel() + refs, err := remote.ListContext(ctx, &git.ListOptions{ClientOptions: boundToContext(ctx, auth)}) if errors.Is(err, transport.ErrEmptyRemoteRepository) { return nil, nil // Valid state, not an error } if err != nil { - return nil, fmt.Errorf("failed to list remote refs: %w", err) + return nil, fmt.Errorf("failed to list remote refs: %w", boundedCallError(ctx, err)) } return refs, nil } +// fetchBounded is SmartFetch's transfer, bounded by gitCallTimeout: see network_bound.go. +func fetchBounded( + ctx context.Context, repo *git.Repository, remoteName string, refSpecs []config.RefSpec, + auth []gitclient.Option, +) error { + ctx, cancel := boundGitCall(ctx) + defer cancel() + err := repo.FetchContext(ctx, &git.FetchOptions{ + RemoteName: remoteName, + ClientOptions: boundToContext(ctx, auth), + RefSpecs: refSpecs, + Depth: 1, + Force: true, + Prune: true, + }) + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return fmt.Errorf("smart fetch failed: %w", boundedCallError(ctx, err)) + } + return nil +} + // analyzeRemoteRefs scans the reference list to find the default branch and check if the target exists. func analyzeRemoteRefs(ctx context.Context, refs []*plumbing.Reference, targetFullStr string) (string, string, bool) { logger := log.FromContext(ctx) diff --git a/internal/git/gittargetignore_writer_test.go b/internal/git/gittargetignore_writer_test.go index f5404e9d..0677dc87 100644 --- a/internal/git/gittargetignore_writer_test.go +++ b/internal/git/gittargetignore_writer_test.go @@ -45,7 +45,7 @@ func TestWriter_IgnoreShadowsManagedPath(t *testing.T) { pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - err = worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false) + err = worker.commitPendingWrites([]PendingWrite{*pendingWrite}) assert.Contains(t, refusalIssueKinds(t, err), manifestanalyzer.IssueIgnoreShadowsManaged) } @@ -65,7 +65,7 @@ func TestWriter_ForeignFileRefused(t *testing.T) { pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - err = worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false) + err = worker.commitPendingWrites([]PendingWrite{*pendingWrite}) assert.Contains(t, refusalIssueKinds(t, err), manifestanalyzer.IssueForeignFile) } @@ -87,7 +87,7 @@ func TestWriter_IgnoredForeignFileAllowsWrite(t *testing.T) { pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) } // TestWriter_BenignPassengerAllowsWrite proves the live writer accepts a folder whose only @@ -107,5 +107,5 @@ func TestCommitPendingWrites_BenignPassengerAllowsWrite(t *testing.T) { pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) } diff --git a/internal/git/helpers_test.go b/internal/git/helpers_test.go index 2ffedbd3..796a1227 100644 --- a/internal/git/helpers_test.go +++ b/internal/git/helpers_test.go @@ -398,3 +398,35 @@ func createTestEvent(tb testing.TB, name string) Event { }, } } + +// finalizeOpenWindow closes the open window and commits it, as the wake that closed it would, without +// pushing. It reports whether the window's write is still pending afterwards: false when the window was +// dropped before it was decided, or its commit failed for good. +func (l *branchWorkerEventLoop) finalizeOpenWindow() bool { + before := len(l.pendingWrites) + l.finalizeOpenWindowWithReason(windowFinalizeReasonUnspecified) + if len(l.pendingWrites) == before { + return false + } + // A window's write always carries events, and their backing array identifies it wherever the + // write moves among the pending writes. + events := l.pendingWrites[len(l.pendingWrites)-1].Events + l.commitDecided() + for i := range l.pendingWrites { + if len(l.pendingWrites[i].Events) > 0 && &l.pendingWrites[i].Events[0] == &events[0] { + return true + } + } + return false +} + +// resyncItem is the FIFO item of a resync queued without coalescing. +func resyncItem(req *ResyncRequest) WorkItem { + return WorkItem{Resync: &resyncMarker{request: req}} +} + +// takenResync is the request a marker runs, taken as the loop takes it. +func takenResync(w *BranchWorker, marker *resyncMarker) *ResyncRequest { + request, _ := w.takePendingResync(marker) + return request +} diff --git a/internal/git/intake.go b/internal/git/intake.go new file mode 100644 index 00000000..bf48e703 --- /dev/null +++ b/internal/git/intake.go @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "fmt" + "sync" +) + +// This file is the branch's intake gate: whether new payload may enter the worker's FIFO. See +// docs/design/gittarget-branch-worker-pending-writes.md, "Recovery contract". +// +// Keeping decided writes through an outage makes the worker's memory grow for as long as the outage +// lasts, and nothing it accepted may be evicted: a decided write can carry a save, and dropping one +// would need a snapshot to re-derive it. So the bound is at intake. While a failed attempt waits +// for its retry, a payload that would take what the branch holds past its retained-byte budget is +// refused, and the refusal pauses the branch: every later payload is refused too, until a +// publication lands. A healthy branch never pauses, because its pending writes drain at the next push. +// +// What the budget counts is everything accepted and not yet published: the payload on the FIFO the +// loop has not handled yet (charged at enqueue, so producers cannot fill the FIFO between two loop +// iterations), and what the loop holds (the open window, the pending writes, deferred heals, saves waiting for +// a window). Every item is charged a fixed overhead beyond its payload, so the budget bounds how +// many items are kept as well as their bytes. The charge is an estimate of the serialized YAML, not +// of the heap: an object's in-memory form is several times larger. +// +// Pausing is a latch, released only when the retry clears: a publication landed, or nothing is owed +// any more. Room under the budget again, a partial replay, or a remote that can be read but still +// refuses the push prove nothing about whether the backlog can drain. A refused producer keeps its +// work: the watch keeps its cursor and delivers again, the controller sends a save again, a refused +// resync's collection is gathered again. Producers wait on IntakePaused for the reopening instead +// of offering the same work again on a timer. +// +// Lifecycle work is not payload and is never refused here: a withdrawal, a refresh tick, shutdown. +// While the branch is paused nothing new enters the FIFO, and the loop keeps draining it, so they +// find room in it. + +// intakeGate is the branch's intake state. Enqueue paths take its lock under pendingResyncsMu; the +// loop takes it alone. +type intakeGate struct { + mu sync.Mutex + // outage is published by the loop: a failed attempt is waiting for its retry. + outage bool + // held is published by the loop: what it holds that is not published yet. See heldBytes. + held int64 + // queued is the payload accepted onto the FIFO that the loop has not finished handling. + queued int64 + // paused is non-nil while intake is paused, and closed when it reopens. + paused chan struct{} +} + +// admit charges a payload against the budget and reports whether it may enter. A refused payload +// pauses the branch. A charge that is not admitted is not counted; one admitted is counted until +// release returns it. +func (g *intakeGate) admit(charge, budget int64) bool { + g.mu.Lock() + defer g.mu.Unlock() + if g.paused != nil { + return false + } + if g.outage && budget > 0 && g.held+g.queued+charge > budget { + g.paused = make(chan struct{}) + return false + } + g.queued += charge + return true +} + +// release returns a charge whose item did not enter the FIFO after all, or that no loop will handle. +// An item the loop handled is released by sync instead, in the same step that counts what it left +// behind. +func (g *intakeGate) release(charge int64) { + g.mu.Lock() + defer g.mu.Unlock() + g.queued -= charge +} + +// sync is the loop's publication, once per iteration: whether a retry is pending, what it holds, and +// the charge of the item it handled, whose work held now counts. Moving that charge from queued to +// held is one step under the lock, so no producer ever sees the item counted in neither. It pauses a +// branch whose held work already fills the budget when the outage begins, and reopens a paused one +// once the retry clears. +func (g *intakeGate) sync(released int64, outage bool, held, budget int64) { + g.mu.Lock() + defer g.mu.Unlock() + g.queued -= released + g.outage, g.held = outage, held + switch { + case g.paused != nil && !outage: + close(g.paused) + g.paused = nil + case g.paused == nil && outage && budget > 0 && held+g.queued >= budget: + g.paused = make(chan struct{}) + } +} + +// pausedC is the channel closed when a paused branch reopens, or nil while intake is open. +func (g *intakeGate) pausedC() <-chan struct{} { + g.mu.Lock() + defer g.mu.Unlock() + return g.paused +} + +// IntakePaused reports a branch that has paused intake: it returns a channel closed when intake +// reopens, or nil while it is open. A producer refused while the branch is paused waits on it +// instead of offering its work again on a timer, and checks again once woken, because the branch +// can pause again before it gets there. +func (w *BranchWorker) IntakePaused() <-chan struct{} { return w.intake.pausedC() } + +// admitPayload charges an item against the budget. Callers hold pendingResyncsMu. +func (w *BranchWorker) admitPayload(charge int64) bool { + return w.intake.admit(charge, w.branchBufferMaxBytes) +} + +// intakeRefusal is what a refused payload's producer is told: the branch is paused, or this item +// alone is larger than the whole budget, which no amount of draining makes room for while the +// outage lasts. +func (w *BranchWorker) intakeRefusal(charge int64) error { + if budget := w.branchBufferMaxBytes; budget > 0 && charge > budget { + return fmt.Errorf("%w: it takes %d bytes, which exceeds the branch's whole retained-byte budget "+ + "of %d; it is accepted once the branch publishes again", ErrFinalizeQueueFull, charge, budget) + } + return errAdmissionClosed +} + +// eventCharge is what a queued live or atomic write is charged: what keeping each of its events +// costs, as a window or a decided write keeps it. +func eventCharge(request *WriteRequest) int64 { + var charge int64 + for i := range request.Events { + charge += eventRetainedCharge(&request.Events[i]) + } + return charge +} + +// eventRetainedCharge is what keeping one event costs: its payload, and the overhead of one kept +// item, which a delete with no object still carries. +func eventRetainedCharge(e *Event) int64 { + return eventPayloadSize(e) + pendingWriteOverheadBytes +} + +// attachCharge is what a save is charged from enqueue until its message is in a decided write: its +// message, and the overhead of one kept item. A save waiting for a window is charged on its own, and +// one attached to the open window is in the window's charge. +func attachCharge(message string) int64 { + return int64(len(message)) + pendingWriteOverheadBytes +} + +// eventPayloadSize is the event object's estimated serialized size, computed once: the producer +// computes it at enqueue, off the loop, and the loop reuses it. +func eventPayloadSize(e *Event) int64 { + if !e.sized { + e.payloadBytes = estimateObjectSize(e) + e.sized = true + } + return e.payloadBytes +} + +// payloadSize is the snapshot's estimated serialized size, computed once like an event's. +func (r *ResyncRequest) payloadSize() int64 { + if !r.sized { + r.payloadBytes = estimateDesiredSize(r.Desired) + r.sized = true + } + return r.payloadBytes +} + +// charge is what a queued resync is charged: its snapshot, and the overhead of one kept item. +func (r *ResyncRequest) charge() int64 { return r.payloadSize() + pendingWriteOverheadBytes } + +// heldBytes is what the loop holds that is not published yet: the open window, the pending writes, the +// deferred heals' snapshots, and the saves waiting for a window. An attached save is in the charge +// of the window it is attached to, and then of the write that window became. +func (l *branchWorkerEventLoop) heldBytes() int64 { + held := l.windowBytes + l.pendingWritesBytes + for _, heal := range l.deferredHeals { + held += heal.charge() + } + for _, pcr := range l.pendingCRs { + if !pcr.attached { + held += attachCharge(pcr.message) + } + } + return held +} + +// syncAdmission publishes the loop's state to the intake gate, and from there to the publication +// report and its gauges, releasing the charge of the item the wake handled. Run once per loop +// iteration. +func (l *branchWorkerEventLoop) syncAdmission(released int64) { + held := l.heldBytes() + l.w.intake.sync(released, l.retry.pending(), held, l.w.branchBufferMaxBytes) + l.publishPublication(held) +} diff --git a/internal/git/intake_test.go b/internal/git/intake_test.go new file mode 100644 index 00000000..240af2b6 --- /dev/null +++ b/internal/git/intake_test.go @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// The branch's intake gate during an outage: what a worker accepts is bounded at enqueue, intake +// stays paused until a publication lands, and the producers it refused are woken when it reopens. +// See docs/design/gittarget-branch-worker-pending-writes.md, "Recovery contract". + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// outageLoop is a worker whose remote cannot be reached, with one write pending and a +// retry pending: the state in which the budget applies. +func outageLoop(t *testing.T, slug string) (*ledgerFixture, *branchWorkerEventLoop, func()) { + t.Helper() + f := newLedgerFixture(t, slug, true) + f.createLedgerTarget("team-a", nil) + f.publish("prime") + f.worker.markWorktreeDirty("a write failed part-way and could not be undone") + _, restoreSyncs := failSyncs(t) + loop := newBranchWorkerEventLoop(f.worker, time.Hour) + t.Cleanup(loop.stopTimers) + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("held-during-the-outage", "alice", ledgerTargetName)}, + CommitMode: CommitModePerEvent, + }}) + loop.finalizeOpenWindow() + loop.publishLoopState(0) + require.True(t, loop.retry.pending()) + require.Len(t, loop.pendingWrites, 1) + return f, loop, restoreSyncs +} + +// The budget bounds what a worker accepts at enqueue. It used to be read only once per loop +// iteration, against what the loop already held, so producers could fill the whole FIFO between +// two iterations however far past the budget that went. +func TestIntake_TheBudgetBoundsAcceptedWorkAtEnqueue(t *testing.T) { + f, loop, _ := outageLoop(t, "intake-bounded-at-enqueue") + budget := loop.heldBytes() + 10*1024 + f.worker.branchBufferMaxBytes = budget + loop.publishLoopState(0) + require.Nil(t, f.worker.IntakePaused(), "below the budget, intake is open") + + const offered = 50 + admitted := 0 + for i := range offered { + if f.worker.Enqueue(configMapTargetEvent(fmt.Sprintf("offered-%d", i), "alice", ledgerTargetName)) { + admitted++ + } + } + assert.Positive(t, admitted, "the room under the budget is used") + assert.Less(t, admitted, offered, "what does not fit is refused at enqueue, before the loop runs") + assert.NotNil(t, f.worker.IntakePaused(), "and the refusal pauses the branch") + + for len(f.worker.eventQueue) > 0 { + loop.handleQueueItem(<-f.worker.eventQueue) + } + assert.LessOrEqual(t, loop.heldBytes(), budget, "everything accepted fits the budget once the loop holds it") +} + +// Everything the loop keeps counts against the budget: the open window, the pending writes, a deferred heal's +// snapshot, and a save registered while it waits for a window. +func TestIntake_EveryKindOfHeldWorkCounts(t *testing.T) { + _, loop, _ := outageLoop(t, "intake-held-kinds") + held := loop.heldBytes() + + req := attachReq("alice", time.Hour) + req.GitTargetName = ledgerTargetName + req.Message = "save once the window opens" + loop.handleAttachCommitRequest(req) + require.Contains(t, loop.pendingCRs, req.id()) + assert.GreaterOrEqual(t, loop.heldBytes()-held, int64(pendingWriteOverheadBytes), + "a registered save is charged for what it keeps") + held = loop.heldBytes() + + heal := &ResyncRequest{GitTargetName: ledgerTargetName, GitTargetNamespace: "default", Heal: true, + Desired: []manifestanalyzer.DesiredResource{{Object: bigConfigMap("deferred", 4096)}}, + Result: make(chan ResyncResult, 1)} + loop.openWindow = &openWindow{} // a heal waits for the window to close + loop.handleResyncRequest(heal) + loop.endWake(0) + require.Len(t, loop.deferredHeals, 1) + assert.GreaterOrEqual(t, loop.heldBytes()-held, int64(4096), "a deferred heal's snapshot is held") + held = loop.heldBytes() + + loop.windowBytes = 2048 + assert.Equal(t, held+2048, loop.heldBytes(), "and so is the open window") +} + +// A paused branch reopens only once a publication lands. A budget that has room again, or a remote +// that can be read but still refuses the push, proves nothing about whether the backlog can drain. +func TestIntake_StaysPausedUntilAPublicationLands(t *testing.T) { + f, loop, restoreSyncs := outageLoop(t, "intake-paused-until-published") + f.worker.branchBufferMaxBytes = 1 + loop.publishLoopState(0) + paused := f.worker.IntakePaused() + require.NotNil(t, paused) + + f.worker.branchBufferMaxBytes = 1 << 40 + loop.publishLoopState(0) + assert.NotNil(t, f.worker.IntakePaused(), "room under the budget does not reopen a paused branch") + + restoreSyncs() + restorePushes := failPushes(t, f.worker) + fireRetry(loop) + loop.publishLoopState(0) + require.NotEmpty(t, loop.pendingWrites) + assert.NotNil(t, f.worker.IntakePaused(), "a remote that can be read but refuses the push stays paused") + assert.True(t, loop.retry.pending(), "and the backoff continues") + + restorePushes() + fireRetry(loop) + loop.publishLoopState(0) + require.Empty(t, loop.pendingWrites) + assert.Nil(t, f.worker.IntakePaused(), "a landed publication reopens intake") + select { + case <-paused: + default: + t.Fatal("the producers waiting on the pause are woken") + } +} + +// A snapshot larger than the whole budget is accepted on a healthy branch, whose pending writes drain +// at the next push. During an outage it can never fit, so it is refused with that said, pauses the branch +// rather than being offered again on every reconnect, and is accepted once a publication lands. +func TestIntake_ASnapshotLargerThanTheBudget(t *testing.T) { + f, loop, restoreSyncs := outageLoop(t, "intake-oversized-snapshot") + f.worker.branchBufferMaxBytes = loop.heldBytes() + 4096 + loop.publishLoopState(0) + require.Nil(t, f.worker.IntakePaused()) + + oversized := func() *ResyncRequest { + return &ResyncRequest{GitTargetName: ledgerTargetName, GitTargetNamespace: "default", + Desired: []manifestanalyzer.DesiredResource{{Object: bigConfigMap("huge", 64*1024)}}, + Result: make(chan ResyncResult, 1)} + } + refused := oversized() + require.False(t, f.worker.EnqueueResync(refused)) + err := (<-refused.Result).Err + require.ErrorIs(t, err, ErrFinalizeQueueFull) + assert.Contains(t, err.Error(), "exceeds", "the refusal says the snapshot cannot fit") + assert.NotNil(t, f.worker.IntakePaused(), "the refusal pauses the branch") + + restoreSyncs() + fireRetry(loop) + loop.publishLoopState(0) + require.Nil(t, f.worker.IntakePaused()) + assert.True(t, f.worker.EnqueueResync(oversized()), "a healthy branch accepts it") +} + +// Lifecycle work is not payload: while intake is paused, a withdrawal still enters the FIFO and +// cancels a save that is only waiting for a window. +func TestIntake_AWithdrawalProgressesWhilePaused(t *testing.T) { + f, loop, _ := outageLoop(t, "intake-withdraw-while-paused") + req := attachReq("alice", time.Hour) + req.GitTargetName = ledgerTargetName + loop.handleAttachCommitRequest(req) + f.worker.branchBufferMaxBytes = 1 + loop.publishLoopState(0) + require.NotNil(t, f.worker.IntakePaused()) + + f.worker.EnqueueWithdraw(req) + require.Len(t, f.worker.eventQueue, 1, "a withdrawal is not refused by the pause") + loop.handleQueueItem(<-f.worker.eventQueue) + outcome, ok := f.worker.LookupCommitRequestOutcome(req.Namespace, req.Name, req.UID) + require.True(t, ok) + assert.ErrorIs(t, outcome.Err, ErrCommitRequestWithdrawn) +} + +// bigConfigMap is a ConfigMap whose data is about size bytes. +func bigConfigMap(name string, size int) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("v1") + obj.SetKind("ConfigMap") + obj.SetName(name) + obj.SetNamespace("default") + obj.Object["data"] = map[string]interface{}{"blob": strings.Repeat("x", size)} + return obj +} + +// A handled item's charge moves from the FIFO's count to what the loop holds in one step. It used +// to be released before the loop published what the item left behind, and a producer running in +// between found room the branch did not have: 35 KB held against an 18 KB budget. +func TestIntake_AHandledItemsChargeMovesToHeldInOneStep(t *testing.T) { + f, loop, _ := outageLoop(t, "intake-charge-transfer") + first := configMapTargetEvent("event-1", "alice", ledgerTargetName) + first.Object = bigConfigMap("event-1", 16*1024) + f.worker.branchBufferMaxBytes = loop.heldBytes() + eventCharge(&WriteRequest{Events: []Event{first}}) + loop.publishLoopState(0) + + require.True(t, f.worker.Enqueue(first)) + loop.handleQueueItem(<-f.worker.eventQueue) + f.worker.intake.mu.Lock() + queued, held := f.worker.intake.queued, f.worker.intake.held + f.worker.intake.mu.Unlock() + assert.Zero(t, queued, "the handled item is no longer counted on the FIFO") + assert.Equal(t, loop.heldBytes(), held, "it is counted in what the loop holds instead") + + second := configMapTargetEvent("event-2", "alice", ledgerTargetName) + second.Object = bigConfigMap("event-2", 16*1024) + assert.False(t, f.worker.Enqueue(second), "the first item still fills the budget") + assert.LessOrEqual(t, loop.heldBytes(), f.worker.branchBufferMaxBytes) +} + +// An event costs an item's overhead to keep whether or not it carries an object. A delete carries +// none, and a window used to charge it nothing, so a window admitted any number of them: 100 +// against a budget with room for two. +func TestIntake_AWindowChargesTheEventsItKeeps(t *testing.T) { + f, loop, _ := outageLoop(t, "intake-window-deletes") + f.worker.branchBufferMaxBytes = loop.heldBytes() + 2*pendingWriteOverheadBytes + loop.publishLoopState(0) + + accepted := 0 + for i := range 100 { + if !f.worker.Enqueue(deleteEvent(fmt.Sprintf("deleted-%d", i))) { + break + } + accepted++ + loop.handleQueueItem(<-f.worker.eventQueue) + } + assert.Equal(t, 2, accepted, "the loop keeps what intake admitted, and charges it") + assert.LessOrEqual(t, loop.heldBytes(), f.worker.branchBufferMaxBytes) +} + +// The window keeps one event per path, so an edit to a path it holds replaces that event's charge +// instead of adding to it. +func TestIntake_AWindowChargesAReplacedEventOnce(t *testing.T) { + _, loop, _ := outageLoop(t, "intake-window-replaced") + edit := func() { + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{deleteEvent("edited-twice")}, CommitMode: CommitModePerEvent, + }}) + } + edit() + require.NotNil(t, loop.openWindow) + once := loop.windowBytes + edit() + assert.Equal(t, once, loop.windowBytes) +} + +// deleteEvent is a live delete for the ledger target, which carries no object. +func deleteEvent(name string) Event { + event := configMapTargetEvent(name, "alice", ledgerTargetName) + event.Operation = "DELETE" + event.Object = nil + return event +} + +// A save attached to a window is the window's to keep, and stays charged until its message is in +// the decided write's charge. Attaching it used to stop its charge while the window collected. +func TestIntake_AnAttachedSaveStaysCharged(t *testing.T) { + f, loop, _ := outageLoop(t, "intake-attached-save") + req := attachReq("alice", time.Hour) + req.GitTargetName = ledgerTargetName + req.Message = strings.Repeat("x", 1024) + f.worker.EnqueueAttach(req) + loop.handleQueueItem(<-f.worker.eventQueue) + waiting := loop.heldBytes() + + require.True(t, f.worker.Enqueue(configMapTargetEvent("opens-window", "alice", ledgerTargetName))) + loop.handleQueueItem(<-f.worker.eventQueue) + require.NotNil(t, loop.openWindow) + require.NotNil(t, loop.openWindow.pendingCR, "the save attached to the window") + assert.Greater(t, loop.heldBytes(), waiting, "the window holds the save and its event") + + require.True(t, loop.finalizeOpenWindow()) + assert.Greater(t, loop.heldBytes(), waiting, "and so does the write the window became") +} + +// A coalesced resync's FIFO position does not keep the request it replaced. The queue held the +// first request of each key, and its snapshot with it, while the budget was charged only for the +// newest: ten 32 KB snapshots replaced by empty ones kept 350 KB against a 67 KB budget. +func TestEnqueueResync_ACoalescedSnapshotIsNotKeptByTheQueue(t *testing.T) { + f, loop, _ := outageLoop(t, "resync-coalesced-snapshot") + f.worker.branchBufferMaxBytes = loop.heldBytes() + 64*1024 + loop.publishLoopState(0) + for i := range 10 { + collection := itypes.CollectionKey{Resource: "configmaps", Namespace: fmt.Sprintf("ns-%d", i)} + scope := &ResyncScope{Collection: collection} + large := &ResyncRequest{ + GitTargetName: ledgerTargetName, GitTargetNamespace: "default", Scope: scope, + Desired: []manifestanalyzer.DesiredResource{{Object: bigConfigMap("huge", 32*1024)}}, + Result: make(chan ResyncResult, 1), + } + small := &ResyncRequest{ + GitTargetName: ledgerTargetName, GitTargetNamespace: "default", Scope: scope, + Result: make(chan ResyncResult, 1), + } + require.True(t, f.worker.EnqueueResync(large)) + require.True(t, f.worker.EnqueueResync(small)) + } + + var reachable int64 + for len(f.worker.eventQueue) > 0 { + item := <-f.worker.eventQueue + require.NotNil(t, item.Resync) + reachable += item.Resync.request.charge() + } + f.worker.intake.mu.Lock() + queued := f.worker.intake.queued + f.worker.intake.mu.Unlock() + assert.Equal(t, queued, reachable, "the queue keeps exactly the snapshots it is charged for") + assert.LessOrEqual(t, loop.heldBytes()+reachable, f.worker.branchBufferMaxBytes) +} diff --git a/internal/git/network_bound.go b/internal/git/network_bound.go new file mode 100644 index 00000000..8566c3fe --- /dev/null +++ b/internal/git/network_bound.go @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + "errors" + "fmt" + "net" + "slices" + "time" + + gitclient "github.com/go-git/go-git/v6/plumbing/client" + "github.com/go-git/go-git/v6/plumbing/transport" +) + +// Every call that reaches a Git server is bounded, because the branch worker makes them +// synchronously on its event loop: a server that accepts a connection and then stalls would +// otherwise hold every target on the branch, its saves, withdrawals, timers and later writes, +// for as long as the TCP connection lives. Shutdown waits on that loop too. +// +// Measured against go-git v6.0.0-alpha.5 (docs/design/gittarget-branch-worker-pending-writes.md, step 6): +// +// - HTTP honors the context in every phase (advertisement, upload-pack, receive-pack) when the +// context-taking API is used. Remote.List and Repository.Fetch take none, and over the default +// HTTP client they hung indefinitely. +// - SSH uses the context only to dial. The SSH handshake and every read of the git protocol +// ignore it, so a context alone hangs on a server that never sends its banner, and on one +// that completes the handshake and then says nothing. +// +// So a bounded call needs both: the context-taking API, and a connection that closes when the +// context ends (boundToContext). Closing the socket fails the blocked read on the calling +// goroutine; nothing runs the operation on another goroutine, so nothing outlives the call with +// the checkout in hand. + +//nolint:gochecknoglobals // vars, not consts, so a test can shorten them. +var ( + // gitCallTimeout bounds one round trip to a Git server: an advertisement, a fetch, or one push + // session. A depth-1 fetch of a large repository over a slow link is the long case it must cover. + gitCallTimeout = 2 * time.Minute + // gitPublishTimeout bounds one whole push cycle, contention retries and their replays included, + // so three slow attempts cannot each start a fresh budget. + gitPublishTimeout = 5 * time.Minute +) + +// boundGitCall derives the context for one call to a Git server. A tighter deadline the caller +// already carries (a push cycle's) wins. +func boundGitCall(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(ctx, gitCallTimeout) +} + +// boundToContext returns auth with a dialer whose connections close when ctx ends. It matters for +// SSH (and git://), whose transports read without consulting the context; HTTP ignores the dialer +// option and honors the context itself. +func boundToContext(ctx context.Context, auth []gitclient.Option) []gitclient.Option { + return append(slices.Clip(auth), gitclient.WithDialer(contextBoundDialer(ctx))) +} + +// contextBoundDialer dials with the transport's own dial context, then ties the connection to the +// operation's context. The dial context is not the one to tie it to: go-git cancels it as soon as +// the dial returns. +func contextBoundDialer(operation context.Context) transport.DialContextFunc { + return func(dialCtx context.Context, network, address string) (net.Conn, error) { + conn, err := (&net.Dialer{}).DialContext(dialCtx, network, address) + if err != nil { + return nil, err + } + stop := context.AfterFunc(operation, func() { _ = conn.Close() }) + return &contextBoundConn{Conn: conn, stop: stop}, nil + } +} + +// contextBoundConn is a connection closed by its operation's context. Closing it normally +// releases that tie. +type contextBoundConn struct { + net.Conn + + stop func() bool +} + +func (c *contextBoundConn) Close() error { + c.stop() + return c.Conn.Close() +} + +// boundedCallError names the bound when it is what ended a call. A connection closed under an SSH +// session surfaces as EOF or a closed-connection error, which says nothing about why. +func boundedCallError(ctx context.Context, err error) error { + if err == nil || ctx.Err() == nil || errors.Is(err, ctx.Err()) { + return err + } + return fmt.Errorf("%w: %w", ctx.Err(), err) +} diff --git a/internal/git/network_bound_test.go b/internal/git/network_bound_test.go new file mode 100644 index 00000000..797a711c --- /dev/null +++ b/internal/git/network_bound_test.go @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "io" + "net" + "net/http" + "net/http/httptest" + "os/exec" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/config" + "github.com/go-git/go-git/v6/plumbing" + gitclient "github.com/go-git/go-git/v6/plumbing/client" + "github.com/go-git/go-git/v6/plumbing/transport" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gossh "golang.org/x/crypto/ssh" +) + +// A stalled Git server must hand control back to the branch worker. See network_bound.go for the +// measurement this pins: HTTP honors a context only through go-git's context-taking API, and SSH +// does not honor one at all past the dial. + +const ( + stallBound = 300 * time.Millisecond + // stallGuard is how long a call may take before the test calls it hung. It is generous: the + // assertion is "returns at all", and the deadline is checked separately through the error. + stallGuard = 5 * time.Second +) + +// shortenGitBounds sets both bounds for one test. +func shortenGitBounds(t *testing.T, call, publish time.Duration) { + t.Helper() + oldCall, oldPublish := gitCallTimeout, gitPublishTimeout + gitCallTimeout, gitPublishTimeout = call, publish + t.Cleanup(func() { gitCallTimeout, gitPublishTimeout = oldCall, oldPublish }) +} + +// requireReturns runs op and fails the test if it does not come back within stallGuard. +func requireReturns(t *testing.T, name string, op func() error) error { + t.Helper() + done := make(chan error, 1) + go func() { done <- op() }() + select { + case err := <-done: + return err + case <-time.After(stallGuard): + t.Fatalf("%s did not return within %v against a stalled server", name, stallGuard) + return nil + } +} + +// holdUntilClientLeaves stalls a request: it answers nothing until the client gives up. It reads +// the body first, because the server only notices a client hanging up once the body is consumed. +func holdUntilClientLeaves(_ http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + <-r.Context().Done() +} + +// stallPhase installs a hook on sim that holds every request matching match. +func stallPhase(sim *adoSimulator, match func(*http.Request) bool) { + around := func(w http.ResponseWriter, r *http.Request, backend http.Handler) { + if match(r) { + holdUntilClientLeaves(w, r) + return + } + backend.ServeHTTP(w, r) + } + sim.around.Store(&around) +} + +func isInfoRefs(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/info/refs") } +func isUploadPack(r *http.Request) bool { + return r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/git-upload-pack") +} +func isReceivePack(r *http.Request) bool { + return r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/git-receive-pack") +} + +// Before step 6 SmartFetchFrom, CheckRepo and the advertisement called go-git's context-free List +// and Fetch, and each of these hung past its context. +func TestGitNetworkCalls_HTTPStallsReturnAtTheCallBound(t *testing.T) { + shortenGitBounds(t, stallBound, time.Minute) + projectRoot, repoDir := newADORepo(t) + sim := startRealGitServer(t, projectRoot, repoDir) + clone, err := git.PlainClone(t.TempDir(), &git.CloneOptions{URL: sim.RepoURL}) + require.NoError(t, err) + main := plumbing.NewBranchReferenceName("main") + head, err := clone.Head() + require.NoError(t, err) + ctx := context.Background() + + calls := []struct { + name string + stall func(*http.Request) bool + call func() error + }{ + {"SmartFetchFrom, advertisement", isInfoRefs, func() error { + _, err := SmartFetchFrom(ctx, clone, main, "", nil) + return err + }}, + {"SmartFetchFrom, transfer", isUploadPack, func() error { + _, err := SmartFetchFrom(ctx, clone, main, "", nil) + return err + }}, + {"CheckRepo", isInfoRefs, func() error { + _, err := CheckRepo(ctx, sim.RepoURL, nil) + return err + }}, + {"advertiseRemoteBranch", isInfoRefs, func() error { + _, err := advertiseRemoteBranch(ctx, sim.RepoURL, main, "", nil) + return err + }}, + {"PushAtomic, advertisement", isInfoRefs, func() error { + _, err := PushAtomic(ctx, clone, head.Hash(), main, nil) + return err + }}, + } + for _, c := range calls { + t.Run(c.name, func(t *testing.T) { + // The transfer only stalls when there is something to transfer. + if c.name == "SmartFetchFrom, transfer" { + simulateClientCommitOnDisk(t, repoDir, "main", "moved-"+strconv.Itoa(time.Now().Nanosecond()), "x\n") + } + stallPhase(sim, c.stall) + t.Cleanup(func() { sim.around.Store(nil) }) + err := requireReturns(t, c.name, c.call) + require.Error(t, err) + require.ErrorIs(t, err, context.DeadlineExceeded) + }) + } +} + +// insecureSSHAuth is a password auth that trusts any host key, with the host key algorithms set so +// go-git does not consult an on-disk known_hosts. +type insecureSSHAuth struct{} + +func (insecureSSHAuth) ClientConfig(context.Context, *transport.Request) (*gossh.ClientConfig, error) { + return &gossh.ClientConfig{ + User: "git", + Auth: []gossh.AuthMethod{gossh.Password("x")}, + HostKeyCallback: gossh.InsecureIgnoreHostKey(), + HostKeyAlgorithms: []string{gossh.KeyAlgoED25519}, + }, nil +} + +// silentTCP accepts connections and never writes a byte: no SSH banner. +func silentTCP(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + t.Cleanup(func() { _ = c.Close() }) + } + }() + return ln.Addr().String() +} + +// silentSSH completes the SSH handshake and accepts the exec request, then sends nothing: the git +// protocol stalls. +func silentSSH(t *testing.T) string { + t.Helper() + _, priv, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + signer, err := gossh.NewSignerFromKey(priv) + require.NoError(t, err) + cfg := &gossh.ServerConfig{PasswordCallback: func(gossh.ConnMetadata, []byte) (*gossh.Permissions, error) { + return &gossh.Permissions{}, nil + }} + cfg.AddHostKey(signer) + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + t.Cleanup(func() { _ = c.Close() }) + go serveSilentSSH(c, cfg) + } + }() + return ln.Addr().String() +} + +func serveSilentSSH(c net.Conn, cfg *gossh.ServerConfig) { + _, chans, reqs, err := gossh.NewServerConn(c, cfg) + if err != nil { + return + } + go gossh.DiscardRequests(reqs) + for nc := range chans { + ch, chReqs, err := nc.Accept() + if err != nil { + continue + } + go func() { + defer func() { _ = ch.Close() }() + for r := range chReqs { + _ = r.Reply(true, nil) + } + }() + } +} + +// sshCloneOf is a local repository whose origin is url, so the fetch and push paths have a +// checkout to work from without a server that can serve one. +func sshCloneOf(t *testing.T, url string) *git.Repository { + t.Helper() + repo, err := git.PlainInit(t.TempDir(), false) + require.NoError(t, err) + _, err = repo.CreateRemote(&config.RemoteConfig{Name: "origin", URLs: []string{url}}) + require.NoError(t, err) + return repo +} + +// go-git's SSH transport uses the context only to dial, so before step 6 every one of these hung +// with a context that had long expired. The connection is now closed when the call's bound ends. +func TestGitNetworkCalls_SSHStallsReturnAtTheCallBound(t *testing.T) { + shortenGitBounds(t, stallBound, time.Minute) + auth := []gitclient.Option{gitclient.WithSSHAuth(insecureSSHAuth{})} + main := plumbing.NewBranchReferenceName("main") + ctx := context.Background() + + for _, server := range []struct { + name string + addr string + }{ + {"no SSH banner", silentTCP(t)}, + {"SSH handshake done, git silent", silentSSH(t)}, + } { + url := "ssh://git@" + server.addr + "/repo.git" + repo := sshCloneOf(t, url) + for _, c := range []struct { + name string + call func() error + }{ + {"CheckRepo", func() error { _, err := CheckRepo(ctx, url, auth); return err }}, + {"SmartFetchFrom", func() error { _, err := SmartFetchFrom(ctx, repo, main, "", auth); return err }}, + {"advertiseRemoteBranch", func() error { + _, err := advertiseRemoteBranch(ctx, url, main, "", auth) + return err + }}, + {"PushAtomic", func() error { + _, err := PushAtomic(ctx, repo, plumbing.ZeroHash, main, auth) + return err + }}, + } { + t.Run(server.name+"/"+c.name, func(t *testing.T) { + err := requireReturns(t, c.name, c.call) + require.Error(t, err) + require.ErrorIs(t, err, context.DeadlineExceeded) + }) + } + } +} + +// Worker shutdown cancels the worker's context. A stalled SSH call used to ignore that too, so +// Stop, which waits for the loop, waited for the TCP connection to die on its own. +func TestGitNetworkCalls_ASSHStallEndsWhenTheWorkerIsCancelled(t *testing.T) { + shortenGitBounds(t, time.Hour, time.Hour) + auth := []gitclient.Option{gitclient.WithSSHAuth(insecureSSHAuth{})} + url := "ssh://git@" + silentSSH(t) + "/repo.git" + + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(stallBound, cancel) + err := requireReturns(t, "CheckRepo", func() error { + _, err := CheckRepo(ctx, url, auth) + return err + }) + require.Error(t, err) + require.ErrorIs(t, err, context.Canceled) +} + +// A push whose reply is lost after the server applied it is an uncertain outcome. The cycle's +// probe finds the branch at the commits it sent, and settles the push as published instead of +// replaying them. Replaying would re-plan the same writes onto a tree that already holds them, and a +// save's empty commit would land on the remote twice. +func TestPushCycle_ALostReplyThatLandedIsPublishedNotReplayed(t *testing.T) { + shortenGitBounds(t, stallBound, time.Minute) + f := newLedgerFixture(t, "lost-reply", true) + f.createLedgerTarget("apps", nil) + + record, err := f.worker.buildRequestRecordWrite(f.worker.ctx, &pendingCommitRequest{ + id: commitRequestID{Namespace: "default", Name: "save", UID: "save-uid"}, + author: "alice", + gitTargetName: ledgerTargetName, + gitTargetNamespace: "default", + message: "save", + }) + require.NoError(t, err) + require.NoError(t, f.worker.commitPendingWrites([]PendingWrite{*record})) + local := workerHead(t, f.worker) + before := countCommits(t, f.repoDir) + + var swallowed atomic.Int32 + around := func(w http.ResponseWriter, r *http.Request, backend http.Handler) { + if isReceivePack(r) && swallowed.Add(1) == 1 { + backend.ServeHTTP(httptest.NewRecorder(), r) // the server applies the push... + holdUntilClientLeaves(w, r) // ...and the reply never arrives + return + } + backend.ServeHTTP(w, r) + } + f.sim.around.Store(&around) + + err = requireReturns(t, "pushPendingCommits", func() error { + return f.worker.pushPendingCommits([]PendingWrite{*record}) + }) + require.NoError(t, err, "the probe proves the push landed") + assert.Equal(t, int32(1), swallowed.Load(), "nothing was pushed a second time") + assert.Equal(t, local.String(), revParseMain(t, f.repoDir), "the remote is at the commit we sent") + assert.Equal(t, before+1, countCommits(t, f.repoDir), "the save's empty commit landed exactly once") +} + +// The cycle's deadline covers the whole cycle. With a per-call bound longer than the cycle's, a +// stalled upload still hands control back when the cycle's budget is spent, and the writes stay +// with the caller for the retry. +func TestPushCycle_OneDeadlineCoversTheWholeCycle(t *testing.T) { + f := newLedgerFixture(t, "cycle-deadline", true) + f.commit("cm-a") + shortenGitBounds(t, time.Hour, stallBound) + stallPhase(f.sim, isReceivePack) + + started := time.Now() + err := requireReturns(t, "pushPendingCommits", func() error { + return f.worker.pushPendingCommits(f.pending) + }) + require.Error(t, err) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Less(t, time.Since(started), 2*time.Second, "returned at the cycle's budget, not a call's") + assert.NotEqual(t, workerHead(t, f.worker).String(), revParseMain(t, f.repoDir), "nothing landed") +} + +func workerHead(t *testing.T, w *BranchWorker) plumbing.Hash { + t.Helper() + repo, err := git.PlainOpen(w.repoPath()) + require.NoError(t, err) + _, head, err := GetCurrentBranch(repo) + require.NoError(t, err) + return head +} + +func countCommits(t *testing.T, repoDir string) int { + t.Helper() + out, err := exec.Command("git", "-C", repoDir, "rev-list", "--count", "refs/heads/main").Output() + require.NoError(t, err) + n, err := strconv.Atoi(strings.TrimSpace(string(out))) + require.NoError(t, err) + return n +} diff --git a/internal/git/open_window.go b/internal/git/open_window.go index 8dac2665..8679f247 100644 --- a/internal/git/open_window.go +++ b/internal/git/open_window.go @@ -106,14 +106,16 @@ func (w *openWindow) canAppend(e Event) bool { } // add records an event in the window. If the path was already present the -// event replaces the previous one (last-write-wins inside a single window); -// otherwise pathOrder is extended. -func (w *openWindow) add(e Event) { +// event replaces the previous one (last-write-wins inside a single window), which +// it returns; otherwise pathOrder is extended. +func (w *openWindow) add(e Event) (Event, bool) { key := windowPathKey(e, w.writer) - if _, exists := w.pathToEvent[key]; !exists { + replaced, exists := w.pathToEvent[key] + if !exists { w.pathOrder = append(w.pathOrder, key) } w.pathToEvent[key] = e + return replaced, exists } // orderedEvents returns one event per distinct path, in the order paths were diff --git a/internal/git/parent_recovery.go b/internal/git/parent_recovery.go index b91c5b27..7258a0d9 100644 --- a/internal/git/parent_recovery.go +++ b/internal/git/parent_recovery.go @@ -8,41 +8,27 @@ import ( "time" "github.com/go-git/go-git/v6/plumbing" - - itypes "github.com/ConfigButler/gitops-reverser/internal/types" ) -// This file is the worker's recovery from a parent branch that is missing or does not resolve: -// work it could not write then, and the obligation to write it once the parent exists. +// This file is the worker's recovery from a parent branch that is missing or does not resolve: the +// obligation to write what was decided while it was gone, once it exists. // // An observation flip is not an obligation. "The parent is Found again" says nothing about whether -// the writes held back meanwhile have landed: the first attempt after it can fail transiently, the -// push timer is stopped by a failed push, the refresher skips a branch that holds work, and with -// --git-refresh-interval=0 there is no refresher at all. So the worker latches an obligation the -// moment a cycle fails on the parent, and clears it only when the work is published. -// -// The obligation has two halves: +// the writes decided meanwhile have landed: the first attempt after it can fail transiently, the +// refresher skips a branch that holds work, and with --git-refresh-interval=0 there is no refresher +// at all. So the worker latches an obligation the moment an attempt fails on the parent, and clears +// it only when nothing is owed any more: the pending writes were published, or every one of them +// settled without needing a push (endOutage). // -// - Retained writes: the loop's pendingWrites. Nothing extra is needed to remember them; the -// obligation is open while any are held, and pushPending publishing them is what closes it. -// - Dropped writes, per (GitTarget, collection) scope: a live window or a resync the worker had -// to drop because the parent was missing. The worker cannot re-derive what the cluster holds, -// so it asks the controller for a snapshot (SnapshotRequestSeq), and a scope clears only when -// a resync of it is published: pushed, or settled as a no-op. A successful resync reply alone -// is not enough, because its write may still be waiting for the push. Scopes are per -// collection because a ConfigMap snapshot of target A proves nothing about a Deployment of A, -// or about target B on the same worker. +// The obligation is the pending writes themselves. A write decided while the parent is missing stays +// pending like any other a remote failure holds back (pending_writes_loop.go), and admission +// backpressure bounds them however long the parent stays away, so nothing is dropped and no snapshot +// is owed. // -// The worker probes for the parent itself, on the event loop, with one advertisement per deadline: -// 10s, doubling, capped at 5m, shared by every target on the worker. Until the parent is found, -// nothing else may spend a connection on it: a refresh tick before the deadline is a no-op, and a -// live write or resync fails at once without a fetch. Reconciles never probe. - -const ( - parentProbeInitialBackoff = 10 * time.Second - parentProbeMaxBackoff = 5 * time.Minute - parentProbeBackoffFactor = 2 -) +// The worker probes for the parent itself, on the event loop, with one advertisement per deadline of +// its one retry schedule (retry.go), shared by every target on the worker. Until the parent is +// found, nothing else may spend a connection on it: a refresh tick before the deadline is a no-op, +// and a write or resync is decided without a fetch. Reconciles never probe. // errAwaitingParentProbe is the answer to a cycle that would fetch for a parent already known to be // missing, before the next probe is due. It is a missing parent as far as every caller is @@ -56,14 +42,7 @@ func isParentUnavailable(err error) bool { return errors.Is(err, ErrParentBranchNotFound) || errors.Is(err, ErrDefaultBranchUnresolved) } -// recoveryScope is one slice of a GitTarget whose writes were dropped. A zero collection is a write -// whose collection is unknown, which any resync of the target covers. -type recoveryScope struct { - target itypes.ResourceReference - collection itypes.CollectionKey -} - -// parentRecovery is the loop's half of the obligation. Loop-goroutine only. +// parentRecovery is the loop's latch on the obligation. Loop-goroutine only. type parentRecovery struct { // active is the latch. active bool @@ -71,18 +50,9 @@ type parentRecovery struct { found bool // gen is the parent generation the latch was set under; a change makes the probe due at once. gen uint64 - - // scopes are the dropped writes still owed a published snapshot. - scopes map[recoveryScope]struct{} - // awaitingPush are scopes a resync has re-derived into a commit that is not pushed yet. - awaitingPush map[recoveryScope]struct{} - - backoff time.Duration - nextProbeAt time.Time - timer *time.Timer } -// now is the loop's clock, injectable so the probe schedule can be tested without sleeping. +// now is the loop's clock, injectable so the retry schedule can be tested without sleeping. func (w *BranchWorker) now() time.Time { if w.clock != nil { return w.clock() @@ -90,192 +60,64 @@ func (w *BranchWorker) now() time.Time { return time.Now() } -// noteParentUnavailable latches the obligation when err was caused by the parent, and remembers -// the scopes whose writes were dropped for it. Any other error is not this obligation's concern. -func (l *branchWorkerEventLoop) noteParentUnavailable(err error, scopes ...recoveryScope) { +// noteParentUnavailable latches the obligation when err was caused by the parent and pending writes +// are owed. Any other error is not this obligation's concern, and neither is a failure that left +// nothing owed: an obligation with nothing to publish would never close. It does not schedule the +// next attempt: whoever observed the failure does, once (scheduleRetry), and a new latch starts that +// schedule over so the first probe is one initial backoff away. +func (l *branchWorkerEventLoop) noteParentUnavailable(err error) { if !isParentUnavailable(err) { return } r := &l.recovery - switch { - case !r.active: + if !r.active { + if len(l.pendingWrites) == 0 { + return + } r.active = true - r.backoff = parentProbeInitialBackoff - r.nextProbeAt = l.w.now().Add(r.backoff) r.gen = l.w.parentSnapshot().gen - case r.found: - // Found, then lost again on the attempt that followed: wait a whole backoff before asking. - r.nextProbeAt = l.w.now().Add(r.backoff) + l.clearRetry() } r.found = false - for _, scope := range scopes { - if r.scopes == nil { - r.scopes = map[recoveryScope]struct{}{} - } - r.scopes[scope] = struct{}{} - // A snapshot of this scope that is still waiting for its push was taken before this write - // was dropped, so publishing it proves nothing about this write. The scope needs a new one. - delete(r.awaitingPush, scope) - } - l.armRecoveryTimer() - l.publishRecovery() -} - -// windowScopes are the scopes of a dropped live window: one per collection its events came from. -func windowScopes(namespace, name string, events []Event) []recoveryScope { - target := itypes.NewResourceReference(name, namespace) - seen := map[itypes.CollectionKey]bool{} - var scopes []recoveryScope - for i := range events { - collection := events[i].SourceCollection - if !seen[collection] { - seen[collection] = true - scopes = append(scopes, recoveryScope{target: target, collection: collection}) - } - } - return scopes -} - -// atomicScope is the scope of a dropped atomic write. -func atomicScope(request *WriteRequest) recoveryScope { - name, namespace := atomicRefusalTarget(request) - return recoveryScope{target: itypes.NewResourceReference(name, namespace), collection: request.sourceCollection()} -} - -// resyncScope is the scope of a resync. -func resyncScope(req *ResyncRequest) recoveryScope { - return recoveryScope{ - target: itypes.NewResourceReference(req.GitTargetName, req.GitTargetNamespace), - collection: req.refusalCollection(), - } -} - -// noteResyncApplied records that a resync of one scope succeeded. A resync succeeds only after -// fetching the parent, so it is also proof the parent is back. A no-op settles the scope now; one -// that committed settles it when its commit is pushed. -func (l *branchWorkerEventLoop) noteResyncApplied(namespace, name string, collection itypes.CollectionKey, - committed bool) { - r := &l.recovery - if !r.active { - return - } - target := itypes.NewResourceReference(name, namespace) - for scope := range r.scopes { - if scope.target != target || !resyncCovers(collection, scope.collection) { - continue - } - if committed { - if r.awaitingPush == nil { - r.awaitingPush = map[recoveryScope]struct{}{} - } - r.awaitingPush[scope] = struct{}{} - } else { - delete(r.scopes, scope) - } - } - l.closeRecoveryIfDone() -} - -// resyncCovers reports whether a resync of one collection re-derives a dropped scope: the same -// collection, or either side unknown (a resync of the whole target, or a write whose collection is -// not known). -func resyncCovers(resynced, dropped itypes.CollectionKey) bool { - var unknown itypes.CollectionKey - return resynced == dropped || resynced == unknown || dropped == unknown -} - -// noteRecoveryPublished runs after every successful push: the scopes whose snapshot it carried are -// settled, and the retained writes it published were the other half. -func (l *branchWorkerEventLoop) noteRecoveryPublished() { - r := &l.recovery - if !r.active { - return - } - for scope := range r.awaitingPush { - delete(r.scopes, scope) - } - r.awaitingPush = nil - l.closeRecoveryIfDone() -} - -// closeRecoveryIfDone clears the latch once nothing is owed. -func (l *branchWorkerEventLoop) closeRecoveryIfDone() { - r := &l.recovery - if !r.active || len(l.pendingWrites) > 0 || len(r.scopes) > 0 { - return - } - if r.timer != nil { - r.timer.Stop() - } - *r = parentRecovery{} l.publishRecovery() } // probeDue reports whether the next background look at the remote may be spent now. func (l *branchWorkerEventLoop) probeDue() bool { r := &l.recovery - return r.active && (!l.w.now().Before(r.nextProbeAt) || r.gen != l.w.parentSnapshot().gen) + return r.active && (!l.w.now().Before(l.retry.due) || r.gen != l.w.parentSnapshot().gen) } -// runParentProbe is the deadline: while the parent is missing it spends one advertisement; once it -// is found it retries the retained writes and asks again for the snapshots still owed. Whatever -// remains open is scheduled for the next deadline. +// runParentProbe is the attempt while the obligation is open: while the parent is missing it spends +// one advertisement, and once it is found it publishes the pending writes. A publication that fails schedules +// the next attempt itself. func (l *branchWorkerEventLoop) runParentProbe() { r := &l.recovery if !r.active { return } if gen := l.w.parentSnapshot().gen; gen != r.gen { - r.gen, r.found, r.backoff = gen, false, parentProbeInitialBackoff + r.gen, r.found = gen, false + l.retry.backoff = 0 } if !r.found { missing, err := l.probeParent() if err != nil || missing { if err != nil { l.w.Log.V(1).Info("Parent branch probe failed", "branch", l.w.Branch, "error", err.Error()) + } else { + err = errAwaitingParentProbe } - l.scheduleNextProbe() + l.scheduleRetry(err) return } r.found = true - r.backoff = parentProbeInitialBackoff + l.retry.backoff = 0 l.publishRecovery() - l.w.Log.Info("The parent branch is on the remote again; recovering held-back work", - "branch", l.w.Branch, "parentBranch", l.w.ParentBranch(), - "pendingWrites", len(l.pendingWrites), "scopes", len(r.scopes)) - } - if len(l.pendingWrites) > 0 { - l.pushPending() + l.w.Log.Info("The parent branch is on the remote again; publishing the writes decided meanwhile", + "branch", l.w.Branch, "parentBranch", l.w.ParentBranch(), "pendingWrites", len(l.pendingWrites)) } - if r.active && r.found { - for target := range l.recoveryTargets() { - l.w.bumpSnapshotRequest(target) - } - } - if r.active { - l.scheduleNextProbe() - } -} - -// recoveryTargets are the GitTargets owed a snapshot. -func (l *branchWorkerEventLoop) recoveryTargets() map[itypes.ResourceReference]struct{} { - targets := map[itypes.ResourceReference]struct{}{} - for scope := range l.recovery.scopes { - if _, waiting := l.recovery.awaitingPush[scope]; !waiting { - targets[scope.target] = struct{}{} - } - } - return targets -} - -// scheduleNextProbe doubles the backoff and moves the deadline that far on: the first deadline is -// parentProbeInitialBackoff after the latch, the next twice that after it, up to the cap. -func (l *branchWorkerEventLoop) scheduleNextProbe() { - r := &l.recovery - r.backoff = min(parentProbeBackoffFactor*r.backoff, parentProbeMaxBackoff) - r.nextProbeAt = l.w.now().Add(r.backoff) - l.armRecoveryTimer() - l.publishRecovery() + l.pushPending() } // probeParent reads one advertisement and records what it says. It reports whether the parent is @@ -292,6 +134,7 @@ func (l *branchWorkerEventLoop) probeParent() (bool, error) { } parent := w.parentSnapshot() advertisement, err := advertiseRemoteBranchFn( + w.ctx, w.repo.URL, plumbing.NewBranchReferenceName(w.Branch), parent.name, @@ -303,25 +146,6 @@ func (l *branchWorkerEventLoop) probeParent() (bool, error) { return w.recordAdvertisement(advertisement, parent.name), nil } -func (l *branchWorkerEventLoop) armRecoveryTimer() { - r := &l.recovery - if r.timer != nil { - r.timer.Stop() - r.timer = nil - } - if !r.active { - return - } - r.timer = time.NewTimer(max(r.nextProbeAt.Sub(l.w.now()), 0)) -} - -func (l *branchWorkerEventLoop) recoveryTimerC() <-chan time.Time { - if l.recovery.timer == nil { - return nil - } - return l.recovery.timer.C -} - // publishRecovery mirrors the latch for the readers outside the loop: the controller, and the // fetch gate the write path consults. func (l *branchWorkerEventLoop) publishRecovery() { @@ -329,7 +153,7 @@ func (l *branchWorkerEventLoop) publishRecovery() { l.w.parentRecoveryOpen.Store(r.active) l.w.parentRecoveryFound.Store(r.active && r.found) if r.active && !r.found { - l.w.parentProbeHold.Store(&parentProbeHold{until: r.nextProbeAt, gen: r.gen}) + l.w.parentProbeHold.Store(&parentProbeHold{until: l.retry.due, gen: r.gen}) } else { l.w.parentProbeHold.Store(nil) } @@ -348,33 +172,8 @@ func (w *BranchWorker) awaitingParentProbe() bool { return hold != nil && hold.gen == w.parentSnapshot().gen && w.now().Before(hold.until) } -// bumpSnapshotRequest asks the controller for a fresh snapshot of one GitTarget. -func (w *BranchWorker) bumpSnapshotRequest(target itypes.ResourceReference) { - w.snapshotRequestsMu.Lock() - defer w.snapshotRequestsMu.Unlock() - if w.snapshotRequests == nil { - w.snapshotRequests = map[itypes.ResourceReference]uint64{} - } - w.snapshotRequests[target]++ -} - -// BumpSnapshotRequestForTest asks for a snapshot of the GitTarget, as the recovery probe does, for -// tests in another package. -func (w *BranchWorker) BumpSnapshotRequestForTest(target itypes.ResourceReference) { - w.bumpSnapshotRequest(target) -} - -// SnapshotRequestSeq is how many times the worker has asked for a fresh snapshot of the GitTarget, -// to re-derive writes it dropped while the parent branch was missing. The controller forces one -// recheck each time it rises, and remembers the value it acted on. -func (w *BranchWorker) SnapshotRequestSeq(target itypes.ResourceReference) uint64 { - w.snapshotRequestsMu.Lock() - defer w.snapshotRequestsMu.Unlock() - return w.snapshotRequests[target] -} - -// ParentRecovery reports whether the worker still owes work held back by a missing parent (the -// first result), and whether the parent has been found since (the second). +// ParentRecovery reports whether the worker still owes work decided while the parent branch was +// missing (the first result), and whether the parent has been found since (the second). func (w *BranchWorker) ParentRecovery() (bool, bool) { return w.parentRecoveryOpen.Load(), w.parentRecoveryFound.Load() } diff --git a/internal/git/parent_recovery_test.go b/internal/git/parent_recovery_test.go index 3c164920..13acef57 100644 --- a/internal/git/parent_recovery_test.go +++ b/internal/git/parent_recovery_test.go @@ -2,10 +2,10 @@ package git -// Work a missing parent branch held back is an obligation, not an observation: it stays open until -// the work is published, so a transient failure after the parent reappears cannot strand it, and -// it is driven by the worker on its own deadline, whatever the refresh configuration. These pin -// the obligation, its per-scope bookkeeping, and the probe budget. +// Work decided while the parent branch is missing is an obligation, not an observation: it stays +// pending until it is published, so a transient failure after the parent reappears cannot strand it, +// and it is driven by the worker on its own deadline, whatever the refresh configuration. These pin +// the obligation, that nothing decided meanwhile is dropped, and the probe budget. import ( "context" @@ -108,7 +108,7 @@ func TestParentRecovery_RetainedWritesLandAfterAFailedFirstAttempt(t *testing.T) release := simulateClientCommitOnDisk(t, f.repoDir, "release", "RELEASE.md", "back\n") failNextSync(t) - clock.advance(parentProbeMaxBackoff) + clock.advance(retryMaxBackoff) deadline() assert.True(t, f.ref("feature").IsZero(), "the first attempt failed") @@ -117,7 +117,7 @@ func TestParentRecovery_RetainedWritesLandAfterAFailedFirstAttempt(t *testing.T) require.True(t, open, "the obligation outlives a failed attempt") assert.True(t, found) - clock.advance(parentProbeMaxBackoff) + clock.advance(retryMaxBackoff) deadline() tip := f.ref("feature") @@ -134,72 +134,53 @@ func refreshTarget() itypes.ResourceReference { return itypes.NewResourceReference(newBranchTarget, "default") } -// Test 2. Writes dropped for two targets on one worker. Each scope clears only when a resync of it -// is published; a resync that committed but is not pushed yet does not clear it, and a target -// whose resync failed is asked again on the next deadline. -func TestParentRecovery_DroppedWritesClearPerScopeWhenPublished(t *testing.T) { - f := newNewBranchFixture(t, nil) +// Test 2. Writes for two targets on one worker, decided while the parent is missing, stay pending +// with their authors and collections, and so does a resync decided meanwhile, whose caller is told +// it could not be applied yet. The probe that finds the parent publishes all of it, and nothing is +// owed after that: no snapshot has to re-derive anything. +func TestParentRecovery_WritesDecidedWhileTheParentIsMissingAreKept(t *testing.T) { + f := newRealServerNewBranch(t, "recovery-kept-writes", seedMain(t)) clock := useTestClock(f.worker) createPlainGitTarget(t, f.worker, "other-target", "other") f.worker.SetParentBranch("release") - loop := newBranchWorkerEventLoop(f.worker, 0) - defer loop.stopTimers() + loop := f.loop - a := itypes.NewResourceReference(newBranchTarget, "default") - b := itypes.NewResourceReference("other-target", "default") - collectionA := itypes.CollectionKey{Resource: "configmaps", Namespace: "default"} - collectionB := itypes.CollectionKey{Resource: "configmaps", Namespace: "default", LabelSelector: "team=b"} - write := func(target string, collection itypes.CollectionKey, name string) { + write := func(target, name string) { event := configMapTargetEvent(name, "alice", target) - event.SourceCollection = collection loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) } - resync := func(target itypes.ResourceReference, collection itypes.CollectionKey, name string) error { - scope := ResyncScope{Collection: collection} - req := &ResyncRequest{ - Desired: []manifestanalyzer.DesiredResource{desiredCM(name, "blue")}, - ResourceVersion: "1", - GitTargetName: target.Name, - GitTargetNamespace: target.Namespace, - Scope: &scope, - Result: make(chan ResyncResult, 1), - } - loop.applyResync(req) - return (<-req.Result).Err - } + write(newBranchTarget, "a1") + write("other-target", "b1") + require.Len(t, loop.pendingWrites, 2, "both are kept") + open, _ := f.worker.ParentRecovery() + require.True(t, open) - write(newBranchTarget, collectionA, "a1") - write("other-target", collectionB, "b1") - assert.Empty(t, loop.pendingWrites, "both were dropped") - assert.Len(t, loop.recovery.scopes, 2) + req := &ResyncRequest{ + Desired: []manifestanalyzer.DesiredResource{desiredCM("a1", "a1"), desiredCM("a2", "blue")}, + ResourceVersion: "1", + GitTargetName: newBranchTarget, + GitTargetNamespace: "default", + Scope: &ResyncScope{ + Collection: itypes.CollectionKey{Resource: "configmaps", Namespace: "default"}, + }, + Result: make(chan ResyncResult, 1), + } + loop.applyResync(req) + loop.endWake(0) + require.ErrorIs(t, (<-req.Result).Err, ErrParentBranchNotFound, "its caller hears it could not be applied yet") + require.Len(t, loop.pendingWrites, 3, "and the resync waits in its place") - f.pushToRelease("RELEASE.md", "release\n") - clock.advance(parentProbeMaxBackoff) + gitIn(t, f.repoDir, "branch", "release", "main") + clock.advance(retryMaxBackoff) loop.runParentProbe() - assert.Equal(t, uint64(1), f.worker.SnapshotRequestSeq(a)) - assert.Equal(t, uint64(1), f.worker.SnapshotRequestSeq(b)) - - loop.lastPushAt = time.Now() // cooldown: A's resync commits, and its push waits - require.NoError(t, resync(a, collectionA, "a1")) - require.Len(t, loop.pendingWrites, 1) - assert.Contains(t, loop.recovery.scopes, recoveryScope{target: a, collection: collectionA}, - "a resync that is not pushed yet does not clear its scope") - - failNextSync(t) - require.Error(t, resync(b, collectionB, "b1")) - - clock.advance(parentProbeMaxBackoff) - loop.runParentProbe() // publishes A's commit, and asks for B again - assert.Empty(t, loop.pendingWrites) - assert.NotContains(t, loop.recovery.scopes, recoveryScope{target: a, collection: collectionA}, "published") - assert.Equal(t, uint64(1), f.worker.SnapshotRequestSeq(a), "A is not asked again") - assert.Equal(t, uint64(2), f.worker.SnapshotRequestSeq(b), "B is") - - require.NoError(t, resync(b, collectionB, "b1")) - loop.pushPending() - open, _ := f.worker.ParentRecovery() - assert.False(t, open, "every scope is published") + assert.Empty(t, loop.pendingWrites, "all of it was published") + open, _ = f.worker.ParentRecovery() + assert.False(t, open, "and nothing is owed") + names := gitIn(t, f.repoDir, "ls-tree", "-r", "--name-only", "refs/heads/feature") + for _, name := range []string{"a1", "b1", "a2"} { + assert.Contains(t, names, name) + } } // Test 4. However many targets share the worker, and however often refreshes tick and live writes @@ -207,7 +188,7 @@ func TestParentRecovery_DroppedWritesClearPerScopeWhenPublished(t *testing.T) { // schedule, and never fetched. That holds with work retained too: there, every write first wants to // rebuild the retained writes, which is a fetch the deadline must hold back as well. func TestParentRecovery_ProbeBudgetIsPerWorker(t *testing.T) { - for name, retained := range map[string]bool{"dropped writes": false, "retained writes": true} { + for name, retained := range map[string]bool{"decided writes": false, "retained writes": true} { t.Run(name, func(t *testing.T) { reader, err := telemetry.InitTestExporter() require.NoError(t, err) @@ -229,7 +210,7 @@ func TestParentRecovery_ProbeBudgetIsPerWorker(t *testing.T) { require.Len(t, f.loop.pendingWrites, 1) gitIn(t, f.repoDir, "update-ref", "-d", "refs/heads/release") f.loop.pushPending() // the push that finds the parent gone - require.True(t, f.worker.replayRequired(), "the retained writes wait for a rebuild") + require.False(t, f.loop.checkoutCurrent(), "the retained writes wait for a rebuild") } else { f.worker.SetParentBranch("missing") liveWrite(f.loop, "first") // the one fetch that finds the parent missing @@ -267,58 +248,48 @@ func TestParentRecovery_ProbeBudgetIsPerWorker(t *testing.T) { } } -// A second outage while a recovery snapshot waits for its push. The write dropped during it was -// made after that snapshot was taken, so publishing the snapshot must not settle its scope: the -// worker asks for another one, and the obligation stays open until that is published. -func TestParentRecovery_ASecondOutageKeepsTheWorkDroppedInIt(t *testing.T) { +// A second outage while the first outage's writes still wait to be published. The write decided +// during it is kept with them, and the obligation stays open until all of it is published. +func TestParentRecovery_ASecondOutageKeepsItsWritesToo(t *testing.T) { f := newRealServerNewBranch(t, "recovery-second-outage", func(dir string) { simulateClientCommitOnDisk(t, dir, "main", "README.md", "main\n") }) clock := useTestClock(f.worker) f.worker.SetParentBranch("release") - collection := itypes.CollectionKey{Resource: "configmaps", Namespace: "default"} - write := func(name string) { - event := configMapTargetEvent(name, "alice", newBranchTarget) - event.SourceCollection = collection - f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) - } - write("cm1") // dropped: release does not exist + liveWrite(f.loop, "cm1") // kept: release does not exist gitIn(t, f.repoDir, "branch", "release", "main") - clock.advance(parentProbeMaxBackoff) - f.loop.runParentProbe() - seq := f.worker.SnapshotRequestSeq(refreshTarget()) - require.Equal(t, uint64(1), seq) - - f.loop.lastPushAt = time.Now() // the snapshot commits, and its push waits for the cooldown - req := &ResyncRequest{ - Desired: []manifestanalyzer.DesiredResource{desiredCM("cm1", "blue")}, - ResourceVersion: "1", GitTargetName: newBranchTarget, GitTargetNamespace: "default", - Scope: &ResyncScope{Collection: collection}, - Result: make(chan ResyncResult, 1), - } - f.loop.applyResync(req) - require.NoError(t, (<-req.Result).Err) - require.Len(t, f.loop.recovery.awaitingPush, 1) + failNextPushes(t, 1) + clock.advance(retryMaxBackoff) + f.loop.runParentProbe() // finds the parent; the push fails + require.True(t, f.ref("feature").IsZero()) + require.Len(t, f.loop.pendingWrites, 1) gitIn(t, f.repoDir, "update-ref", "-d", "refs/heads/release") // the second outage - f.loop.pushPending() - write("cm2") // dropped during it - require.Len(t, f.loop.pendingWrites, 1, "cm2 was dropped; only the snapshot is retained") + liveWrite(f.loop, "cm2") + require.Len(t, f.loop.pendingWrites, 2, "the write decided during it is kept with the first") + clock.advance(retryMaxBackoff) + f.loop.runParentProbe() // the push finds the parent gone again + require.Len(t, f.loop.pendingWrites, 2) + open, found := f.worker.ParentRecovery() + require.True(t, open) + require.False(t, found) gitIn(t, f.repoDir, "branch", "release", "main") - clock.advance(parentProbeMaxBackoff) + clock.advance(retryMaxBackoff) f.loop.runParentProbe() - require.False(t, f.ref("feature").IsZero(), "the snapshot was published") - open, _ := f.worker.ParentRecovery() - assert.True(t, open, "cm2 still needs a snapshot") - assert.Greater(t, f.worker.SnapshotRequestSeq(refreshTarget()), seq, "and the worker asks for one") + assert.Empty(t, f.loop.pendingWrites) + files := gitIn(t, f.repoDir, "ls-tree", "-r", "--name-only", "refs/heads/feature") + assert.Contains(t, files, "cm1") + assert.Contains(t, files, "cm2") + open, _ = f.worker.ParentRecovery() + assert.False(t, open) } // Test 5. An unresolved default branch, once repaired, recovers through the same path: the probe -// finds it and asks for the snapshot the dropped write needs. -func TestParentRecovery_ARepairedRemoteHeadAsksForTheSnapshot(t *testing.T) { +// finds it and publishes the write kept meanwhile. +func TestParentRecovery_ARepairedRemoteHeadPublishesTheKeptWrite(t *testing.T) { f := newRealServerNewBranch(t, "recovery-dangling-head", func(repoDir string) { simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") gitIn(t, repoDir, "symbolic-ref", "HEAD", "refs/heads/master") @@ -327,15 +298,15 @@ func TestParentRecovery_ARepairedRemoteHeadAsksForTheSnapshot(t *testing.T) { liveWrite(f.loop, "cm1") assert.True(t, f.ref("feature").IsZero()) + require.Len(t, f.loop.pendingWrites, 1, "kept") gitIn(t, f.repoDir, "symbolic-ref", "HEAD", "refs/heads/main") - clock.advance(parentProbeMaxBackoff) + clock.advance(retryMaxBackoff) f.loop.runParentProbe() - assert.Equal(t, uint64(1), f.worker.SnapshotRequestSeq(refreshTarget())) - open, found := f.worker.ParentRecovery() - assert.True(t, open, "open until the snapshot is published") - assert.True(t, found) + assert.False(t, f.ref("feature").IsZero(), "published") + open, _ := f.worker.ParentRecovery() + assert.False(t, open) } func TestParentRecovery_AParentChangeMakesTheProbeDue(t *testing.T) { @@ -396,7 +367,7 @@ func TestParentRecovery_AHeldCommitRequestIsCommittedAfterALongRecovery(t *testi assert.True(t, phase().Held()) release := simulateClientCommitOnDisk(t, f.repoDir, "release", "RELEASE.md", "back\n") - clock.advance(parentProbeMaxBackoff) + clock.advance(retryMaxBackoff) f.worker.EnqueueRefresh(&RefreshRequest{Target: refreshTarget(), MaxAge: time.Nanosecond}) require.Eventually(t, func() bool { @@ -410,3 +381,73 @@ func TestParentRecovery_AHeldCommitRequestIsCommittedAfterALongRecovery(t *testi assert.Equal(t, res.Commit, tip.String()) assert.Equal(t, []plumbing.Hash{release}, f.parentOf(tip)) } + +// A save that reached no window and asked for its message to be recorded waits for a missing parent +// like any other decided write, and is committed once the parent exists. +func TestParentRecovery_AnEmptySaveWaitsForTheParent(t *testing.T) { + f := newRealServerNewBranch(t, "parent-recovery-empty-save", seedMain(t)) + clock := useTestClock(f.worker) + f.worker.SetParentBranch("release") + + req := attachReq("alice", 0) + req.GitTargetName = newBranchTarget + req.CommitEmpty = true + req.Message = "empty save" + serviceAttach(f.loop, req) + + _, resolved := outcome(t, f.worker) + require.False(t, resolved, "held, not failed") + require.Len(t, f.loop.pendingWrites, 1) + + gitIn(t, f.repoDir, "branch", "release", "main") + clock.advance(retryMaxBackoff) + f.loop.runParentProbe() + + res, resolved := outcome(t, f.worker) + require.True(t, resolved) + require.NoError(t, res.Err) + assert.NotEmpty(t, res.Commit, "the message is recorded in Git") + open, _ := f.worker.ParentRecovery() + assert.False(t, open) +} + +// An obligation with nothing to publish would never close, so a parent failure that left nothing +// pending opens none. Found in review of #413, when a failed empty save left the target +// RecoveringParentBranch with nothing owed. +func TestParentRecovery_NothingOwedOpensNoObligation(t *testing.T) { + w := &BranchWorker{} + loop := newBranchWorkerEventLoop(w, time.Second) + defer loop.stopTimers() + + loop.noteParentUnavailable(ErrParentBranchNotFound) + + open, _ := w.ParentRecovery() + assert.False(t, open) +} + +// Recovery ends when nothing is owed, not only when a push lands. A resync held while the parent +// was missing can find nothing to change once it is back, and leave nothing pending without a push: +// recovery and its retry then stayed open, and intake stayed paused for good. +func TestParentRecovery_EndsWhenTheHeldWorkNeedsNoCommit(t *testing.T) { + f := newRealServerNewBranch(t, "recovery-noop-resync", seedMain(t)) + clock := useTestClock(f.worker) + f.worker.SetParentBranch("release") + req := &ResyncRequest{GitTargetName: newBranchTarget, GitTargetNamespace: "default", + Result: make(chan ResyncResult, 1)} + f.loop.handleQueueItem(resyncItem(req)) + require.ErrorIs(t, (<-req.Result).Err, ErrParentBranchNotFound) + require.Len(t, f.loop.pendingWrites, 1) + require.True(t, f.loop.recovery.active) + f.worker.branchBufferMaxBytes = 1 + f.loop.publishLoopState(0) + require.NotNil(t, f.worker.IntakePaused()) + + gitIn(t, f.repoDir, "branch", "release", "main") + clock.advance(retryMaxBackoff) + fireRetry(f.loop) + + require.Empty(t, f.loop.pendingWrites, "the resync had nothing to change") + assert.False(t, f.loop.recovery.active, "nothing is owed, so there is nothing to recover") + assert.False(t, f.loop.retry.pending(), "or to retry") + assert.Nil(t, f.worker.IntakePaused(), "and intake reopens") +} diff --git a/internal/git/pending_writes_loop.go b/internal/git/pending_writes_loop.go new file mode 100644 index 00000000..1481d49a --- /dev/null +++ b/internal/git/pending_writes_loop.go @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "errors" + "fmt" + "slices" + + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// This file is the branch worker's ordered pending writes, and the one driver that turns them into +// commits and publications. See docs/design/gittarget-branch-worker-pending-writes.md. +// +// Every write the loop makes, whatever produced it (a closed window, an atomic batch, a resync, a +// save's empty record, a refusal's empty commit), takes one path: +// +// - decide appends it to the pending writes. The write gates and building the write have passed, so the +// decision stands from here on. Deciding does nothing else: no commit, no push, no settlement. +// - advance is the one driver, run once at the end of every wake of the loop. It commits what was +// decided, schedules the push, and closes the outage once nothing is owed. +// - materialize makes the checkout the projection of the pending writes: the remote tip the writes were +// planned on plus one commit for each, in order. It rebuilds the writes committed before when +// the checkout is behind them, then commits each decided write not committed yet. +// - materialize classifies each attempt in one place: committed (settleCommitted); failed for +// good, because the write itself was refused or cannot be made (settleFailed); or left pending, +// because the remote could not be reached. A write left pending needs no settling, only its +// waiting resync callers an answer (replyToDeferredResyncs). +// +// Settling an outcome can decide more work (a refused write's empty commit). Because deciding only +// appends, that work joins the pending writes behind the ones decided before it, and the pass already running +// commits it in order: nothing is ever re-entered. + +// errWriteFailed marks a failure of the write itself, as opposed to reaching the remote or reading +// the GitProvider: the plan was refused or could not be applied. It is terminal for the write, while +// any other commit failure leaves a decided write pending for the retry. Match it with errors.Is. +var errWriteFailed = errors.New("the write failed") + +// writeFailedError carries errWriteFailed without changing the message of the error it wraps. +type writeFailedError struct{ err error } + +func (e writeFailedError) Error() string { return e.err.Error() } +func (e writeFailedError) Unwrap() error { return e.err } +func (e writeFailedError) Is(target error) bool { return target == errWriteFailed } + +// writeOrigin is what a write's outcome is settled against: the caller waiting on a resync, the +// atomic request, or the refusal an empty commit answers. They are process objects, kept beside +// the write's data rather than in it. +type writeOrigin struct { + resync *ResyncRequest + atomic *WriteRequest + refusal *refusalTouchOrigin + // answered records that the resync's caller was answered already: the remote held it back, + // and it is applied later with nobody waiting. + answered bool +} + +// refusalTouchOrigin is the refusal an empty commit answers, recorded once the commit exists. +type refusalTouchOrigin struct { + key refusalKey + observation string +} + +// errAdmissionClosed answers a resync refused at admission. It is ErrFinalizeQueueFull to every +// caller: the work was not accepted, and the producer that sent it keeps it. +var errAdmissionClosed = fmt.Errorf( + "%w: the branch has paused intake while the remote cannot take what it holds", ErrFinalizeQueueFull) + +// pendingWriteOverheadBytes is what every retained item is charged beyond its payload: each event a +// window or a write keeps (its identity, author, path), and a write that keeps none (its target +// metadata, commit configuration, signer, the save it carries). Without it an item with no payload, +// a delete, a save's empty record or a refusal's empty commit, cost nothing to keep, and a branch +// whose remote was down admitted them for as long as the outage lasted. +const pendingWriteOverheadBytes = 1024 + +// retainedCharge is what keeping a decided write is charged against the retained-byte budget: its +// payload, its message, and one item's overhead for each event it keeps, or for itself when it +// keeps none. That is never more than the window it closed or the save it records was charged, so +// deciding a write never takes what the branch holds past what intake admitted. It is fixed when +// the write is decided, so the refund when it stops being pending matches it whatever the write's later +// rendering changes. +func retainedCharge(pendingWrite *PendingWrite) int64 { + items := int64(max(len(pendingWrite.Events), 1)) + return pendingWrite.ByteSize + int64(len(pendingWrite.CommitMessage)) + items*pendingWriteOverheadBytes +} + +// decide appends a write the loop has decided to make to the pending writes, and charges what +// keeping it costs. +// A request riding it is held from here on, so the controller never fails a save whose write can +// still land. Committing it is advance's job, at the end of the wake. +func (l *branchWorkerEventLoop) decide(pendingWrite PendingWrite) { + pendingWrite.charge = retainedCharge(&pendingWrite) + pendingWrite.decidedAt = l.w.now() + l.pendingWrites = append(l.pendingWrites, pendingWrite) + l.pendingWritesBytes += pendingWrite.charge + if id := pendingWrite.CommitRequest; id != nil { + l.w.setCommitRequestPhase(*id, PhaseWaitingForPush) + // Bound to its write from here on: see pendingCommitRequest.attached. + if pcr := l.pendingCRs[*id]; pcr != nil { + pcr.attached = true + } + } +} + +// advance drives the pending writes once the wake's handlers have decided what they decide: it +// commits them, schedules their push, and ends the outage once nothing is owed. It is the only +// caller of materialize outside a publication, and the only place where having no pending writes +// closes the retry and parent recovery, so no exit that settles the last one can leave intake +// paused. +// +// A push that lands settles the writes a replay refused, and settling one can decide more (a +// refusal's empty commit). Those are committed on the next turn; the push after them waits for the +// cooldown, so the loop ends. +func (l *branchWorkerEventLoop) advance() { + for { + l.commitDecided() + if len(l.pendingWrites) == 0 { + l.stopPushTimer() + l.endOutage() + return + } + l.maybeSchedulePush() + if l.retry.pending() || l.materializedPrefix() == len(l.pendingWrites) { + return + } + } +} + +// commitDecided commits every decided write not committed yet. Deciding is not committing: a remote +// that cannot be reached leaves the writes pending for the retry. Until a pending retry is due, a +// commit that would need a connection waits for that retry instead of spending one. A resync always +// needs one, and its caller is waiting to hear what it found: it is answered at once with the +// failure the retry is waiting out, and stays pending to be applied when the retry is due +// (replyToDeferredResyncs). +func (l *branchWorkerEventLoop) commitDecided() { + if l.materializedPrefix() == len(l.pendingWrites) { + return + } + if l.awaitingRetry() && !l.materializeIsLocal() && !l.w.awaitingParentProbe() { + l.replyToDeferredResyncs(l.retry.cause) + return + } + if err := l.materialize(); err != nil { + l.noteParentUnavailable(err) + if !l.retry.pending() { + l.scheduleRetry(err) + } + l.w.Log.Error(err, "Cannot commit the decided writes yet; they stay pending for the retry", + "pendingWrites", len(l.pendingWrites), "retryAt", l.retry.due) + } +} + +// materialize makes the checkout the projection of the pending writes, committing every one not +// committed yet, including any that settling one decides. It returns the error that stopped it when +// the remote could not be reached; the writes it could not commit stay pending. +func (l *branchWorkerEventLoop) materialize() error { + for { + i := l.materializedPrefix() + // A resync judges its snapshot against the newest remote tree, so it fetches first whatever + // the checkout holds. + refetch := "" + if i < len(l.pendingWrites) && l.pendingWrites[i].Kind == PendingWriteResync { + refetch = fetchReasonForcedRecheck + } + if err := l.materializePrefix(refetch); err != nil { + l.replyToDeferredResyncs(err) + l.w.recordMaterializationFailure(err) + return err + } + // Read again: a rebuild settles the writes the new tree refuses out of the prefix. + i = l.materializedPrefix() + if i == len(l.pendingWrites) { + return nil + } + // A write is planned under the policy in force when it is committed, exactly as a replay is: + // an operator who tightened pruning since it was decided is obeyed. + if err := l.w.tightenPendingPruneModes(l.w.ctx, l.pendingWrites[i:i+1]); err != nil { + l.replyToDeferredResyncs(err) + l.w.recordMaterializationFailure(err) + return err + } + err := l.w.commitPendingWrites(l.pendingWrites[i : i+1]) + switch { + case err == nil: + l.settleCommitted(i) + case errors.Is(err, errWriteFailed): + l.settleFailed(i, err) + default: + l.replyToDeferredResyncs(err) + l.w.recordMaterializationFailure(err) + return err + } + } +} + +// materializePrefix rebuilds the writes committed before when the checkout is behind them: a write +// failed part-way and could not be undone, a reset discarded their commits and the replay did not +// finish, or their root was chosen under an older parent configuration. A parent change is read +// from the generations, not taken from a flag: a rebuild that fails leaves them apart, so the next +// attempt rebuilds again. +// +// refetch, when set, is the fetch series of a caller that needs the remote tip whatever the checkout +// holds. With nothing committed and no refetch there is nothing to project: commitPendingWrites' own +// base check fetches when the base is untrusted or the worktree dirty, which it can do safely +// because nothing is lost. +func (l *branchWorkerEventLoop) materializePrefix(refetch string) error { + m := l.materializedPrefix() + if refetch == "" && (m == 0 || l.w.checkoutHolds(m) && !l.w.rootParentStale()) { + return nil + } + // The rebuild fetches, and a parent known to be missing is not fetched again before its probe + // is due: every commit path comes through here, so without the check each write arriving in the + // meantime would cost a connection. + if l.w.awaitingParentProbe() { + return errAwaitingParentProbe + } + if m == 0 { + // Nothing to replay, so fetch and reset directly. Leaving it to ensureBaseForCycle would + // work, but that call records `publication`, and a resync is not one. + return l.w.syncWithRemote(l.w.ctx, refetch) + } + reason := refetch + if reason == "" { + // The same series the nothing-committed case records in ensureBaseForCycle: this is one + // event, and which half of it an operator sees must not depend on whether a push happened + // to be in cooldown at the time. + reason = fetchReasonRecovery + l.w.Log.Info("Rebuilding retained writes onto the remote tip", + "pendingWrites", m, + "worktreeDirty", l.w.worktreeDirty(), + "committedWrites", l.w.checkoutApplied.Load()) + } + if err := l.w.refreshRemoteAndRebuildPendingWrites(l.w.ctx, l.pendingWrites[:m], reason); err != nil { + return err + } + l.settleReplayRefusals(l.takeReplayRefusals()) + return nil +} + +// materializedPrefix is how many writes at the head of the pending writes have been committed before. Writes +// not committed yet are always the tail: they are decided in order and committed in order. +func (l *branchWorkerEventLoop) materializedPrefix() int { + for i := range l.pendingWrites { + if !l.pendingWrites[i].committedOnce { + return i + } + } + return len(l.pendingWrites) +} + +// materializeIsLocal reports whether committing the uncommitted pending writes now needs no +// connection: the checkout already holds the writes before them, on a base the worker can vouch for, +// and none of them is a resync, which fetches whatever the checkout holds. +func (l *branchWorkerEventLoop) materializeIsLocal() bool { + m := l.materializedPrefix() + for i := m; i < len(l.pendingWrites); i++ { + if l.pendingWrites[i].Kind == PendingWriteResync { + return false + } + } + if m > 0 { + return l.w.checkoutHolds(m) && !l.w.rootParentStale() + } + return l.w.baseTrusted() && !l.w.worktreeDirty() +} + +// checkoutCurrent reports whether the checkout is the projection of every pending write, so it may be +// pushed as it stands. +func (l *branchWorkerEventLoop) checkoutCurrent() bool { + m := l.materializedPrefix() + if m < len(l.pendingWrites) { + return false + } + return m == 0 || l.w.checkoutHolds(m) && !l.w.rootParentStale() +} + +// removeAt takes the write at i out of the pending writes. The slot it vacates is cleared, so the backing array +// does not keep the write's payload reachable once the budget has refunded it. +func (l *branchWorkerEventLoop) removeAt(i int) PendingWrite { + pendingWrite := l.pendingWrites[i] + l.pendingWrites = slices.Delete(l.pendingWrites, i, i+1) + l.pendingWritesBytes -= pendingWrite.charge + return pendingWrite +} + +// takeReplayRefusals takes out of the pending writes every one the last completed replay refused, +// and returns them in order. The checkout holds none of them (replayPendingWrites), so after this it +// is again the projection of the committed prefix. +func (l *branchWorkerEventLoop) takeReplayRefusals() []PendingWrite { + var refused []PendingWrite + for i := 0; i < len(l.pendingWrites); { + if l.pendingWrites[i].replayRefusal == nil { + i++ + continue + } + refused = append(refused, l.removeAt(i)) + } + return refused +} + +// settleReplayRefusals settles writes a replay refused, the way a refusal at first commit is settled: +// the refusal is reported, a request riding the write fails, and the write is gone. Taking them out +// of the pending writes is a separate step (takeReplayRefusals), because the push path must do it before it +// resolves the writes it published, and settling one can decide more work. +func (l *branchWorkerEventLoop) settleReplayRefusals(refused []PendingWrite) { + for _, pendingWrite := range refused { + l.settleDropped(pendingWrite, pendingWrite.replayRefusal) + } +} + +// settleCommitted acts on a write whose commit was made. A resync answers its caller with what it +// found, and one that committed nothing stops being pending: it is neither retained nor pushed. A +// refusal's empty commit records the refusal it covered, so the same observation is not a new +// trigger. +func (l *branchWorkerEventLoop) settleCommitted(i int) { + l.pendingWrites[i].committedOnce = true + pendingWrite := l.pendingWrites[i] + switch { + case pendingWrite.origin.resync != nil: + l.settleResyncApplied(i) + case pendingWrite.origin.refusal != nil: + l.w.recordRefusalObservation(pendingWrite.origin.refusal.key, pendingWrite.origin.refusal.observation) + } +} + +// settleResyncApplied answers a resync whose plan was accepted. +func (l *branchWorkerEventLoop) settleResyncApplied(i int) { + req := l.pendingWrites[i].origin.resync + answered := l.pendingWrites[i].origin.answered + stats := l.pendingWrites[i].ResyncStats + committed := l.pendingWrites[i].retained() + if !committed { + l.removeAt(i) + } + // The plan was accepted, so the refusal standing over THIS SCOPE is over: the observation the + // last empty commit covered is forgotten and any commit still queued for it is cancelled. This + // is the recovery path the dedupe depends on: a per-type reconcile runs after the reconciler + // reverts the edit, and it is what makes the NEXT refusal, including a re-made byte-identical + // one, a new trigger rather than a repeat. Scoped to the collection this request evaluated, + // because a ConfigMap resync succeeding is no evidence about a Deployment that is still refused. + l.refusalRecovered( + itypes.NewResourceReference(req.GitTargetName, req.GitTargetNamespace), req.refusalCollection()) + l.w.Log.Info("Resync request applied", + "committed", committed, + "created", stats.Created, + "updated", stats.Updated, + "deleted", stats.Deleted, + "skipped", stats.Skipped, + "placementSkipped", stats.PlacementSkipped, + "pendingWrites", len(l.pendingWrites)) + if !answered { + answerResync(req, ResyncResult{Stats: *stats}) + if committed { + // Its caller has its answer, so a replay that later refuses it reports the refusal + // the way a live write's is, and does not answer again. + l.pendingWrites[i].origin.answered = true + } + } +} + +// answerResync answers a resync's caller. A resync the remote held back was answered then +// (writeOrigin.answered), and is applied later with nobody waiting: its outcome is then reported the +// way a live write's is. +func answerResync(req *ResyncRequest, result ResyncResult) { + req.reply(result) +} + +// settleFailed acts on a write whose commit failed for good: the plan was refused, or the write +// cannot be made, and retrying the same broken state helps nobody. It is no longer pending, a +// request riding it fails, and a refusal is surfaced as GitPathAccepted=False instead of being logged as a +// write fault (a resync's refusal reaches its caller, which classifies it). +func (l *branchWorkerEventLoop) settleFailed(i int, err error) { + l.settleDropped(l.removeAt(i), err) +} + +// settleDropped settles a write already taken out of the pending writes whose commit failed for good. +func (l *branchWorkerEventLoop) settleDropped(pendingWrite PendingWrite, err error) { + switch { + case pendingWrite.origin.resync != nil && !pendingWrite.origin.answered: + req := pendingWrite.origin.resync + // A refusal reaches the caller on ResyncResult.Err, where the watch layer classifies it + // and blocks the collection. spec.onRefusal needs it here too, and this is the path that + // matters: a per-type reconcile evaluates the same objects a live write would, so it is + // normally what discovers a write-boundary refusal first. + var refused *manifestanalyzer.AcceptanceRefusedError + if errors.As(err, &refused) { + l.touchBranchForRefusal(req.GitTargetName, req.GitTargetNamespace, err.Error(), refused, + refusalObservationForDesired(req.Desired, refused), req.refusalCollection()) + } + l.w.Log.Error(err, "Resync commit failed; dropping request", "resources", len(req.Desired)) + answerResync(req, ResyncResult{Err: err}) + case pendingWrite.origin.resync != nil: + // Nobody is waiting any more: report it as a live write's refusal is reported. + req := pendingWrite.origin.resync + collection := req.refusalCollection() + if isRefusal, refused := l.w.reportPathRefusal(err, req.GitTargetName, req.GitTargetNamespace, + collection); isRefusal { + l.touchBranchForRefusal(req.GitTargetName, req.GitTargetNamespace, err.Error(), refused, + refusalObservationForDesired(req.Desired, refused), collection) + } + l.w.Log.Error(err, "Resync commit failed after its caller was answered; dropping it", + "resources", len(req.Desired)) + case pendingWrite.origin.refusal != nil: + l.w.Log.Error(err, "The empty commit for a refused write failed", + "gitTarget", pendingWrite.origin.refusal.key.target.String()) + case pendingWrite.Kind == PendingWriteRequestRecord: + l.w.Log.Error(err, "The empty commit recording a CommitRequest failed", + "gitTarget", pendingWrite.GitTargetNamespace+"/"+pendingWrite.GitTargetName) + l.failDecidedRequest(pendingWrite, fmt.Errorf("record the message in an empty commit: %w", err)) + default: + l.settleFailedChange(pendingWrite, err) + } +} + +// settleFailedChange settles a window or an atomic batch whose commit failed for good. +func (l *branchWorkerEventLoop) settleFailedChange(pendingWrite PendingWrite, err error) { + kind, name, namespace, collection := changeIdentity(pendingWrite) + if isRefusal, refused := l.w.reportPathRefusal(err, name, namespace, collection); isRefusal { + l.w.recordCommitFailure(kind, commitFailureRefused) + l.touchBranchForRefusal(name, namespace, err.Error(), refused, + refusalObservationForEvents(pendingWrite.Events, refused), collection) + } else { + l.w.recordCommitFailure(kind, commitFailureError) + l.w.Log.Error(err, "Commit failed; dropping the write", + "kind", kind, "gitTarget", namespace+"/"+name, "events", len(pendingWrite.Events)) + } + l.failDecidedRequest(pendingWrite, fmt.Errorf("commit failed: %w", err)) +} + +// changeIdentity is a window's or an atomic batch's metric kind, GitTarget, and source collection. +func changeIdentity(pendingWrite PendingWrite) (string, string, string, itypes.CollectionKey) { + if request := pendingWrite.origin.atomic; request != nil { + name, namespace := atomicRefusalTarget(request) + return commitFailureKindAtomic, name, namespace, request.sourceCollection() + } + name, namespace := pendingWrite.windowTarget() + return commitFailureKindWindow, name, namespace, sourceCollectionForEvents(pendingWrite.Events) +} + +// replyToDeferredResyncs answers the callers of the resyncs among the writes not committed yet, when +// the remote could not be reached or the parent branch a new write branch is created from is +// missing. The writes themselves all stay pending for the retry, and admission backpressure bounds +// them however long that lasts. A resync's caller is answered now with the error, because it is +// waiting to hear what the resync found and that cannot be known yet; the resync itself stays, in +// its place, and is applied with the rest. +func (l *branchWorkerEventLoop) replyToDeferredResyncs(err error) { + for i := l.materializedPrefix(); i < len(l.pendingWrites); i++ { + origin := &l.pendingWrites[i].origin + if origin.resync == nil || origin.answered { + continue + } + req := origin.resync + l.w.Log.Error(err, "Cannot refresh the remote before resync; it stays pending", + "resources", len(req.Desired), "gitTarget", req.GitTargetNamespace+"/"+req.GitTargetName) + answerResync(req, ResyncResult{Err: fmt.Errorf("refresh remote before resync: %w", err)}) + origin.answered = true + } +} + +// failDecidedRequest fails the request a dropped write carried. +func (l *branchWorkerEventLoop) failDecidedRequest(pendingWrite PendingWrite, cause error) { + if pendingWrite.CommitRequest != nil { + l.resolveCommitRequest(*pendingWrite.CommitRequest, FinalizeResult{Branch: l.w.Branch, Err: cause}) + } +} diff --git a/internal/git/provider_repoint_test.go b/internal/git/provider_repoint_test.go index 52645bb7..4484bee4 100644 --- a/internal/git/provider_repoint_test.go +++ b/internal/git/provider_repoint_test.go @@ -53,7 +53,7 @@ func TestRepoint_TheWorkerStopsRatherThanWriteWithTheNewProvidersCredentials(t * elsewhere := f.repointProvider(t) - require.Error(t, f.worker.commitPendingWrites([]PendingWrite{*write}, false), + require.Error(t, f.worker.commitPendingWrites([]PendingWrite{*write}), "the GitProvider names a repository this worker is not for") _, err = f.worker.buildGroupedPendingWrite(f.worker.ctx, events) require.Error(t, err, "and nothing new is planned against it either") diff --git a/internal/git/publication.go b/internal/git/publication.go new file mode 100644 index 00000000..16b0182d --- /dev/null +++ b/internal/git/publication.go @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + "fmt" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + + "github.com/ConfigButler/gitops-reverser/internal/telemetry" +) + +// PublicationStatus is a branch worker's report of its publication, for every GitTarget on the +// branch and every save it holds. The zero value is a branch with nothing failing. +type PublicationStatus struct { + // Failing is set while a failed attempt waits for its retry. + Failing bool + // Since is when this run of failures began. It holds until a publication lands. + Since time.Time + // Cause is what the last attempt met. + Cause string + // ParentUnavailable is set when the cause is the parent branch a new write branch is created + // from: the parent status already says so, more precisely. + ParentUnavailable bool + // IntakePaused is set while the branch refuses new writes, saves and resyncs. + IntakePaused bool +} + +// Message says what the report means, in words that change only when the report does. +func (p PublicationStatus) Message() string { + if !p.Failing { + return "" + } + message := fmt.Sprintf("Cannot publish to the Git remote since %s: %s. The kept writes are retried on schedule", + p.Since.UTC().Format(time.RFC3339), p.Cause) + if p.IntakePaused { + message += "; intake of new changes is paused until a push lands" + } + return message +} + +// Publication is the worker's current report. It reads memory only, and costs no connection. +func (w *BranchWorker) Publication() PublicationStatus { + if report := w.publication.Load(); report != nil { + return *report + } + return PublicationStatus{} +} + +// publicationBacklog is what the loop keeps until a push lands, published for the gauges. Each +// field is an atomic so a scrape never waits on the loop. The timestamps are Unix seconds, zero +// when there is nothing to date. +type publicationBacklog struct { + retainedBytes atomic.Int64 + retainedWrites atomic.Int64 + oldestWrite atomic.Int64 + nextRetry atomic.Int64 +} + +// publishPublication publishes the loop's publication report and backlog. The report is replaced, +// and the GitTargets on the branch told, only when what it says changes: a retry that fails the +// same way again leaves it alone, so no status is written per attempt. When the next retry is due +// is in the backlog for the gauges, and deliberately not in the report. +func (l *branchWorkerEventLoop) publishPublication(held int64) { + backlog := &l.w.backlog + backlog.retainedBytes.Store(held) + backlog.retainedWrites.Store(int64(len(l.pendingWrites))) + backlog.oldestWrite.Store(0) + if len(l.pendingWrites) > 0 { + backlog.oldestWrite.Store(l.pendingWrites[0].decidedAt.Unix()) + } + backlog.nextRetry.Store(0) + if l.retry.pending() { + backlog.nextRetry.Store(l.retry.due.Unix()) + } + + report := PublicationStatus{} + if l.retry.pending() { + report = PublicationStatus{ + Failing: true, + Since: l.retry.since, + IntakePaused: l.w.intake.pausedC() != nil, + } + if cause := l.retry.cause; cause != nil { + report.Cause = cause.Error() + report.ParentUnavailable = isParentUnavailable(cause) + } + } + if report == l.w.Publication() { + return + } + l.w.publication.Store(&report) + if l.w.publicationReporter != nil { + l.w.publicationReporter() + } +} + +// recordMaterializationFailure counts an attempt to commit decided writes that stopped before any +// push: the remote could not be reached, or the parent branch is missing. +func (w *BranchWorker) recordMaterializationFailure(err error) { + if telemetry.GitMaterializationFailuresTotal == nil { + return + } + reason := "unreachable" + if isParentUnavailable(err) { + reason = "parent_unavailable" + } + telemetry.GitMaterializationFailuresTotal.Add(context.Background(), 1, + metric.WithAttributes(w.providerAttrs(attribute.String("reason", reason))...)) +} + +// setPublicationGaugeSources installs the scrape-time sources of the publication gauges: one +// sample per live worker, read from the backlog its loop last published. +func (m *WorkerManager) setPublicationGaugeSources() { + source := func(read func(*BranchWorker) (int64, bool)) telemetry.GaugeSource { + return func() []telemetry.GaugeSample { + var samples []telemetry.GaugeSample + for _, worker := range m.liveWorkers() { + if value, ok := read(worker); ok { + samples = append(samples, telemetry.GaugeSample{Value: value, Attrs: worker.providerAttrs()}) + } + } + return samples + } + } + always := func(value int64) (int64, bool) { return value, true } + dated := func(value int64) (int64, bool) { return value, value != 0 } + telemetry.SetGaugeSource(telemetry.GaugeGitRetainedBytes, source(func(w *BranchWorker) (int64, bool) { + return always(w.backlog.retainedBytes.Load()) + })) + telemetry.SetGaugeSource(telemetry.GaugeGitRetainedWrites, source(func(w *BranchWorker) (int64, bool) { + return always(w.backlog.retainedWrites.Load()) + })) + telemetry.SetGaugeSource(telemetry.GaugeGitIntakePaused, source(func(w *BranchWorker) (int64, bool) { + if w.IntakePaused() != nil { + return 1, true + } + return 0, true + })) + telemetry.SetGaugeSource(telemetry.GaugeGitOldestRetainedWrite, source(func(w *BranchWorker) (int64, bool) { + return dated(w.backlog.oldestWrite.Load()) + })) + telemetry.SetGaugeSource(telemetry.GaugeGitNextRetry, source(func(w *BranchWorker) (int64, bool) { + return dated(w.backlog.nextRetry.Load()) + })) +} + +// publicationGauges are the gauges setPublicationGaugeSources installs. +// +//nolint:gochecknoglobals // a fixed list +var publicationGauges = []string{ + telemetry.GaugeGitRetainedBytes, + telemetry.GaugeGitRetainedWrites, + telemetry.GaugeGitIntakePaused, + telemetry.GaugeGitOldestRetainedWrite, + telemetry.GaugeGitNextRetry, +} + +// publicationEventBuffer bounds the GitTarget events a burst of report changes can queue. A full +// buffer drops the arriving event: the GitTarget's own requeue reads the report later. +const publicationEventBuffer = 256 + +// PublicationEvents carries a GitTarget whose branch worker's publication report changed, for the +// GitTarget controller to re-project its status within one reconcile instead of at its next +// requeue. Nil for a manager not built by NewWorkerManager. +func (m *WorkerManager) PublicationEvents() <-chan event.GenericEvent { return m.publicationEvents } + +// notifyPublication tells every GitTarget on the branch that its worker's report changed. It reads +// the GitTargets from the cache, and never blocks: the worker's loop calls it, by way of a +// goroutine, and a status that lags is better than a loop that waits for a controller. +func (m *WorkerManager) notifyPublication(ctx context.Context, key BranchKey) { + if m.Client == nil || m.publicationEvents == nil { + return + } + var targets configv1alpha3.GitTargetList + if err := m.Client.List(ctx, &targets, client.InNamespace(key.RepoNamespace)); err != nil { + m.Log.V(1).Info("Cannot list the GitTargets of a branch to report its publication", + "branch", key.String(), "error", err.Error()) + return + } + for i := range targets.Items { + target := &targets.Items[i] + if target.Spec.GitProviderRef.Name != key.RepoName || target.Spec.Branch != key.Branch { + continue + } + select { + case m.publicationEvents <- event.GenericEvent{Object: target}: + default: + } + } +} diff --git a/internal/git/publication_retry.go b/internal/git/publication_retry.go deleted file mode 100644 index 708ce868..00000000 --- a/internal/git/publication_retry.go +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package git - -import "time" - -// This file is the worker's retry of a publication that failed: retained writes whose push, or -// whose rebuild before the push, did not succeed. -// -// The cooldown in maybeSchedulePush spaces SUCCESSFUL pushes; it says nothing about failed ones, -// because lastPushAt advances only on success. Without a schedule of its own a failure had two -// outcomes, both wrong: a branch that then went quiet kept its work indefinitely (and any -// CommitRequest riding it stayed WaitingForPush, since the controller never fails a request the -// worker holds), and a branch that kept receiving commits spent one failed push on each of them. -// -// So a failure opens a retry: the next attempt is due one backoff later, 10s doubling to 5m, and -// the push timer is armed for it whether or not anything else arrives. Until then new commits -// accumulate locally instead of pushing. A successful push, or nothing left to push, closes it. -// -// A failure while parent recovery is open is not retried here. Recovery owns that work, with its -// own probe schedule shared across the worker's targets (parent_recovery.go); retrying it on a -// second schedule would spend connections that schedule exists to save. Its deadline still paces -// new commits, though: once the parent is found, recovery's missing-parent hold is gone, so -// without deferring to its deadline every new commit would retry the failing push at once. - -//nolint:gochecknoglobals // vars, not consts, so a test can run the real loop on a short schedule. -var ( - publicationRetryInitialBackoff = 10 * time.Second - publicationRetryMaxBackoff = 5 * time.Minute -) - -const publicationRetryBackoffFactor = 2 - -// publicationRetry is the loop's schedule for retrying a failed publication. The zero value is -// "no failure outstanding". Loop-goroutine only. -type publicationRetry struct { - backoff time.Duration - nextAttempt time.Time - // deferToRecovery marks a failure while parent recovery was open: the next attempt is - // recovery's next probe deadline, and recovery's timer, not the push timer, drives it. - deferToRecovery bool -} - -// pending reports whether a failed publication is waiting for its next attempt. -func (r publicationRetry) pending() bool { - return !r.nextAttempt.IsZero() || r.deferToRecovery -} - -// notePublicationFailed schedules the next attempt after a failed push or rebuild. While parent -// recovery is open the attempt is recovery's, and new commits wait for its deadline. -func (l *branchWorkerEventLoop) notePublicationFailed() { - if l.recovery.active { - l.publicationRetry = publicationRetry{deferToRecovery: true} - return - } - r := &l.publicationRetry - if r.backoff == 0 { - r.backoff = publicationRetryInitialBackoff - } else { - r.backoff = min(publicationRetryBackoffFactor*r.backoff, publicationRetryMaxBackoff) - } - r.nextAttempt = l.w.now().Add(r.backoff) - l.armPushTimerAt(r.nextAttempt) -} - -// notePublicationSettled closes the retry: the work was published, or nothing is retained. -func (l *branchWorkerEventLoop) notePublicationSettled() { - l.publicationRetry = publicationRetry{} -} - -// awaitingPublicationRetry reports that a failed publication's next attempt is not due yet, so a -// new commit must not push early. It makes sure the attempt is still scheduled. -func (l *branchWorkerEventLoop) awaitingPublicationRetry() bool { - r := l.publicationRetry - if r.deferToRecovery { - if !l.recovery.active { - l.publicationRetry = publicationRetry{} - return false - } - return l.w.now().Before(l.recovery.nextProbeAt) - } - if !r.pending() || !l.w.now().Before(r.nextAttempt) { - return false - } - if l.pushTimer == nil { - l.armPushTimerAt(r.nextAttempt) - } - return true -} - -// armPushTimerAt (re)arms the push timer to fire at the given time on the worker's clock. -func (l *branchWorkerEventLoop) armPushTimerAt(at time.Time) { - l.stopPushTimer() - l.pushTimer = time.NewTimer(max(at.Sub(l.w.now()), 0)) -} diff --git a/internal/git/publication_test.go b/internal/git/publication_test.go new file mode 100644 index 00000000..d3b6c076 --- /dev/null +++ b/internal/git/publication_test.go @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// The worker's report of its publication: what every GitTarget on the branch and every save it +// holds says during an outage, and what the metrics read. See +// docs/design/gittarget-branch-worker-pending-writes.md, step 5c. + +import ( + "context" + "testing" + + "github.com/fluxcd/pkg/apis/meta" + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// An outage is reported once, from its start: a later failed attempt keeps that start, a pause is +// added to it, and a landed publication clears it. +func TestPublication_ReportsAnOutageFromItsStart(t *testing.T) { + f, loop, restoreSyncs := outageLoop(t, "publication-outage") + report := f.worker.Publication() + require.True(t, report.Failing) + assert.Contains(t, report.Cause, "connection refused") + assert.False(t, report.IntakePaused) + assert.False(t, report.ParentUnavailable) + require.False(t, report.Since.IsZero()) + + fireRetry(loop) + loop.publishLoopState(0) + require.True(t, loop.retry.pending()) + assert.Equal(t, report.Since, f.worker.Publication().Since, "a later failure keeps the outage's start") + + f.worker.branchBufferMaxBytes = 1 + loop.publishLoopState(0) + assert.True(t, f.worker.Publication().IntakePaused) + assert.Contains(t, f.worker.Publication().Message(), "intake of new changes is paused") + + restoreSyncs() + fireRetry(loop) + loop.publishLoopState(0) + assert.Equal(t, PublicationStatus{}, f.worker.Publication(), "a landed publication clears the report") +} + +// The GitTargets on the branch are told when the report changes, and only then: a retry that fails +// the same way again changes nothing a status would say. +func TestPublication_NotifiesOnlyWhenTheReportChanges(t *testing.T) { + f, loop, restoreSyncs := outageLoop(t, "publication-notify") + notified := 0 + f.worker.publicationReporter = func() { notified++ } + + fireRetry(loop) + loop.publishLoopState(0) + assert.Zero(t, notified, "the same failure again is no news") + + f.worker.branchBufferMaxBytes = 1 + loop.publishLoopState(0) + assert.Equal(t, 1, notified, "the pause is") + + restoreSyncs() + fireRetry(loop) + loop.publishLoopState(0) + assert.Equal(t, 2, notified, "and so is the recovery") +} + +// The backlog an outage leaves is measurable without a status write per tick: what is kept, +// whether intake is paused, how old the oldest kept write is, and when the next retry is due. A +// failure to commit decided writes is counted, which no push metric could see. +func TestPublication_MetricsDescribeTheBacklog(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + f, loop, restoreSyncs := outageLoop(t, "publication-metrics") + manager := &WorkerManager{Log: logr.Discard(), workers: map[BranchKey]*BranchWorker{{Branch: "main"}: f.worker}} + manager.setPublicationGaugeSources() + t.Cleanup(func() { + for _, name := range publicationGauges { + telemetry.SetGaugeSource(name, nil) + } + }) + branch := map[string]string{"branch": "main"} + gauge := func(name string) (int64, bool) { + return telemetry.CollectInt64Sum(reader, "gitopsreverser_"+name, branch) + } + + failures, ok := telemetry.CollectInt64Sum(reader, "gitopsreverser_git_materialization_failures_total", + map[string]string{"branch": "main", "reason": "unreachable"}) + require.True(t, ok, "a decided write the remote could not take is counted") + assert.Positive(t, failures) + + kept, ok := gauge(telemetry.GaugeGitRetainedWrites) + require.True(t, ok) + assert.Equal(t, int64(1), kept) + bytes, _ := gauge(telemetry.GaugeGitRetainedBytes) + assert.Positive(t, bytes) + paused, _ := gauge(telemetry.GaugeGitIntakePaused) + assert.Zero(t, paused) + oldest, ok := gauge(telemetry.GaugeGitOldestRetainedWrite) + require.True(t, ok) + assert.Positive(t, oldest) + nextRetry, ok := gauge(telemetry.GaugeGitNextRetry) + require.True(t, ok) + assert.Equal(t, loop.retry.due.Unix(), nextRetry) + + f.worker.branchBufferMaxBytes = 1 + loop.publishLoopState(0) + paused, _ = gauge(telemetry.GaugeGitIntakePaused) + assert.Equal(t, int64(1), paused) + + restoreSyncs() + fireRetry(loop) + loop.publishLoopState(0) + kept, _ = gauge(telemetry.GaugeGitRetainedWrites) + assert.Zero(t, kept) + _, ok = gauge(telemetry.GaugeGitOldestRetainedWrite) + assert.False(t, ok, "nothing kept, nothing to date") + _, ok = gauge(telemetry.GaugeGitNextRetry) + assert.False(t, ok, "nothing owed, no retry due") +} + +// A change of the report reaches every GitTarget on the branch, and none on another branch: they +// share the worker, so they share its outage, and a second branch's worker is its own. +func TestPublication_ChangesReachEveryGitTargetOnTheBranch(t *testing.T) { + target := func(name, branch string) *configv1alpha3.GitTarget { + return &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "shop"}, + Spec: configv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "repo"}, + Branch: branch, + }, + } + } + k8s := fake.NewClientBuilder().WithScheme(replayScheme(t)).WithObjects( + target("apps", "main"), target("infra", "main"), target("other", "release")).Build() + manager := NewWorkerManager(k8s, logr.Discard(), BranchWorkerLimits{}, itypes.SensitiveResourcePolicy{}) + + manager.notifyPublication(context.Background(), BranchKey{RepoNamespace: "shop", RepoName: "repo", Branch: "main"}) + + var told []string + for len(manager.PublicationEvents()) > 0 { + told = append(told, (<-manager.PublicationEvents()).Object.GetName()) + } + assert.ElementsMatch(t, []string{"apps", "infra"}, told) +} diff --git a/internal/git/refresh.go b/internal/git/refresh.go index 4780a9d7..e6b1d941 100644 --- a/internal/git/refresh.go +++ b/internal/git/refresh.go @@ -54,14 +54,14 @@ func (w *BranchWorker) EnqueueRefresh(req *RefreshRequest) { defer w.pendingResyncsMu.Unlock() if w.stoppingLocked() { w.inflightItems.Add(-1) - w.recordQueueDrop(queueDropRefresh) + w.recordQueueRefusal(queueRefusalRefresh) return } select { case w.eventQueue <- WorkItem{Refresh: req}: default: w.inflightItems.Add(-1) - w.recordQueueDrop(queueDropRefresh) + w.recordQueueRefusal(queueRefusalRefresh) w.Log.V(1).Info("Event queue full, refresh dropped; the next reconcile asks again", "branch", w.Branch, "gitTarget", req.Target.String()) } @@ -107,7 +107,7 @@ func (l *branchWorkerEventLoop) handleRefreshRequest(req *RefreshRequest) { // commits, and the worktree may hold a partial write — which is also why this is the one exit // that does not re-read the folder: a layout resolved from a half-written tree is worse than // a slightly old one. A target mid-cycle is writing, and a write publishes its own layout. - if len(l.pendingWrites) > 0 || l.openWindow != nil || w.worktreeDirty() || w.replayRequired() { + if len(l.pendingWrites) > 0 || l.openWindow != nil || w.worktreeDirty() { w.Log.V(1).Info("Skipping refresh: this branch is mid-cycle", "branch", w.Branch, "gitTarget", req.Target.String()) return @@ -201,7 +201,7 @@ func (l *branchWorkerEventLoop) refreshFromRemote(provider *configv1alpha3.GitPr // different repository. parentBranch := w.ParentBranch() advertisement, err := advertiseRemoteBranchFn( - w.repo.URL, plumbing.NewBranchReferenceName(w.Branch), parentBranch, auth) + ctx, w.repo.URL, plumbing.NewBranchReferenceName(w.Branch), parentBranch, auth) if err != nil { return fmt.Errorf("read the remote advertisement: %w", err) } @@ -424,13 +424,14 @@ type remoteAdvertisement struct { // means the advertisement did not carry the branch, which includes an empty repository — an // observation rather than an error: a branch does not exist without a commit. func advertiseRemoteBranch( + ctx context.Context, remoteURL string, branch plumbing.ReferenceName, parentBranch string, auth []gitclient.Option, ) (remoteAdvertisement, error) { remote := gogit.NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{remoteURL}}) - refs, err := listRemoteRefs(remote, auth) + refs, err := listRemoteRefs(ctx, remote, auth) if err != nil { return remoteAdvertisement{}, err } diff --git a/internal/git/refresh_test.go b/internal/git/refresh_test.go index 9a726c9e..7c294d47 100644 --- a/internal/git/refresh_test.go +++ b/internal/git/refresh_test.go @@ -168,7 +168,7 @@ func TestRefresh_AMovedBranchFetchesAndResets(t *testing.T) { // A reset here would destroy the local commits behind retained writes. func TestRefresh_SkipsABranchMidCycle(t *testing.T) { h := newRefreshHarness(t, "refresh-mid-cycle") - h.commit(false, "retained") + h.commit("retained") h.loop.pendingWrites = h.pending h.reported = nil @@ -534,9 +534,9 @@ func TestEnqueueRefresh_CountsTheDropOnAFullQueue(t *testing.T) { "provider_namespace": h.worker.GitProviderNamespace, "provider_name": h.worker.GitProviderRef, "branch": h.worker.Branch, - "kind": queueDropRefresh, + "kind": queueRefusalRefresh, } - drops, ok := telemetry.CollectInt64Sum(reader, queueDropsMetric, labels) + drops, ok := telemetry.CollectInt64Sum(reader, queueRefusalsMetric, labels) require.True(t, ok, "a dropped refresh must be counted like any other lost work item") assert.Equal(t, int64(1), drops) } diff --git a/internal/git/refusal_observation_test.go b/internal/git/refusal_observation_test.go index 9d8ba83d..cd800cd1 100644 --- a/internal/git/refusal_observation_test.go +++ b/internal/git/refusal_observation_test.go @@ -83,6 +83,7 @@ func refuseResync(t *testing.T, f *ledgerFixture, l *branchWorkerEventLoop, imag }, Result: result, }) + l.endWake(0) var refused *manifestanalyzer.AcceptanceRefusedError require.ErrorAs(t, (<-result).Err, &refused, "the diamond must refuse this write at the boundary") @@ -141,6 +142,34 @@ func TestRefusalTouch_ASecondRefusedEditEarnsItsOwnCommit(t *testing.T) { "a genuinely different refused edit must still move the branch") } +// TestRefusalTouch_AResyncsRefusalIsPushedByTheLoopItself: a refusal a resync discovers decides its +// empty commit while the resync's outcome is being settled, inside the commit pass. Nothing may push +// from inside a pass, so the pass's starter has to schedule it once the pass ends. Found by the +// refusal e2e on #413: the commit was made and never left the checkout. The fixture's refuseResync +// pushes by hand, which is how the unit tests missed it. +func TestRefusalTouch_AResyncsRefusalIsPushedByTheLoopItself(t *testing.T) { + f := standingRefusalFixture(t, "standing-refusal-pushed") + l := newBranchWorkerEventLoop(f.worker, time.Hour) + t.Cleanup(l.stopTimers) + before := remoteCommits(t, f) + + event := overridesDeploymentEvent("ghcr.io/example/podinfo:9.9.9", 3) + scope := deploymentResyncScope() + result := make(chan ResyncResult, 1) + l.applyResync(&ResyncRequest{ + GitTargetName: ledgerTargetName, GitTargetNamespace: "default", Scope: &scope, + ResourceVersion: "1", + Desired: []manifestanalyzer.DesiredResource{{Resource: event.Identifier, Object: event.Object}}, + Result: result, + }) + l.endWake(0) + var refused *manifestanalyzer.AcceptanceRefusedError + require.ErrorAs(t, (<-result).Err, &refused) + + assert.Empty(t, l.pendingWrites, "the empty commit was published") + assert.Equal(t, 1, remoteCommits(t, f)-before, "with no push started by hand") +} + // TestRefusalObservation_IsContentAddressedAndOrderIndependent pins the digest itself: same // objects and issues, same observation, whatever order they arrive in. func TestRefusalObservation_IsContentAddressedAndOrderIndependent(t *testing.T) { @@ -351,6 +380,7 @@ func TestRefusalTouch_AQueuedCommitIsDroppedOnceSomethingElseCoveredIt(t *testin loop.stopRefusalTimer() loop.flushPendingRefusalTouch() + loop.endWake(0) assert.Empty(t, loop.refusalPending, "the entry is consumed either way") limited, _ := w.refusalRateLimited(editingRef()) @@ -381,6 +411,7 @@ func TestRefusalTouch_AnAcceptedSiblingTypeDoesNotRearmAStandingRefusal(t *testi GitTargetName: ledgerTargetName, GitTargetNamespace: "default", Scope: &configMaps, Result: result, }) + l.endWake(0) require.NoError(t, (<-result).Err, "an empty ConfigMap snapshot is accepted") require.NoError(t, f.worker.pushPendingCommits(l.pendingWrites)) l.pendingWrites, l.pendingWritesBytes = nil, 0 @@ -536,6 +567,7 @@ func TestRefusalTouch_RecoveryCancelsACommitQueuedForTheSameCollection(t *testin Desired: []manifestanalyzer.DesiredResource{{Resource: event.Identifier, Object: event.Object}}, Result: result, }) + l.endWake(0) err := (<-result).Err require.NoError(t, f.worker.pushPendingCommits(l.pendingWrites)) l.pendingWrites, l.pendingWritesBytes = nil, 0 @@ -559,6 +591,7 @@ func TestRefusalTouch_RecoveryCancelsACommitQueuedForTheSameCollection(t *testin f.worker.refusalTouchMu.Unlock() l.stopRefusalTimer() l.flushPendingRefusalTouch() + l.endWake(0) require.NoError(t, f.worker.pushPendingCommits(l.pendingWrites)) assert.Equal(t, before, remoteCommits(t, f), diff --git a/internal/git/refusal_touch.go b/internal/git/refusal_touch.go index f9941a03..daf1ff05 100644 --- a/internal/git/refusal_touch.go +++ b/internal/git/refusal_touch.go @@ -485,30 +485,12 @@ func (l *branchWorkerEventLoop) commitRefusalTouch(key refusalKey, detail, obser return } - // An empty commit is only empty on a clean worktree: the refused write that brought us here can - // have left staged changes behind, and they would otherwise ride along in it. - if err := l.recoverRetainedWrites(); err != nil { - l.w.Log.Error(err, "Cannot recover the worktree for the empty commit for a refused write", - "gitTarget", target.String()) - return - } - - // Single-element batch for the same reason every other commit site uses one: the executor - // stamps CommitSHA back into the slice it is given, and the retained write is what the push - // counts from. - batch := []PendingWrite{*pendingWrite} - if err := l.w.commitPendingWrites(batch, len(l.pendingWrites) > 0); err != nil { - l.w.Log.Error(err, "The empty commit for a refused write failed", - "gitTarget", target.String()) - return - } - l.pendingWrites = append(l.pendingWrites, batch...) - l.pendingWritesBytes += batch[0].ByteSize - // Recorded once the commit exists, beside the rate-limit stamp and for the same reason: a - // build or commit failure has produced no trigger, so it must leave the next observation of - // the same refusal free to make one. - l.w.recordRefusalObservation(key, observation) - l.maybeSchedulePush() + // Decided like every other write, and recorded once its commit exists (settleCommitted): a build + // or commit failure has produced no trigger, so it must leave the next observation of the same + // refusal free to make one. Deciding it while a refused write is being settled appends it to the + // pending writes for the running pass, behind the writes decided before it. + pendingWrite.origin.refusal = &refusalTouchOrigin{key: key, observation: observation} + l.decide(*pendingWrite) } // refusalIsAWriteBoundary reports whether this refusal is one where the FOLDER is accepted and diff --git a/internal/git/refusal_touch_test.go b/internal/git/refusal_touch_test.go index dd49d85b..6c72d99d 100644 --- a/internal/git/refusal_touch_test.go +++ b/internal/git/refusal_touch_test.go @@ -174,7 +174,7 @@ func TestRefusalTouch_LandsAnEmptyCommitOnTheRemote(t *testing.T) { require.NoError(t, err) batch := []PendingWrite{*write} - require.NoError(t, f.worker.commitPendingWrites(batch, false)) + require.NoError(t, f.worker.commitPendingWrites(batch)) f.pending = append(f.pending, batch...) f.push() @@ -254,6 +254,7 @@ func TestRefusalTouch_TrailingCommitRechecksConsent(t *testing.T) { // the way the loop calls it. loop.stopRefusalTimer() loop.flushPendingRefusalTouch() + loop.endWake(0) assert.Empty(t, loop.refusalPending, "a target set back to Ignore must not get the commit queued under the old setting") @@ -357,7 +358,7 @@ func TestRefusalTouch_ReplayKeepsAcceptedWritesAndTheEmptyDiff(t *testing.T) { f.publish("prime") // An ordinary accepted write, committed and retained but not yet pushed. - f.commit(false, "accepted") + f.commit("accepted") require.Len(t, f.pending, 1) // The refusal's empty commit joins the same retained batch, exactly as the event loop would @@ -366,7 +367,7 @@ func TestRefusalTouch_ReplayKeepsAcceptedWritesAndTheEmptyDiff(t *testing.T) { f.worker.ctx, itypes.NewResourceReference("target-a", "default"), "unsupported folder content") require.NoError(t, err) batch := []PendingWrite{*touch} - require.NoError(t, f.worker.commitPendingWrites(batch, true)) + require.NoError(t, f.worker.commitPendingWrites(batch)) f.pending = append(f.pending, batch...) // Somebody else moves the branch before our push, so the compare-and-swap rejects it and the @@ -489,6 +490,7 @@ func TestRefusalTouch_OneTargetDoesNotDropAnothersPendingCommit(t *testing.T) { // bravo's consent is withdrawn; alpha's is not, and alpha must survive it. loop.stopRefusalTimer() loop.flushPendingRefusalTouch() + loop.endWake(0) assert.Empty(t, loop.refusalPending, "every due entry is consumed exactly once") limitedAlpha, _ := w.refusalRateLimited(alpha) diff --git a/internal/git/replay_failure_test.go b/internal/git/replay_failure_test.go index 574a7ace..d0e90eac 100644 --- a/internal/git/replay_failure_test.go +++ b/internal/git/replay_failure_test.go @@ -176,7 +176,7 @@ func remoteFileNames(t *testing.T, repoDir string) string { // to completion BEFORE `executePendingWrites` starts, caching one read per target — so failing // those reads always aborts before the first write, whatever the batch size. There is no seam that // fails between two writes, and adding one to production code to reach a state the flag already -// covers by construction is not worth it: `replayRequired` is per-worker, not per-write, so a +// covers by construction is not worth it: `checkoutApplied` counts whole replays, so a // partly-rebuilt batch cannot be settled either. This test pins the batch behaviour that IS // reachable, and the comment records why the other shape is not. func TestReplayFailure_HoldsAWholeBatchNotJustItsFirstWrite(t *testing.T) { @@ -236,7 +236,7 @@ func TestReplayFailure_RecoveryRunsBeforeEveryCommitPathToo(t *testing.T) { restoreAPI := failGitTargetReads(t, f.worker, errors.New("etcdserver: request timed out")) require.Error(t, f.worker.refreshRemoteAndRebuildPendingWrites( f.worker.ctx, loop.pendingWrites, fetchReasonForcedRecheck)) - require.True(t, f.worker.replayRequired()) + require.False(t, loop.checkoutCurrent()) restoreAPI() // A second write arrives. Finalizing it recovers first, so both writes are real commits again. @@ -245,7 +245,7 @@ func TestReplayFailure_RecoveryRunsBeforeEveryCommitPathToo(t *testing.T) { CommitMode: CommitModePerEvent, }}) require.True(t, loop.finalizeOpenWindow()) - assert.False(t, f.worker.replayRequired(), "the commit path rebuilt the stranded write") + assert.True(t, loop.checkoutCurrent(), "the commit path rebuilt the stranded write") loop.pushPending() names := remoteFileNames(t, f.repoDir) @@ -352,7 +352,7 @@ func TestReplayFailure_AResetThatDiesHalfwayIsAlsoHeld(t *testing.T) { require.False(t, f.worker.worktreeDirty(), "no write failed, so the dirty flag is false and cannot describe this") - require.True(t, f.worker.replayRequired(), + require.False(t, loop.checkoutCurrent(), "but the commits behind the retained writes are gone, and that must be recorded") // Without the flag, this push finds the branch already at the remote tip, sends nothing, diff --git a/internal/git/replay_refusal_test.go b/internal/git/replay_refusal_test.go new file mode 100644 index 00000000..0d8c8044 --- /dev/null +++ b/internal/git/replay_refusal_test.go @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// A retained write that the remote now refuses, met during a replay. +// +// A write is accepted against the tree it was first committed on. If the remote moves before the +// write is published, the write is replayed onto the new tip, and the new tip can hold content that +// makes the write's folder unacceptable: somebody else turned it into a kustomize root the writer +// does not support, say. Retrying cannot help, because the content is not going to change by +// itself. Aborting the whole replay used to keep every write behind the refused one waiting +// with it, indefinitely. A refusal met during a replay now settles only its own entry, as a refusal +// at first commit does: it is reported, its save fails, and it is no longer pending. The writes after it +// are replayed and published. +// +// See docs/design/gittarget-branch-worker-pending-writes.md, step 4. + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// replayRefusalTargetB is a second GitTarget on the same branch, in a folder nobody contends, so +// its write is one a refusal in the first target's folder has no reason to touch. +const replayRefusalTargetB = "target-b" + +// newReplayRefusalFixture builds a ledger fixture with two targets on one branch, primed with one +// published commit, and an event loop waiting out its push cooldown. +func newReplayRefusalFixture(t *testing.T, slug string) (*ledgerFixture, *branchWorkerEventLoop) { + t.Helper() + f := newLedgerFixture(t, slug, true) + f.createLedgerTarget("team-a", nil) + createPlainGitTarget(t, f.worker, replayRefusalTargetB, "team-b") + f.publish("prime") + + loop := newBranchWorkerEventLoop(f.worker, time.Hour) + loop.lastPushAt = time.Now() + t.Cleanup(loop.stopTimers) + return f, loop +} + +// retainTwoWrites commits a write into each target's folder, in that order, and keeps both in the +// log. The first carries a save, so its outcome can be read off the request. +func retainTwoWrites(t *testing.T, loop *branchWorkerEventLoop) { + t.Helper() + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("refused-on-replay", "alice", ledgerTargetName)}, + CommitMode: CommitModePerEvent, + }}) + require.NotNil(t, loop.openWindow) + req := attachReq("alice", 0) + req.GitTargetName = ledgerTargetName + req.Message = "save: lands in a folder that is refused before it is pushed" + serviceAttach(loop, req) + require.Len(t, loop.pendingWrites, 1) + require.NotNil(t, loop.pendingWrites[0].CommitRequest) + + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("kept-behind-it", "bob", replayRefusalTargetB)}, + CommitMode: CommitModePerEvent, + }}) + require.True(t, loop.finalizeOpenWindow()) + require.Len(t, loop.pendingWrites, 2) + require.True(t, loop.checkoutCurrent(), "both writes are committed locally") +} + +// refuseFirstTargetsFolder makes the remote refuse every write into team-a from now on: another +// writer turns it into a kustomize root using a feature the writer does not support. Pushing it +// also moves the remote, so the worker's next push is rejected. +func refuseFirstTargetsFolder(f *ledgerFixture) { + f.contend("team-a/kustomization.yaml", hardKustomizeYAML) +} + +// assertOnlyTheRefusedWriteWasDropped checks the outcome both replay paths must reach. +func assertOnlyTheRefusedWriteWasDropped( + t *testing.T, f *ledgerFixture, loop *branchWorkerEventLoop, refusals *[]capturedRefusal, +) { + t.Helper() + names := remoteFileNames(t, f.repoDir) + assert.Contains(t, names, "kept-behind-it", "the write behind the refused one is published") + assert.NotContains(t, names, "refused-on-replay", "the refused write is not") + assert.Contains(t, names, "team-a/kustomization.yaml", "and the other writer's commit is kept") + assert.Empty(t, loop.pendingWrites, "nothing is left pending to block the writes after it") + assert.False(t, loop.retry.pending(), "nothing is owed, so no retry is scheduled") + + require.Len(t, *refusals, 1, "the refusal is reported on the target's status, once") + assert.Equal(t, itypes.NewResourceReference(ledgerTargetName, "default"), (*refusals)[0].target) + + res, resolved := f.worker.LookupCommitRequestOutcome("default", crName, "uid-"+crName) + require.True(t, resolved, "the refused write's save is answered rather than held") + require.Error(t, res.Err, "and it fails: its write will never land") + assert.Contains(t, res.Err.Error(), "kustomization.yaml", "naming what refused it") +} + +// TestReplayRefusal_OnTheContentionPathDropsOnlyItsOwnEntry is the replay the push cycle runs when +// its push is rejected because the remote moved. +func TestReplayRefusal_OnTheContentionPathDropsOnlyItsOwnEntry(t *testing.T) { + f, loop := newReplayRefusalFixture(t, "replay-refusal-contention") + refusals := captureRefusals(f.worker) + retainTwoWrites(t, loop) + refuseFirstTargetsFolder(f) + + loop.pushPending() + + assertOnlyTheRefusedWriteWasDropped(t, f, loop, refusals) +} + +// TestReplayRefusal_OnTheLoopsRebuildDropsOnlyItsOwnEntry is the replay materialize runs when the +// checkout no longer holds the retained writes, before anything is pushed. +func TestReplayRefusal_OnTheLoopsRebuildDropsOnlyItsOwnEntry(t *testing.T) { + f, loop := newReplayRefusalFixture(t, "replay-refusal-rebuild") + refusals := captureRefusals(f.worker) + retainTwoWrites(t, loop) + refuseFirstTargetsFolder(f) + // A write that failed part-way and could not be undone: the checkout must be rebuilt from the + // remote before anything is committed on it or pushed. + f.worker.markWorktreeDirty("test: a failed write left leftovers") + + loop.pushPending() + + assertOnlyTheRefusedWriteWasDropped(t, f, loop, refusals) +} + +// TestReplayRefusal_AnotherFailureStillHoldsEveryWrite: only a refusal is a final answer about one +// write. Any other failure during the same replay says nothing about the write it hit, so the +// attempt is abandoned and every write is kept for the retry, the refused one included. +func TestReplayRefusal_AnotherFailureStillHoldsEveryWrite(t *testing.T) { + f, loop := newReplayRefusalFixture(t, "replay-refusal-other-failure") + refusals := captureRefusals(f.worker) + retainTwoWrites(t, loop) + refuseFirstTargetsFolder(f) + + restoreAPI := failGitTargetReads(t, f.worker, errors.New("etcdserver: request timed out")) + loop.pushPending() + + require.Len(t, loop.pendingWrites, 2, "an aborted replay settles nothing") + assert.Empty(t, *refusals, "and reports no refusal it did not finish deciding") + _, resolved := f.worker.LookupCommitRequestOutcome("default", crName, "uid-"+crName) + assert.False(t, resolved, "the save stays held") + assert.True(t, loop.retry.pending(), "the retry is scheduled") + + restoreAPI() + loop.pushPending() + + assertOnlyTheRefusedWriteWasDropped(t, f, loop, refusals) +} + +// TestReplayRefusal_AResyncAnsweredEarlierIsReportedNotAnsweredAgain: a committed resync has already +// told its caller what it found. When a replay later refuses it, nobody is waiting for it any more, +// so its refusal is reported on the target, the way a live write's is. +func TestReplayRefusal_AResyncAnsweredEarlierIsReportedNotAnsweredAgain(t *testing.T) { + f, loop := newReplayRefusalFixture(t, "replay-refusal-resync") + f.worker.mapper = configMapMapper() + refusals := captureRefusals(f.worker) + + req := &ResyncRequest{ + Desired: []manifestanalyzer.DesiredResource{desiredCM("reconciled-then-refused", "blue")}, + GitTargetName: ledgerTargetName, + GitTargetNamespace: "default", + Result: make(chan ResyncResult, 1), + } + loop.handleQueueItem(resyncItem(req)) + result := <-req.Result + require.NoError(t, result.Err) + require.Equal(t, 1, result.Stats.Created) + require.Len(t, loop.pendingWrites, 1, "the resync is retained as a local commit") + + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("kept-behind-it", "bob", replayRefusalTargetB)}, + CommitMode: CommitModePerEvent, + }}) + require.True(t, loop.finalizeOpenWindow()) + require.Len(t, loop.pendingWrites, 2) + + refuseFirstTargetsFolder(f) + loop.pushPending() + + names := remoteFileNames(t, f.repoDir) + assert.Contains(t, names, "kept-behind-it", "the write behind the refused resync is published") + assert.NotContains(t, names, "reconciled-then-refused", "the refused resync is not") + assert.Empty(t, loop.pendingWrites) + require.Len(t, *refusals, 1, "the refusal is reported on the target's status") + assert.Empty(t, req.Result, "and the caller, answered already, is not answered a second time") +} diff --git a/internal/git/resync_flush.go b/internal/git/resync_flush.go index 56f49f39..08b4b4ed 100644 --- a/internal/git/resync_flush.go +++ b/internal/git/resync_flush.go @@ -77,76 +77,23 @@ func resyncHealKey(req *ResyncRequest) healKey { } } -// prepareBaseForResync gets the worktree into a state the snapshot may honestly be judged against, -// which is three things in order: the RefreshRemote prelude when one was asked for, the fetch a -// resync keeps regardless, and recovery from a worktree a failed write left dirty. +// applyResync decides one revision-pinned resync in order on the worker goroutine. It mirrors the +// atomic-commit path: for a non-heal resync any open live window is finalized first so arrival order +// is preserved (a heal reaches here only when no window is open, so it finalizes nothing); the +// resync is then decided like every other write, and its caller answered when its commit is made +// or fails (settleResyncApplied, settleFailed), or as soon as the remote holds it back +// (replyToDeferredResyncs). A resync that commits nothing stops being pending at once, so it never +// schedules a push. A build or commit failure answers with the error and commits nothing: the +// gatherer already guaranteed the snapshot is complete, so a failure here is a write fault, never a +// partial-snapshot drop. // // A resync keeps fetching, deliberately, even now that a publication does not. The rest of the // write path is safe without one because it always reaches the push, whose advertisement catches a -// base that turned out to be stale. A resync does not: applyResync retains its write only -// `if committed` and schedules a push only `if committed || closedWindow`, so a resync that finds -// nothing to change never opens a connection at all. It would conclude "Git already matches the -// cluster" against a tree nobody had checked, and nothing downstream would ever contradict it. The -// round trip this design removes is the PUBLICATION one, and a resync is not a publication. -// -// It has to be invalidateAndRefresh rather than a bare invalidateBase. commitPendingWrites is -// called with hasPendingCommits set from the retained slice, and ensureBaseForCycle returns early -// on that flag, so on a target holding work a bare invalidation reached nothing at all and the -// snapshot judged a tree nobody had read. That is the exact hazard this function exists to close. -func (l *branchWorkerEventLoop) prepareBaseForResync(req *ResyncRequest) error { - // A parent known to be missing is not fetched again before its probe is due. - if l.w.awaitingParentProbe() { - return errAwaitingParentProbe - } - // The invalidation is skipped when RefreshRemote already fetched a moment ago, which would - // otherwise make a forced recheck pay twice. - if req.RefreshRemote { - if err := l.refreshRemoteForResync(req); err != nil { - return err - } - } else if err := l.invalidateAndRefresh("resync snapshot", fetchReasonForcedRecheck); err != nil { - l.w.Log.Error(err, "Failed to refresh the remote before resync", "resources", len(req.Desired)) - return err - } - - // The refresh above already reset and replayed when anything was retained, so this is a no-op - // on that path; it still covers a dirty tree with nothing to replay. - if err := l.recoverRetainedWrites(); err != nil { - l.w.Log.Error(err, "Failed to recover a dirty worktree before resync", "resources", len(req.Desired)) - return err - } - - return nil -} - -// refreshRemoteForResync runs the RefreshRemote prelude, branching on whether writes are retained: -// a replay keeps them, a plain sync is enough without them. -func (l *branchWorkerEventLoop) refreshRemoteForResync(req *ResyncRequest) error { - if len(l.pendingWrites) > 0 { - if err := l.w.refreshRemoteAndRebuildPendingWrites( - l.w.ctx, l.pendingWrites, fetchReasonForcedRecheck); err != nil { - l.w.Log.Error(err, "Failed to refresh remote before resync and replay pending writes", - "resources", len(req.Desired), - "gitTarget", req.GitTargetNamespace+"/"+req.GitTargetName, - "pendingWrites", len(l.pendingWrites)) - return fmt.Errorf("refresh remote before resync: %w", err) - } - return nil - } - if err := l.w.syncWithRemote(l.w.ctx, fetchReasonForcedRecheck); err != nil { - l.w.Log.Error(err, "Failed to refresh remote before resync", "resources", len(req.Desired)) - return fmt.Errorf("refresh remote before resync: %w", err) - } - return nil -} - -// applyResync applies one revision-pinned resync in order on the worker goroutine. It mirrors the -// atomic-commit path: for a non-heal resync any open live window is finalized first so arrival order -// is preserved (a heal reaches here only when no window is open, so it finalizes nothing); the -// resync is committed as one local commit, retained for the normal cooldown-driven push, and the -// caller is replied to with the plan's change counts. A build or commit failure replies with the -// error and commits nothing — the gatherer already guaranteed the snapshot is complete, so a failure -// here is a write fault, never a partial-snapshot drop. +// base that turned out to be stale. A resync does not: a resync that finds nothing to change is +// neither retained nor pushed, so it would conclude "Git already matches the cluster" against a +// tree nobody had checked, and nothing downstream would ever contradict it. The round trip this +// design removes is the PUBLICATION one, and a resync is not a publication. That fetch also serves +// a forced GitTarget recheck, whose trigger is often "I changed Git; look again". func (l *branchWorkerEventLoop) applyResync(req *ResyncRequest) { l.w.Log.Info("Handling resync request", "resources", len(req.Desired), @@ -158,81 +105,19 @@ func (l *branchWorkerEventLoop) applyResync(req *ResyncRequest) { "pendingWrites", len(l.pendingWrites)) // A heal must never finalize a window (it only ever runs at idle); only a non-heal resync // force-finalizes the open window to preserve arrival order before its mark-and-sweep. - closedWindow := false if !req.Heal { - closedWindow = l.finalizeOpenWindowWithReason(windowFinalizeReasonResyncBeforeApply) - } - if err := l.prepareBaseForResync(req); err != nil { - l.noteParentUnavailable(err, resyncScope(req)) - req.reply(ResyncResult{Err: err}) - return + l.finalizeOpenWindowWithReason(windowFinalizeReasonResyncBeforeApply) } - stats := &ResyncStats{} - committed := false - pendingWrite, err := l.w.buildResyncPendingWrite(l.w.ctx, req, stats) + pendingWrite, err := l.w.buildResyncPendingWrite(l.w.ctx, req, &ResyncStats{}) if err != nil { l.w.Log.Error(err, "Failed to build resync pending write", "resources", len(req.Desired)) req.reply(ResyncResult{Err: err}) return } - pendingWrite.Committed = &committed - - if err := l.w.commitPendingWrites([]PendingWrite{*pendingWrite}, len(l.pendingWrites) > 0); err != nil { - // A refusal reaches the caller on ResyncResult.Err, where the watch layer classifies it - // and blocks the collection. spec.onRefusal needs it HERE too, and this is the path that - // matters: a per-type reconcile evaluates the same objects a live write would, so it is - // normally what discovers a write-boundary refusal first. Hooking only the live-event path - // left the action almost unreachable, because by the time an edit arrives the collection this - // refusal blocks is already blocked. - var refused *manifestanalyzer.AcceptanceRefusedError - if errors.As(err, &refused) { - l.touchBranchForRefusal(req.GitTargetName, req.GitTargetNamespace, err.Error(), refused, - refusalObservationForDesired(req.Desired, refused), req.refusalCollection()) - } - l.w.Log.Error(err, "Resync commit failed; dropping request", "resources", len(req.Desired)) - l.noteParentUnavailable(err, resyncScope(req)) - req.reply(ResyncResult{Err: err}) - return - } - - // The plan was accepted, so the refusal standing over THIS SCOPE is over: the observation the - // last empty commit covered is forgotten and any commit still queued for it is cancelled. This - // is the recovery path the dedupe depends on — a per-type reconcile runs after the reconciler - // reverts the edit, and it is what makes the NEXT refusal, including a re-made byte-identical - // one, a new trigger rather than a repeat. Scoped to the collection this request evaluated, because - // a ConfigMap resync succeeding is no evidence about a Deployment that is still refused. - l.refusalRecovered( - itypes.NewResourceReference(req.GitTargetName, req.GitTargetNamespace), req.refusalCollection()) - - // Only retain the resync's own pending write when it actually committed. A no-op - // resync (e.g. the empty initial snapshot before any rule selects a resource) - // retains nothing of its own. - if committed { - l.pendingWrites = append(l.pendingWrites, *pendingWrite) - l.pendingWritesBytes += pendingWrite.ByteSize - } - l.noteResyncApplied(req.GitTargetNamespace, req.GitTargetName, req.refusalCollection(), committed) - // Schedule a push whenever this request CLOSED a live window — that window's - // commit is now in pendingWrites and must reach the remote — or the resync itself - // committed. Any finalize that closes a window must schedule its push, or the - // window's commit is stranded: committed locally but never pushed (the - // stranded-write fix in the CommitRequest window contract). - // maybeSchedulePush no-ops when nothing is pending, so a pure no-op resync that - // closed no window stays a no-op and does not disturb the push cooldown. - if committed || closedWindow { - l.maybeSchedulePush() - } - l.w.Log.Info("Resync request applied", - "committed", committed, - "closedWindow", closedWindow, - "created", stats.Created, - "updated", stats.Updated, - "deleted", stats.Deleted, - "skipped", stats.Skipped, - "placementSkipped", stats.PlacementSkipped, - "pendingWrites", len(l.pendingWrites)) - req.reply(ResyncResult{Stats: *stats}) + pendingWrite.Committed = new(bool) + pendingWrite.origin.resync = req + l.decide(*pendingWrite) } // buildResyncPendingWrite resolves the GitTarget's write metadata (path, encryption, @@ -278,7 +163,7 @@ func (w *BranchWorker) buildResyncPendingWrite( Targets: map[pendingTargetKey]ResolvedTargetMetadata{ {Name: targetMetadata.Name, Namespace: targetMetadata.Namespace}: targetMetadata, }, - ByteSize: estimateDesiredSize(req.Desired), + ByteSize: req.payloadSize(), }, nil } diff --git a/internal/git/resync_flush_test.go b/internal/git/resync_flush_test.go index 275fc6b7..8f7edac1 100644 --- a/internal/git/resync_flush_test.go +++ b/internal/git/resync_flush_test.go @@ -464,6 +464,7 @@ func TestResync_ReportsWhatItsFetchProved(t *testing.T) { Desired: nil, Result: result, }) + loop.endWake(0) require.NotEmpty(t, reported, "the resync's own fetch must reach the status surface") assert.Equal(t, ObservedByFetch, reported[len(reported)-1].By, diff --git a/internal/git/resync_heal_test.go b/internal/git/resync_heal_test.go index 1d178dc1..f865471e 100644 --- a/internal/git/resync_heal_test.go +++ b/internal/git/resync_heal_test.go @@ -46,6 +46,7 @@ func TestHandleResyncRequest_HealDefersWhileWindowOpenThenApplies(t *testing.T) Desired: nil, Result: healCh, }) + loop.endWake(0) // It must be parked, not applied: the window is still open and the heal has not replied. require.NotNil(t, loop.openWindow, "a heal must never force-finalize the open window") @@ -61,7 +62,7 @@ func TestHandleResyncRequest_HealDefersWhileWindowOpenThenApplies(t *testing.T) // The window finalizes on its own (silence boundary) and the loop drains the parked heal. loop.finalizeOpenWindow() - loop.applyDeferredHeals() + loop.endWake(0) require.Empty(t, loop.deferredHeals, "the heal applies once the window is idle") res := <-healCh @@ -92,6 +93,7 @@ func TestHandleResyncRequest_AtomicDrainsDeferredHealFirst(t *testing.T) { GitTargetName: "team-a", GitTargetNamespace: "default", Scope: &scope, Heal: true, Result: healCh, }) + loop.endWake(0) require.Len(t, loop.deferredHeals, 1, "the heal parks behind the open window") // An atomic request arrives: it finalizes the window, and the parked heal must drain at that @@ -128,7 +130,7 @@ func TestHandleResyncRequest_HealDoesNotStealSiblingCommitRequestWindow(t *testi Events: []Event{configMapTargetEvent("held", "alice", crTarget)}, CommitMode: CommitModePerEvent, }}) - loop.serviceCommitRequests() + loop.endWake(0) require.NotNil(t, loop.openWindow, "the edit must open a window") require.NotNil(t, loop.openWindow.pendingCR, "the window must carry the attached CommitRequest") @@ -142,6 +144,7 @@ func TestHandleResyncRequest_HealDoesNotStealSiblingCommitRequestWindow(t *testi Heal: true, Result: healCh, }) + loop.endWake(0) // The sibling's window and its CommitRequest must be untouched, and the heal merely parked. require.Len(t, loop.deferredHeals, 1, "the heal is parked, not applied") diff --git a/internal/git/resync_push_test.go b/internal/git/resync_push_test.go index db20a7d4..4087d923 100644 --- a/internal/git/resync_push_test.go +++ b/internal/git/resync_push_test.go @@ -63,8 +63,7 @@ func TestEnqueueResync_CoalescesSameScope(t *testing.T) { superseded := make(chan ResyncResult, 1) require.True(t, w.EnqueueResync(&ResyncRequest{ GitTargetNamespace: "ns", GitTargetName: "target", ResourceVersion: "1", - RefreshRemote: true, - Result: superseded, + Result: superseded, })) // The queue now has no free slot, yet a second request for the SAME scope is @@ -87,9 +86,8 @@ func TestEnqueueResync_CoalescesSameScope(t *testing.T) { require.Len(t, w.eventQueue, 1, "coalescing must not consume a second FIFO slot") item := <-w.eventQueue require.NotNil(t, item.Resync) - current := w.takePendingResync(item.Resync) + current := takenResync(w, item.Resync) assert.Equal(t, "2", current.ResourceVersion, "the marker runs the newest request for its scope") - assert.True(t, current.RefreshRemote, "coalescing must preserve a pending remote refresh") // The key is cleared, so the next resync for that scope queues a fresh marker. require.True(t, w.EnqueueResync(&ResyncRequest{ @@ -118,7 +116,7 @@ func TestEnqueueResync_CoalescingCarriesTheSurvivingRequest(t *testing.T) { item := <-w.eventQueue require.NotNil(t, item.Resync) - current := w.takePendingResync(item.Resync) + current := takenResync(w, item.Resync) assert.Equal(t, "2", current.ResourceVersion, "the newer snapshot is the one that runs, at the marker's position") assert.Equal(t, collection, current.SourceCollection, "and it still names the collection that gathered it") @@ -175,7 +173,7 @@ func raceEnqueuesOnOneScope(t *testing.T) (int, int, int) { wg.Wait() for len(w.eventQueue) > 0 { - if item := <-w.eventQueue; item.Resync != nil && item.Resync.GitTargetName == "contended" { + if item := <-w.eventQueue; item.Resync != nil && item.Resync.request.GitTargetName == "contended" { markers++ } } @@ -242,7 +240,7 @@ func TestEnqueueResync_ASelectorChangedBackDoesNotJumpTheOtherSelection(t *testi var order []string for range 3 { item := <-w.eventQueue - order = append(order, w.takePendingResync(item.Resync).ResourceVersion) + order = append(order, takenResync(w, item.Resync).ResourceVersion) } assert.Equal(t, []string{"100", "101", "102"}, order) } @@ -303,10 +301,10 @@ func TestEnqueueResync_ALabelExitFencesTheSelectedSnapshot(t *testing.T) { require.Len(t, w.eventQueue, 3, "the re-entry snapshot queues behind the exit") first := <-w.eventQueue - assert.Equal(t, "100", w.takePendingResync(first.Resync).ResourceVersion) + assert.Equal(t, "100", takenResync(w, first.Resync).ResourceVersion) assert.NotNil(t, (<-w.eventQueue).Request) last := <-w.eventQueue - assert.Equal(t, "103", w.takePendingResync(last.Resync).ResourceVersion) + assert.Equal(t, "103", takenResync(w, last.Resync).ResourceVersion) } // TestHandleResyncRequest_ClosedWindowIsPushedEvenWhenNoOpResync pins the @@ -345,6 +343,7 @@ func TestHandleResyncRequest_ClosedWindowIsPushedEvenWhenNoOpResync(t *testing.T Desired: nil, Result: resultCh, }) + loop.endWake(0) res := <-resultCh require.NoError(t, res.Err) require.Zero(t, res.Stats.Created+res.Stats.Updated+res.Stats.Deleted, @@ -401,12 +400,42 @@ func TestEnqueueResync_DoesNotCoalescePastQueuedWrites(t *testing.T) { require.Len(t, w.eventQueue, 3, "the later resync takes its own slot rather than coalescing") firstMarker := <-w.eventQueue require.NotNil(t, firstMarker.Resync) - assert.Equal(t, "100", w.takePendingResync(firstMarker.Resync).ResourceVersion, + assert.Equal(t, "100", takenResync(w, firstMarker.Resync).ResourceVersion, "the earlier marker runs the snapshot it carried, not the newer one") assert.NotNil(t, (<-w.eventQueue).Request, "the write keeps its position between the snapshots") lastMarker := <-w.eventQueue require.NotNil(t, lastMarker.Resync) - assert.Equal(t, "103", w.takePendingResync(lastMarker.Resync).ResourceVersion) + assert.Equal(t, "103", takenResync(w, lastMarker.Resync).ResourceVersion) +} + +// A resync coalesced into a queued marker owns that marker's position. When writes queued behind +// the marker then make a newer resync take its own slot, the marker must still run the request +// coalesced into it: that request was told it was enqueued, and its caller waits for its answer. +// It used to run the superseded request it first carried instead, which had been answered already, +// and the coalesced one was never run nor answered. +func TestEnqueueResync_AReleasedMarkerRunsTheRequestCoalescedIntoIt(t *testing.T) { + w := &BranchWorker{Log: logr.Discard(), Branch: "main", eventQueue: make(chan WorkItem, 4)} + scope := resyncScopePtr(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, "app") + resync := func(rv string) *ResyncRequest { + return &ResyncRequest{GitTargetNamespace: "ns", GitTargetName: "target", ResourceVersion: rv, + Scope: scope, Result: make(chan ResyncResult, 1)} + } + first, coalesced, last := resync("100"), resync("101"), resync("103") + + require.True(t, w.EnqueueResync(first)) + require.True(t, w.EnqueueResync(coalesced), "coalesced into the first marker's position") + require.ErrorIs(t, (<-first.Result).Err, ErrResyncSuperseded) + require.True(t, w.Enqueue(liveEvent("target", "app")), "a write now sits behind that marker") + require.True(t, w.EnqueueResync(last), "so the newer resync takes its own slot") + + marker := <-w.eventQueue + require.NotNil(t, marker.Resync) + assert.Same(t, coalesced, takenResync(w, marker.Resync), + "the marker runs the request coalesced into it, not the one already answered as superseded") + assert.NotNil(t, (<-w.eventQueue).Request) + lastMarker := <-w.eventQueue + require.NotNil(t, lastMarker.Resync) + assert.Same(t, last, takenResync(w, lastMarker.Resync)) } // TestEnqueueResync_DoesNotCoalescePastQueuedAttach pins the same fence for a @@ -470,7 +499,7 @@ func TestEnqueueResync_CoalescesPastUnrelatedWrites(t *testing.T) { require.Len(t, w.eventQueue, 3, "coalescing still costs no extra FIFO slot") marker := <-w.eventQueue require.NotNil(t, marker.Resync) - assert.Equal(t, "103", w.takePendingResync(marker.Resync).ResourceVersion) + assert.Equal(t, "103", takenResync(w, marker.Resync).ResourceVersion) } // TestEnqueueResync_FenceHoldsUnderConcurrentWrites drives writes and resyncs for one diff --git a/internal/git/retry.go b/internal/git/retry.go new file mode 100644 index 00000000..45c50e14 --- /dev/null +++ b/internal/git/retry.go @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import "time" + +// This file is the worker's one retry schedule: the deadline at which the work it still owes is +// attempted again, whatever made the last attempt fail. +// +// The cooldown in maybeSchedulePush spaces SUCCESSFUL pushes; it says nothing about failed ones, +// because lastPushAt advances only on success. Without a schedule of its own a failure had two +// outcomes, both wrong: a branch that then went quiet kept its work indefinitely (and any +// CommitRequest riding it stayed WaitingForPush, since the controller never fails a request the +// worker holds), and a branch that kept receiving commits spent one failed attempt on each of them. +// +// So a failed attempt schedules the next one: due one backoff later, 10s doubling to 5m, on a timer +// of its own, whether or not anything else arrives. Until then new commits accumulate instead of +// pushing. A successful publication, or nothing left owed, clears it. +// +// It used to be two schedules, one for a failed push and one for parent recovery's probe, and the +// two had to hand the work back and forth: a failure while recovery was open deferred to recovery's +// deadline, or every new commit retried at once. There is one deadline now, and what is due when it +// fires decides what the attempt is: while the parent is missing, one advertisement looking for it +// (parent_recovery.go); otherwise, materializing the pending writes and pushing them. + +//nolint:gochecknoglobals // vars, not consts, so a test can run the real loop on a short schedule. +var ( + retryInitialBackoff = 10 * time.Second + retryMaxBackoff = 5 * time.Minute +) + +const retryBackoffFactor = 2 + +// retrySchedule is the loop's one retry deadline. The zero value is "nothing owed a retry". +// Loop-goroutine only. +type retrySchedule struct { + backoff time.Duration + due time.Time + timer *time.Timer + // cause is the failure the schedule is waiting out: what the last attempt met. A caller that + // asks for the remote before the attempt is due hears it instead of spending a connection to + // learn it again. + cause error + // since is when the first of these failed attempts was made. It stays put across retries, so + // the report of the outage does not change with every attempt. + since time.Time +} + +// pending reports whether a failed attempt is waiting for its next one. +func (r *retrySchedule) pending() bool { return !r.due.IsZero() } + +// scheduleRetry makes the next attempt due one backoff from now, doubling the backoff since the last +// failure up to its cap. Whoever observes a failed attempt calls it, once for that attempt, with the +// failure it met. +func (l *branchWorkerEventLoop) scheduleRetry(cause error) { + r := &l.retry + r.cause = cause + if r.due.IsZero() { + r.since = l.w.now() + } + if r.backoff == 0 { + r.backoff = retryInitialBackoff + } else { + r.backoff = min(retryBackoffFactor*r.backoff, retryMaxBackoff) + } + r.due = l.w.now().Add(r.backoff) + if r.timer != nil { + r.timer.Stop() + } + r.timer = time.NewTimer(max(r.due.Sub(l.w.now()), 0)) + l.publishRecovery() +} + +// endOutage closes the retry and parent recovery together: a publication landed, or nothing is owed +// any more. Either ends whatever the last failed attempt was waiting out, and with it the pause on +// intake (intake.go). +func (l *branchWorkerEventLoop) endOutage() { + l.recovery = parentRecovery{} + l.clearRetry() +} + +// clearRetry closes the schedule. It also publishes the recovery latch, which reads the schedule's +// deadline. +func (l *branchWorkerEventLoop) clearRetry() { + if l.retry.timer != nil { + l.retry.timer.Stop() + } + l.retry = retrySchedule{} + l.publishRecovery() +} + +// awaitingRetry reports that a failed attempt's next one is not due yet, so a new commit must not +// push early. +func (l *branchWorkerEventLoop) awaitingRetry() bool { + return l.retry.pending() && l.w.now().Before(l.retry.due) +} + +// runRetry is the deadline. While parent recovery is open, the attempt is its probe; otherwise it +// is the publication, which settles the schedule or moves it on. +func (l *branchWorkerEventLoop) runRetry() { + if l.recovery.active { + l.runParentProbe() + return + } + l.pushPending() +} + +func (l *branchWorkerEventLoop) retryTimerC() <-chan time.Time { + if l.retry.timer == nil { + return nil + } + return l.retry.timer.C +} diff --git a/internal/git/publication_retry_test.go b/internal/git/retry_test.go similarity index 77% rename from internal/git/publication_retry_test.go rename to internal/git/retry_test.go index 8f734005..a3c6db3f 100644 --- a/internal/git/publication_retry_test.go +++ b/internal/git/retry_test.go @@ -39,10 +39,11 @@ func failNextPushes(t *testing.T, n int32) *atomic.Int32 { return &attempts } -// firePushTimer does what the event loop does when the push timer fires. -func firePushTimer(l *branchWorkerEventLoop) { - l.pushTimer = nil - l.pushPending() +// fireRetry does what the event loop does when the retry deadline fires. +func fireRetry(l *branchWorkerEventLoop) { + l.retry.timer = nil + l.runRetry() + l.endWake(0) } func seedMain(t *testing.T) func(string) { @@ -69,17 +70,17 @@ func TestPublicationRetry_AQuietBranchPublishesAfterAFailedPush(t *testing.T) { require.Equal(t, int32(1), attempts.Load(), "the first push was attempted, and failed") require.Len(t, f.loop.pendingWrites, 1, "the work is retained") assert.True(t, f.ref("feature").IsZero()) - require.NotNil(t, f.loop.pushTimer, "the failure armed the push timer") - assert.Equal(t, clock.Now().Add(publicationRetryInitialBackoff), f.loop.publicationRetry.nextAttempt) + require.NotNil(t, f.loop.retry.timer, "the failure armed the retry") + assert.Equal(t, clock.Now().Add(retryInitialBackoff), f.loop.retry.due) - clock.advance(publicationRetryInitialBackoff) - firePushTimer(f.loop) + clock.advance(retryInitialBackoff) + fireRetry(f.loop) assert.Equal(t, int32(2), attempts.Load()) assert.False(t, f.ref("feature").IsZero(), "published with no further write") assert.Empty(t, f.loop.pendingWrites) - assert.False(t, f.loop.publicationRetry.pending(), "a success closes the retry") - assert.Nil(t, f.loop.pushTimer) + assert.False(t, f.loop.retry.pending(), "a success closes the retry") + assert.Nil(t, f.loop.retry.timer) } // The mirror image: lastPushAt advances only on success, so before this an expired cooldown let @@ -97,24 +98,24 @@ func TestPublicationRetry_NewCommitsDoNotBypassTheBackoff(t *testing.T) { liveWrite(f.loop, "cm3") assert.Equal(t, int32(1), attempts.Load(), "commits before the deadline do not push") require.Len(t, f.loop.pendingWrites, 3) - require.NotNil(t, f.loop.pushTimer, "and the retry stays scheduled") + require.NotNil(t, f.loop.retry.timer, "and the retry stays scheduled") - clock.advance(publicationRetryInitialBackoff) - firePushTimer(f.loop) + clock.advance(retryInitialBackoff) + fireRetry(f.loop) require.Equal(t, int32(2), attempts.Load(), "the retry fails again") - assert.Equal(t, clock.Now().Add(2*publicationRetryInitialBackoff), f.loop.publicationRetry.nextAttempt, + assert.Equal(t, clock.Now().Add(2*retryInitialBackoff), f.loop.retry.due, "and the next wait is twice as long") - clock.advance(publicationRetryInitialBackoff) + clock.advance(retryInitialBackoff) liveWrite(f.loop, "cm4") assert.Equal(t, int32(2), attempts.Load(), "still not due") - clock.advance(publicationRetryInitialBackoff) - firePushTimer(f.loop) + clock.advance(retryInitialBackoff) + fireRetry(f.loop) assert.Equal(t, int32(3), attempts.Load()) assert.Empty(t, f.loop.pendingWrites, "all four writes published in one push") assert.False(t, f.ref("feature").IsZero()) - assert.False(t, f.loop.publicationRetry.pending()) + assert.False(t, f.loop.retry.pending()) } // The schedule doubles up to its cap, and a settled publication starts it over. @@ -124,26 +125,27 @@ func TestPublicationRetry_BackoffDoublesToTheCapAndResets(t *testing.T) { loop := newBranchWorkerEventLoop(w, time.Second) t.Cleanup(loop.stopTimers) + unreachable := errors.New("dial tcp: connection refused") var waits []time.Duration for range 8 { - loop.notePublicationFailed() - waits = append(waits, loop.publicationRetry.nextAttempt.Sub(clock.Now())) + loop.scheduleRetry(unreachable) + waits = append(waits, loop.retry.due.Sub(clock.Now())) } assert.Equal(t, []time.Duration{ 10 * time.Second, 20 * time.Second, 40 * time.Second, 80 * time.Second, 160 * time.Second, 5 * time.Minute, 5 * time.Minute, 5 * time.Minute, }, waits) - loop.notePublicationSettled() - assert.False(t, loop.publicationRetry.pending()) - loop.notePublicationFailed() - assert.Equal(t, clock.Now().Add(publicationRetryInitialBackoff), loop.publicationRetry.nextAttempt) + loop.clearRetry() + assert.False(t, loop.retry.pending()) + loop.scheduleRetry(unreachable) + assert.Equal(t, clock.Now().Add(retryInitialBackoff), loop.retry.due) } -// A push that fails because the parent branch is missing belongs to parent recovery, which retries -// it on the worker's probe schedule. A second schedule here would spend the connections that one -// exists to save. -func TestPublicationRetry_ParentRecoveryOwnsAMissingParent(t *testing.T) { +// A push that fails because the parent branch is missing opens parent recovery, and its next attempt +// is the probe, on the worker's one retry schedule: there is no second schedule to spend the +// connections the probe exists to save. +func TestPublicationRetry_AMissingParentIsProbedOnTheOneSchedule(t *testing.T) { f := newRealServerNewBranch(t, "retry-parent", seedMainAndRelease(t)) useTestClock(f.worker) f.worker.SetParentBranch("release") @@ -157,8 +159,10 @@ func TestPublicationRetry_ParentRecoveryOwnsAMissingParent(t *testing.T) { require.Len(t, f.loop.pendingWrites, 1) open, _ := f.worker.ParentRecovery() require.True(t, open, "parent recovery holds the work") - assert.True(t, f.loop.publicationRetry.deferToRecovery, "the publication retry defers to it") - assert.Nil(t, f.loop.pushTimer, "and arms no timer of its own") + require.True(t, f.loop.retry.pending(), "the next attempt is scheduled") + assert.Equal(t, f.worker.now().Add(retryInitialBackoff), f.loop.retry.due, + "a new latch starts the schedule over: the probe is one initial backoff away") + assert.Nil(t, f.loop.pushTimer, "the push timer is only the success cooldown") } // Once the parent is back, recovery's missing-parent hold is gone but its obligation stays open @@ -178,18 +182,18 @@ func TestPublicationRetry_AFailureAfterTheParentReturnsWaitsForRecovery(t *testi gitIn(t, f.repoDir, "branch", "release", "main") // the parent returns attempts := failNextPushes(t, 100) - clock.advance(parentProbeInitialBackoff) + clock.advance(retryInitialBackoff) f.loop.lastPushAt = time.Time{} f.loop.runParentProbe() require.True(t, f.loop.recovery.found) require.Equal(t, int32(1), attempts.Load(), "recovery's attempt failed") - require.True(t, clock.Now().Before(f.loop.recovery.nextProbeAt)) + require.True(t, clock.Now().Before(f.loop.retry.due)) liveWrite(f.loop, "cm2") liveWrite(f.loop, "cm3") assert.Equal(t, int32(1), attempts.Load(), "commits before recovery's deadline do not retry") - clock.advance(parentProbeMaxBackoff) + clock.advance(retryMaxBackoff) f.loop.runParentProbe() assert.Equal(t, int32(2), attempts.Load(), "recovery's next deadline does") } @@ -198,9 +202,9 @@ func TestPublicationRetry_AFailureAfterTheParentReturnsWaitsForRecovery(t *testi // held (the controller never fails a held request), so before this it stayed WaitingForPush until // some unrelated write came along. It now resolves Committed on its own. func TestPublicationRetry_AHeldCommitRequestIsCommittedWithoutAnotherWrite(t *testing.T) { - original := publicationRetryInitialBackoff - publicationRetryInitialBackoff = 50 * time.Millisecond - t.Cleanup(func() { publicationRetryInitialBackoff = original }) + original := retryInitialBackoff + retryInitialBackoff = 50 * time.Millisecond + t.Cleanup(func() { retryInitialBackoff = original }) f := newRealServerNewBranch(t, "retry-held-commit-request", seedMain(t)) attempts := failNextPushes(t, 1) diff --git a/internal/git/secret_write_test.go b/internal/git/secret_write_test.go index bc718286..7a00ae14 100644 --- a/internal/git/secret_write_test.go +++ b/internal/git/secret_write_test.go @@ -128,7 +128,7 @@ func TestBranchWorker_SecretEncryptionFailureDoesNotWritePlaintext(t *testing.T) pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - err = worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false) + err = worker.commitPendingWrites([]PendingWrite{*pendingWrite}) require.Error(t, err) assert.Contains(t, err.Error(), "secret encryption is required") @@ -183,7 +183,7 @@ func TestBranchWorker_SecretWritesSOPSPath(t *testing.T) { pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) sopsPath := filepath.Join(worker.repoPath(), "default", "secrets", "test-secret.sops.yaml") assert.FileExists(t, sopsPath) @@ -237,7 +237,7 @@ func TestBranchWorker_DeleteSecretRemovesSOPSPath(t *testing.T) { pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) localRepoPath := worker.repoPath() _, statErr := os.Stat(filepath.Join(localRepoPath, "v1", "secrets", "default", "test-secret.yaml")) @@ -284,7 +284,7 @@ func TestBranchWorker_DoesNotBootstrapRootSOPSConfig(t *testing.T) { pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) _, statErr := os.Stat(filepath.Join(worker.repoPath(), sopsConfigFileName)) assert.ErrorIs(t, statErr, os.ErrNotExist) @@ -315,7 +315,7 @@ func TestBranchWorker_DoesNotCreateBootstrapOnlyCommit(t *testing.T) { pendingWrite, err := worker.buildGroupedPendingWrite(worker.ctx, []Event{event}) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) repo, err := gogit.PlainOpen(worker.repoPath()) require.NoError(t, err) diff --git a/internal/git/types.go b/internal/git/types.go index 6b404a7b..99c8040a 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -8,6 +8,7 @@ import ( "net/url" "sort" "strings" + "time" gogit "github.com/go-git/go-git/v6" "github.com/go-git/go-git/v6/plumbing" @@ -387,6 +388,24 @@ type PendingWrite struct { // render failed at commit time. Zero means "not stamped", so messageSource() recomputes. committedMessageSource messageResolution + // committedOnce is set once the loop has committed this write at least once. It stays set after + // a reset discards that commit: whether the checkout holds it now is checkoutApplied's to say. A + // decided write the loop could not commit yet (the remote was unreachable) stays pending with it + // unset; such writes always form the tail of the pending writes. Loop-goroutine only. + committedOnce bool + // replayRefusal is the refusal the last completed replay met for this write: the tree the write + // was replayed onto refuses it, so the replay skipped it and the checkout does not hold it. The + // loop settles it out of the pending writes (settleReplayRefusals). Stamped by replayPendingWrites. + replayRefusal error + // charge is what keeping this write was charged against the retained-byte budget when it was + // decided, and is refunded when it stops being pending. See retainedCharge. Loop-goroutine only. + charge int64 + // decidedAt is when the write was decided, which dates the oldest kept write. Loop-goroutine + // only. + decidedAt time.Time + // origin is what the write's outcome is settled against. See writeOrigin. + origin writeOrigin + // CommitSHA is the hash of the commit this write created, captured in // executePendingWrite and refreshed when the write is re-executed on a // rebase-replay (so it is never a stale pre-rebase hash). Zero when the write @@ -402,15 +421,20 @@ type WorkItem struct { // Attach is a CommitRequest attach: bind a message to the author's window and // let its timers close that window. Attach *AttachCommitRequest - // Resync is a streaming-snapshot resync request (M8): a synchronous + // Resync is the FIFO position of a streaming-snapshot resync request (M8): a synchronous // request/reply that materialises a GitTarget's complete desired set. - Resync *ResyncRequest + Resync *resyncMarker // Refresh asks the worker to re-prove where its branch is on the remote. It is the only // work item that never writes anything: see RefreshRequest. Refresh *RefreshRequest // Withdraw cancels a CommitRequest the worker has not acted on yet. It rides the same FIFO as // the attach, so it is always handled after every attach sent before it. Withdraw *AttachCommitRequest + + // charge is what a write or an attach was charged against the intake budget at enqueue, and is + // released once the loop has handled it. A resync's is on its marker, because coalescing can + // change it while the marker waits. See intake.go. + charge int64 } // ResyncScope restricts a resync's mark-and-sweep to the slice of the mirror the desired @@ -498,10 +522,6 @@ type ResyncRequest struct { // for idle recurs on every silence timeout, so it never starves. A first-sync backfill is NOT // a heal: it must establish initial state promptly. Heal bool - // RefreshRemote asks the worker to fetch/reset to the remote tip before evaluating the - // acceptance gate. Forced GitTarget rechecks use it because their trigger is often "I changed - // Git; look again", and the local checkout may still hold the refused revision. - RefreshRemote bool // SourceCollection names the target-watch collection that gathered this snapshot. Zero for a // whole-GitTarget resync, which speaks for no single collection. Diagnostic only: nothing // filters the queue on it. See source_collection.go. @@ -509,6 +529,10 @@ type ResyncRequest struct { // Result receives exactly one reply. It is buffered (cap 1) by the emitter so // the worker never blocks delivering it. Result chan ResyncResult + + // payloadBytes is the snapshot's estimated serialized size once sized is set. See payloadSize. + payloadBytes int64 + sized bool } // refusalCollection is the watched collection this request speaks for: its scope's collection for a per-type @@ -532,19 +556,21 @@ type resyncKey struct { scope string } -// pendingResync is the coalescing entry for one resyncKey: the current request for -// that key, and whether anything for its scope has been queued behind the marker -// that represents it in the FIFO. Once tailPassed is set the marker's position is -// no longer a safe place to run a newer snapshot — see the pendingResyncs field on -// BranchWorker, and "Queue ordering and coalescing" in docs/design/target-watch-plan.md. -type pendingResync struct { - // marker is the request whose pointer sits on the FIFO for this key. It is fixed - // for the entry's life: coalescing swaps request, never marker. Identifying the - // entry by its marker is what keeps a released key unambiguous — once a later - // request re-inserts the same key, the older marker must run the payload it - // carried rather than pick up the newer entry. - marker *ResyncRequest - request *ResyncRequest +// resyncMarker is a queued resync's place in the FIFO, and the request it runs when it comes up. The +// FIFO holds the marker, never a request, so a request coalescing replaced is not kept alive by the +// queue: only the current one is reachable, and only the current one is charged. +// +// While its key is open (BranchWorker.pendingResyncs), a newer request for the same key replaces +// request. Once anything for its scope is queued behind it (tailPassed) the marker's position is no +// longer a safe place to run a newer snapshot, and the key is released: the marker keeps the request +// it holds, and the next request for the key takes a marker of its own. See "Queue ordering and +// coalescing" in docs/design/target-watch-plan.md. +type resyncMarker struct { + // request, charge and tailPassed are guarded by pendingResyncsMu. + request *ResyncRequest + // charge is what the intake budget was charged for the marker: its current request's, adjusted + // whenever coalescing replaces it. See intake.go. + charge int64 tailPassed bool } @@ -674,6 +700,10 @@ type Event struct { // non-stream producer (reconcile, bootstrap, the admission path). Diagnostic only: // nothing filters the queue on it. See source_collection.go. SourceCollection types.CollectionKey + + // payloadBytes is the object's estimated serialized size once sized is set. See payloadSize. + payloadBytes int64 + sized bool } // IsFieldPatch reports whether the event carries a bounded field patch instead of diff --git a/internal/git/worker_manager.go b/internal/git/worker_manager.go index a466fa31..8f027bbc 100644 --- a/internal/git/worker_manager.go +++ b/internal/git/worker_manager.go @@ -10,6 +10,7 @@ import ( "github.com/go-logr/logr" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" "github.com/ConfigButler/gitops-reverser/internal/telemetry" @@ -47,17 +48,19 @@ const DefaultBranchWorkerQueueDepth = 1000 type BranchWorkerLimits struct { // MaxBufferBytes caps totalRetainedBytes: the open commit window plus the writes // committed locally and retained for replay until a push succeeds. Tripping it - // finalizes the window early, ignoring the commit cadence. + // finalizes the window early, ignoring the commit cadence. While a failed attempt waits + // for its retry it is also the intake budget, covering queued work too. See intake.go. MaxBufferBytes int64 // QueueDepth is the depth of the event queue, and it is a HARD drop boundary: the - // enqueue is deliberately non-blocking, so a full queue throws the item away and - // counts git_queue_drops_total rather than stalling the watch path behind a slow - // remote. + // enqueue is deliberately non-blocking, so a full queue refuses the item and counts + // git_queue_refusals_total rather than stalling the watch path behind a slow remote. The + // producer keeps it and offers it again: the watch redelivers from its unadvanced cursor. // - // MaxBufferBytes does NOT cover this queue. That cap is accounted only once the loop - // DEQUEUES an item, so whatever is still on the channel is bounded by count alone and - // costs memory ON TOP of it. The channel itself is trivial (a WorkItem is three + // On a healthy branch MaxBufferBytes does NOT cover this queue: the window cap is accounted + // only once the loop DEQUEUES an item, so whatever is still on the channel is bounded by count + // alone and costs memory ON TOP of it. During an outage the intake gate charges queued + // payload against MaxBufferBytes at enqueue (intake.go), in serialized bytes. The channel itself is trivial (a WorkItem is three // pointers); what it retains is not, because each live-event item holds a sanitized // object as an unstructured map, measured at roughly SIX times the bytes it serializes // to. Budget QueueDepth x serialized size x ~6 per SATURATED worker: ~5MiB at the @@ -156,6 +159,10 @@ type WorkerManager struct { // empty indefinitely. Recorded here instead, a pending recovery survives every failure until // something acknowledges it. replacements map[BranchKey]struct{} + + // publicationEvents carries the GitTargets whose worker's publication report changed. See + // PublicationEvents. + publicationEvents chan event.GenericEvent } // NewWorkerManager creates a new worker manager. limits bounds every worker this manager @@ -176,6 +183,7 @@ func NewWorkerManager( remotes: make(map[BranchKey]RemoteObservation), replacements: make(map[BranchKey]struct{}), renderFidelityGate: NewRenderFidelityGate(), + publicationEvents: make(chan event.GenericEvent, publicationEventBuffer), } } @@ -362,6 +370,8 @@ func (m *WorkerManager) EnsureWorker( worker.pathRefusal = m.pathRefusal worker.layoutReporter = m.layoutReporter worker.remoteReporter = func(observed RemoteObservation) { m.recordRemoteObservation(key, observed) } + managerCtx := m.ctx // read under m.mu; the worker's loop calls the reporter later + worker.publicationReporter = func() { go m.notifyPublication(managerCtx, key) } worker.scanAcceptance = m.scanAcceptance worker.renderFidelityGate = m.renderFidelityGate worker.crOwners = m.commitRequests @@ -592,6 +602,7 @@ func (m *WorkerManager) Start(ctx context.Context) error { m.ctx = ctx m.mu.Unlock() telemetry.SetGaugeSource(telemetry.GaugeGitQueueDepth, m.queueDepthSamples) + m.setPublicationGaugeSources() m.Log.Info("WorkerManager started") m.sweepPeriodically(ctx) @@ -658,13 +669,7 @@ func (m *WorkerManager) sweepPeriodically(ctx context.Context) { // source that waited on the lock a wedged worker holds across its slow work would reintroduce the // staleness the observable gauge exists to remove. func (m *WorkerManager) queueDepthSamples() []telemetry.GaugeSample { - m.mu.RLock() - workers := make([]*BranchWorker, 0, len(m.workers)) - for _, worker := range m.workers { - workers = append(workers, worker) - } - m.mu.RUnlock() - + workers := m.liveWorkers() samples := make([]telemetry.GaugeSample, 0, len(workers)) for _, worker := range workers { samples = append(samples, telemetry.GaugeSample{ @@ -675,6 +680,18 @@ func (m *WorkerManager) queueDepthSamples() []telemetry.GaugeSample { return samples } +// liveWorkers copies the worker pointers out under m.mu, for a gauge source to read each worker's +// atomics without holding the lock. +func (m *WorkerManager) liveWorkers() []*BranchWorker { + m.mu.RLock() + defer m.mu.RUnlock() + workers := make([]*BranchWorker, 0, len(m.workers)) + for _, worker := range m.workers { + workers = append(workers, worker) + } + return workers +} + // NeedLeaderElection ensures only the elected leader manages workers. // This prevents multiple pods from managing the same workers. func (m *WorkerManager) NeedLeaderElection() bool { diff --git a/internal/git/write_gate.go b/internal/git/write_gate.go index 8406abea..bda05261 100644 --- a/internal/git/write_gate.go +++ b/internal/git/write_gate.go @@ -22,14 +22,15 @@ import ( // runs so the target's scan stays fresh. // // A WindowMismatch is not asked: that request was refused by another author's window, which is -// what it reports. Neither is a request whose commit already exists locally. The gates are read -// when the commit is made, as ResolvedTargetMetadata.Suspend already defined for suspension, and a -// commit made before a gate closed is still pushed: reading them at push time would strand it in -// the checkout, to resurface out of order when the gate opens. +// what it reports. Neither is a request whose write is already decided. The gates are read when the +// write is decided (the window closes), which is normally when it is committed, as +// ResolvedTargetMetadata.Suspend already defined for suspension. A write decided before a gate +// closed is still committed and pushed, even when an unreachable remote delays its commit: reading +// the gates later would strand it in the worker, to resurface out of order when the gate opens. // -// They are deliberately not the checkout's three flags (baseTrusted, worktreeDirty, -// replayRequired). Those say whether the worker can commit on its checkout; these say whether the -// target may be written at all, and no reset or replay changes the answer. +// They are deliberately not the checkout's state (baseTrusted, checkoutApplied). That says whether +// the worker can commit on its checkout; these say whether the target may be written at all, and +// no reset or replay changes the answer. var ( // errTargetSuspended is a request's failure on a suspended GitTarget, which writes nothing. diff --git a/internal/git/write_gate_test.go b/internal/git/write_gate_test.go index 19b00633..53259c07 100644 --- a/internal/git/write_gate_test.go +++ b/internal/git/write_gate_test.go @@ -63,7 +63,7 @@ func expire(loop *branchWorkerEventLoop) { pcr.attachDeadline = past } } - loop.serviceCommitRequests() + loop.endWake(0) } // remoteHead is the remote's main. @@ -188,6 +188,7 @@ func forceDueNamed(loop *branchWorkerEventLoop, req *AttachCommitRequest) { if loop.openWindow != nil && loop.openWindow.pendingCR != nil && *loop.openWindow.pendingCR == id { loop.openWindow.timers.maxAt = time.Now().Add(-time.Millisecond) loop.closeOrArmWindow() + loop.endWake(0) } } diff --git a/internal/reconcile/git_target_event_stream.go b/internal/reconcile/git_target_event_stream.go index 956fcde6..e85db66c 100644 --- a/internal/reconcile/git_target_event_stream.go +++ b/internal/reconcile/git_target_event_stream.go @@ -52,7 +52,7 @@ func NewGitTargetEventStream( // OnWatchEvent forwards a live event to the GitTarget's branch worker. An event with no object // payload that is neither a DELETE nor a field patch carries nothing to write and is dropped // (returns nil — there is nothing to durably hand off). It returns a non-nil error only when the -// worker's queue is full and a real event was dropped, so the watch loop does not advance its +// worker refused a real event (its queue was full, or its intake paused), so the watch loop does not advance its // durable cursor past an event the worker never accepted: the watch reconnects from the // un-advanced cursor and redelivers, which is safe because the writer's no-op detection at the // commit boundary makes redelivery idempotent. @@ -66,7 +66,8 @@ func (s *GitTargetEventStream) OnWatchEvent(event git.Event) error { event.GitTargetName = s.gitTargetName event.GitTargetNamespace = s.gitTargetNamespace if !s.branchWorker.Enqueue(event) { - return fmt.Errorf("branch worker queue full for gitTarget %s/%s; dropped %s event for %s", + return fmt.Errorf("branch worker for gitTarget %s/%s refused %s event for %s "+ + "(queue full or intake paused); the watch delivers it again", s.gitTargetNamespace, s.gitTargetName, event.Operation, event.Identifier.Key()) } s.logger.V(1).Info("Forwarded event", "resource", event.Identifier.Key(), "operation", event.Operation) diff --git a/internal/telemetry/exporter.go b/internal/telemetry/exporter.go index 8499f036..3b502a1c 100644 --- a/internal/telemetry/exporter.go +++ b/internal/telemetry/exporter.go @@ -158,14 +158,18 @@ var ( // nothing observed is dropped — but a non-zero rate is an API server // saying something about this watch that it does not say routinely. // - // LOSS (an observed change that did not reach Git): - // route_failed — the writer refused it. Nothing retries until the next resync, so - // the mirror is behind for that object. - // - // route_failed OVERLAPS git_queue_drops_total: a full worker queue is one of the ways a route - // fails, so one dropped event can increment both. They are two views of one event — where it + // RECOVERABLE (refused, and delivered again): + // route_failed — the writer refused it: its queue was full, or its intake paused + // through an outage. The session ends without advancing the cursor + // and the reconnect delivers the event again. Only a cursor that + // expires first turns it into a fresh snapshot, which restores the + // object's content but not this change's own commit, and under + // prune.mode OnEvent does not remove a deleted object's file. + // + // route_failed OVERLAPS git_queue_refusals_total: a refused enqueue is one of the ways a route + // fails, so one refused event can increment both. They are two views of one event — where it // was refused, and by what — so a panel may show either, and a sum over both counts that event - // twice. Neither is a unique-loss total. + // twice. Neither is a unique-refusal total. // // It carries the GitTarget because "which tenant stopped receiving events" is the question, and // deliberately NOT the watch event type (added/modified/deleted): that halves the series @@ -232,7 +236,7 @@ var ( // // This was the largest remaining hole. A commit failure drops the whole window — the events are // already lost to the failed flush — and it happens AFTER routing and BEFORE pushing, so - // neither GitQueueDropsTotal nor GitPushesTotal sees it. The mirror silently falls behind for + // neither GitQueueRefusalsTotal nor GitPushesTotal sees it. The mirror silently falls behind for // every object in that window until the next resync re-derives them, and until now the only // trace was a log line (or, for a refusal, a GitTarget condition nobody is alerting on). GitCommitFailuresTotal metric.Int64Counter @@ -293,15 +297,25 @@ var ( // close_reason and timer_source. The reason and source stay on the counter: a histogram's label // set costs bucket count + 2 series. GitCommitWindowDurationSeconds metric.Float64Histogram - // GitQueueDropsTotal counts work the branch worker threw away because its queue was full, - // labelled by {provider_namespace, provider_name, branch, kind} where kind is `write`, - // `attach` or `resync`. Every increment is lost work: a write is recovered only by the next - // resync, and until then the mirror is behind for that object with no other trace. - // - // The queue-depth gauge said the queue was deep. Nothing said anything had been dropped, which - // is the one thing an operator needs to know, and a saturating queue is exactly when it - // happens. - GitQueueDropsTotal metric.Int64Counter + // GitQueueRefusalsTotal counts work the branch worker refused at enqueue, labelled by + // {provider_namespace, provider_name, branch, kind} where kind is `write`, `attach`, `resync` + // or `refresh`. Its queue was full, its intake was paused through an outage, or it was + // stopping. A refused item is not lost: its producer keeps it and offers it again (the watch + // keeps its cursor and delivers the event again, the controller re-sends a save, a resync is + // gathered again, the next reconcile asks for a refresh). What a refusal can cost is history: a + // watch cursor that expires before the event comes back turns it into a fresh snapshot, which + // restores the object's content but not that change's own commit, and under prune.mode OnEvent + // does not remove a deleted object's file. + // + // The queue-depth gauge said the queue was deep. Nothing said anything had been refused, and a + // saturating queue is exactly when it happens. + GitQueueRefusalsTotal metric.Int64Counter + // GitMaterializationFailuresTotal counts attempts to commit decided writes that stopped because + // the remote could not be reached, or the parent branch a new write branch is created from is + // missing, labelled by {provider_namespace, provider_name, branch, reason} where reason is + // `unreachable` or `parent_unavailable`. Nothing is lost: the writes stay pending for the + // retry. It is the failure before a push cycle starts, which git_pushes_total cannot see. + GitMaterializationFailuresTotal metric.Int64Counter // GitResyncFailuresTotal counts rule-change resyncs whose apply failed or // timed out at the worker AFTER being enqueued. Delivery is marked on enqueue (the @@ -552,7 +566,8 @@ func registerCounters() error { {"gitopsreverser_git_pushes_total", &GitPushesTotal}, {"gitopsreverser_git_push_retries_total", &GitPushRetriesTotal}, {"gitopsreverser_git_fetches_total", &GitFetchesTotal}, - {"gitopsreverser_git_queue_drops_total", &GitQueueDropsTotal}, + {"gitopsreverser_git_queue_refusals_total", &GitQueueRefusalsTotal}, + {"gitopsreverser_git_materialization_failures_total", &GitMaterializationFailuresTotal}, {"gitopsreverser_placements_total", &PlacementsTotal}, {"gitopsreverser_placement_refusals_total", &PlacementRefusalsTotal}, { @@ -698,6 +713,11 @@ func registerObservableGauges() error { source string }{ {"gitopsreverser_git_queue_depth", GaugeGitQueueDepth}, + {"gitopsreverser_git_retained_bytes", GaugeGitRetainedBytes}, + {"gitopsreverser_git_retained_writes", GaugeGitRetainedWrites}, + {"gitopsreverser_git_intake_paused", GaugeGitIntakePaused}, + {"gitopsreverser_git_oldest_retained_write_timestamp_seconds", GaugeGitOldestRetainedWrite}, + {"gitopsreverser_git_next_retry_timestamp_seconds", GaugeGitNextRetry}, {"gitopsreverser_watch_types", GaugeWatchTypes}, {"gitopsreverser_watch_streams_open", GaugeWatchStreamsOpen}, {"gitopsreverser_watch_plan_dirty_targets", GaugeWatchPlanDirtyTargets}, diff --git a/internal/telemetry/gauges.go b/internal/telemetry/gauges.go index 747f72b3..34987c19 100644 --- a/internal/telemetry/gauges.go +++ b/internal/telemetry/gauges.go @@ -36,7 +36,16 @@ type GaugeSource func() []GaugeSample // Names for the observable gauges a producer can install a source for. They are the instrument // names with the prefix stripped, so a reader grepping the exported metric finds the source too. const ( - GaugeGitQueueDepth = "git_queue_depth" + GaugeGitQueueDepth = "git_queue_depth" + // GaugeGitRetainedBytes and the four after it are the branch worker's publication backlog, read + // from what its loop last published: the bytes and writes it keeps until a push lands, whether + // intake is paused, when the oldest kept write was decided, and when the next retry is due. The + // two timestamps have no sample while there is nothing to date. + GaugeGitRetainedBytes = "git_retained_bytes" + GaugeGitRetainedWrites = "git_retained_writes" + GaugeGitIntakePaused = "git_intake_paused" + GaugeGitOldestRetainedWrite = "git_oldest_retained_write_timestamp_seconds" + GaugeGitNextRetry = "git_next_retry_timestamp_seconds" GaugeWatchTypes = "watch_types" GaugeWatchStreamsOpen = "watch_streams_open" GaugeWatchPlanDirtyTargets = "watch_plan_dirty_targets" diff --git a/internal/types/collection.go b/internal/types/collection.go index bd556033..3965d631 100644 --- a/internal/types/collection.go +++ b/internal/types/collection.go @@ -18,8 +18,8 @@ import ( // snapshot owns every managed document of its type in its namespace scope, including documents // the selector never matched, so a selector narrows what the mirror holds rather than which // documents a sweep may reach. Two selectors over one boundary are two collections, and the rule -// compiler refuses a second selector over an overlapping scope so that two snapshots never sweep -// each other's documents. +// resolver refuses a second collection over an overlapping scope of one GitTarget, so two +// snapshots never sweep each other's documents and no object reaches the target on two streams. // // # The version is deliberately absent // @@ -42,9 +42,9 @@ type CollectionKey struct { // Resource is the plural resource name, e.g. "configmaps". Resource string // Namespace restricts the collection to one namespace. Empty is a genuinely cluster-wide - // (all-namespaces) collection, which is a PEER of any named namespace on the same type and never - // a replacement for it: collapsing the two widened the named rule's stream to every namespace - // its credential could read. + // (all-namespaces) collection, never a replacement for a named namespace on the same type: + // collapsing the two widened the named rule's stream to every namespace its credential could + // read. The two overlap, so one GitTarget cannot select both. Namespace string // LabelSelector is the canonical label selector (see CanonicalLabelSelector) the API server // selects the collection with. Empty selects every object. diff --git a/internal/watch/selector_conflict.go b/internal/watch/collection_overlap.go similarity index 64% rename from internal/watch/selector_conflict.go rename to internal/watch/collection_overlap.go index 4336a6e8..4eb56988 100644 --- a/internal/watch/selector_conflict.go +++ b/internal/watch/collection_overlap.go @@ -12,10 +12,9 @@ import ( "github.com/ConfigButler/gitops-reverser/internal/types" ) -// ReasonObjectSelectorConflict is the ResourcesResolved reason for a rule refused because one of -// its collections overlaps a collection the same GitTarget already selects with a different -// object selector. -const ReasonObjectSelectorConflict = "ObjectSelectorConflict" +// ReasonCollectionOverlap is the ResourcesResolved reason for a rule refused because one of its +// collections overlaps a different collection the same GitTarget already selects. +const ReasonCollectionOverlap = "CollectionOverlap" // Rule kinds, as a selectingRule names them in a refusal message. const ( @@ -70,20 +69,24 @@ func describeSelection(c types.CollectionKey) string { return fmt.Sprintf("%s with objectSelector %q", name, c.LabelSelector) } -// refuseSelectorConflicts drops the selections of every rule whose collections overlap a -// collection another rule of the same GitTarget selects with a different object selector, and -// returns the refused rules with the reason for each. +// refuseCollectionOverlaps drops the selections of every rule whose collections overlap a +// different collection of the same GitTarget, and returns the refused rules with the reason for +// each. // -// Overlapping collections must share one selector. Two selections of one object would be two -// watches that disagree about whether it is in the mirror: one would delete a document the other -// still selects, and their snapshots would prune each other's documents on every replay. -// Different selectors in disjoint namespaces are fine. +// One GitTarget watches each object through at most one stream. Two streams over one object would +// deliver it twice, in no defined order: a delayed copy can restore content the other stream has +// already moved past, or split another author's commit window, and their snapshots would prune +// each other's documents. So overlapping collections are refused structurally — same type, and the +// same namespace or one of them all namespaces — whatever their object selectors, because a +// snapshot owns its whole structural scope. An exact duplicate is one collection, so one stream, +// and is not an overlap. Disjoint namespaces never overlap, and another GitTarget has its own +// streams. // // Rules are admitted oldest first, so the rule that got there first keeps its collections and a -// newer rule cannot take a running mirror away by asking for a different selector. A rule whose -// own items conflict with each other is refused whatever its age. A refusal is whole-rule, as a -// denied source namespace is: mirroring part of what a rule asked for is worse than a loud failure. -func refuseSelectorConflicts(selections []watchSelection) ([]watchSelection, map[selectingRule]string) { +// newer rule cannot take a running mirror away. A rule whose own items overlap each other is +// refused whatever its age. A refusal is whole-rule, as a denied source namespace is: mirroring +// part of what a rule asked for is worse than a loud failure. +func refuseCollectionOverlaps(selections []watchSelection) ([]watchSelection, map[selectingRule]string) { byRule := map[selectingRule][]watchSelection{} var rules []selectingRule for _, sel := range selections { @@ -99,7 +102,7 @@ func refuseSelectorConflicts(selections []watchSelection) ([]watchSelection, map var refused map[selectingRule]string for _, rule := range rules { own := collectionsOf(byRule[rule]) - if reason := selectorConflict(own, held); reason != "" { + if reason := collectionOverlap(own, held); reason != "" { if refused == nil { refused = map[selectingRule]string{} } @@ -120,20 +123,26 @@ type heldCollection struct { rule selectingRule } -// selectorConflict returns why a rule's collections cannot be admitted beside the ones already +// overlapsDistinct reports whether two collections can deliver one object on two streams. +func overlapsDistinct(a, b types.CollectionKey) bool { + return a != b && a.Overlaps(b) +} + +// collectionOverlap returns why a rule's collections cannot be admitted beside the ones already // held, or "" when they can. -func selectorConflict(own []types.CollectionKey, held []heldCollection) string { +func collectionOverlap(own []types.CollectionKey, held []heldCollection) string { for i, c := range own { for _, other := range own[i+1:] { - if c.Overlaps(other) && c.LabelSelector != other.LabelSelector { - return fmt.Sprintf("its items select overlapping collections differently: %s, and %s; "+ - "overlapping collections must use one objectSelector", describeSelection(c), describeSelection(other)) + if overlapsDistinct(c, other) { + return fmt.Sprintf("its items select overlapping collections: %s, and %s; "+ + "a GitTarget watches each object through one collection", + describeSelection(c), describeSelection(other)) } } for _, h := range held { - if c.Overlaps(h.collection) && c.LabelSelector != h.collection.LabelSelector { + if overlapsDistinct(c, h.collection) { return fmt.Sprintf("it selects %s, which overlaps %s already selected by the older %s; "+ - "overlapping collections on one GitTarget must use one objectSelector", + "a GitTarget watches each object through one collection", describeSelection(c), describeSelection(h.collection), h.rule) } } diff --git a/internal/watch/collection_overlap_test.go b/internal/watch/collection_overlap_test.go new file mode 100644 index 00000000..da3e871d --- /dev/null +++ b/internal/watch/collection_overlap_test.go @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +func TestRefuseCollectionOverlaps(t *testing.T) { + rec := typesetRecordForTest(configmapsGVR) + older := selectingRule{ + kind: ruleKindWatchRule, + namespace: "ns", + name: "b", + createdAt: metav1.NewTime(time.Unix(1, 0)), + } + newer := selectingRule{ + kind: ruleKindWatchRule, + namespace: "ns", + name: "a", + createdAt: metav1.NewTime(time.Unix(2, 0)), + } + sameAgeLowerName := selectingRule{kind: ruleKindWatchRule, namespace: "ns", name: "a", createdAt: older.createdAt} + + t.Run("the older rule wins regardless of order", func(t *testing.T) { + admitted, refused := refuseCollectionOverlaps([]watchSelection{ + {record: rec, namespace: "x", labelSelector: "b=1", rule: newer}, + {record: rec, namespace: "x", labelSelector: "a=1", rule: older}, + }) + require.Len(t, admitted, 1) + assert.Equal(t, older, admitted[0].rule) + assert.Contains(t, refused, newer) + }) + t.Run("equal age falls back to the lower name", func(t *testing.T) { + _, refused := refuseCollectionOverlaps([]watchSelection{ + {record: rec, namespace: "x", labelSelector: "b=1", rule: older}, + {record: rec, namespace: "x", labelSelector: "a=1", rule: sameAgeLowerName}, + }) + assert.Contains(t, refused, older) + assert.NotContains(t, refused, sameAgeLowerName) + }) + t.Run("a rule overlapping itself is refused whatever its age", func(t *testing.T) { + for _, selector := range []string{"", "a=1"} { + admitted, refused := refuseCollectionOverlaps([]watchSelection{ + {record: rec, namespace: "", labelSelector: selector, rule: older}, + {record: rec, namespace: "x", labelSelector: selector, rule: older}, + }) + assert.Empty(t, admitted) + assert.Contains(t, refused[older], "its items select overlapping collections") + assert.Contains(t, refused[older], "configmaps in all namespaces") + assert.Contains(t, refused[older], "configmaps in x") + } + }) + // The structural rule: a selector never makes two overlapping scopes disjoint, because a + // snapshot owns its whole scope and a later relabel can move an object between them. + for name, selectors := range map[string][2]string{ + "equal selectors": {"a=1", "a=1"}, + "no selectors": {"", ""}, + "different selectors": {"a=1", "b=1"}, + } { + t.Run("all namespaces and a named namespace overlap with "+name, func(t *testing.T) { + admitted, refused := refuseCollectionOverlaps([]watchSelection{ + {record: rec, namespace: "", labelSelector: selectors[0], rule: older}, + {record: rec, namespace: "x", labelSelector: selectors[1], rule: newer}, + }) + require.Len(t, admitted, 1) + assert.Equal(t, older, admitted[0].rule) + assert.Contains(t, refused[newer], "configmaps in x") + assert.Contains(t, refused[newer], "configmaps in all namespaces") + assert.Contains(t, refused[newer], "the older WatchRule ns/b") + }) + } + t.Run("an exact duplicate is one collection", func(t *testing.T) { + admitted, refused := refuseCollectionOverlaps([]watchSelection{ + {record: rec, namespace: "x", labelSelector: "a=1", rule: older}, + {record: rec, namespace: "x", labelSelector: "a=1", rule: newer}, + }) + assert.Len(t, admitted, 2) + assert.Empty(t, refused) + }) + t.Run("disjoint namespaces may use different selectors", func(t *testing.T) { + admitted, refused := refuseCollectionOverlaps([]watchSelection{ + {record: rec, namespace: "x", labelSelector: "a=1", rule: older}, + {record: rec, namespace: "y", labelSelector: "b=1", rule: newer}, + }) + assert.Len(t, admitted, 2) + assert.Empty(t, refused) + }) + t.Run("other types never overlap", func(t *testing.T) { + admitted, refused := refuseCollectionOverlaps([]watchSelection{ + {record: rec, namespace: "", labelSelector: "a=1", rule: older}, + {record: typesetRecordForTest(resolverGVR), namespace: "x", labelSelector: "", rule: newer}, + }) + assert.Len(t, admitted, 2) + assert.Empty(t, refused) + }) +} + +// Equivalent selectors written differently canonicalize to one collection, so two rules asking for +// it are an exact duplicate: neither is refused, and the target streams it once. +func TestResolveWatchedTypeTables_ExactDuplicatesShareOneStream(t *testing.T) { + manager, store := makeWatchedTypeManager(t) + now := time.Unix(1000, 0) + addSelectorRule(store, selectorRule("labels", "team-a", now, labels("team", "a"), ""), "team-a") + addSelectorRule(store, selectorRule("expression", "team-a", now.Add(time.Minute), &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "team", Operator: metav1.LabelSelectorOpIn, Values: []string{"a", "a"}}, + }, + }, ""), "team-a") + + manager.refreshWatchedTypeTables() + table, _ := manager.watchedTypeTableForGitDest(gitDestRef("sel-target")) + for _, name := range []string{"labels", "expression"} { + refused, message := manager.CollectionOverlapForWatchRule(selectorRule(name, "team-a", now, nil, "")) + assert.False(t, refused, message) + } + assert.Equal(t, []targetWatchKey{{GVR: configmapsGVR, Namespace: "team-a", LabelSelector: teamA}}, + targetWatchKeys(table), "one collection, one stream") +} + +// The overlap is per GitTarget. Another target's streams are its own, so the same all-namespaces +// and named-namespace pair is admitted when each half belongs to a different target. +func TestResolveWatchedTypeTables_SeparateGitTargetsMayOverlap(t *testing.T) { + manager, store := makeWatchedTypeManager(t) + now := time.Unix(1000, 0) + wide := selectorRule("wide", "test-ns", now, nil, configv1alpha3.SourceNamespaceWildcard) + wide.Spec.GitTargetRef.Name = "wide-target" + store.AddOrUpdateWatchRule(wide, [][]string{{""}}, + "wide-target", "test-ns", "test-provider", "test-ns", "main", "wide") + addSelectorRule(store, selectorRule("named", "team-a", now.Add(time.Minute), nil, ""), "team-a") + + manager.refreshWatchedTypeTables() + wideTable, _ := manager.watchedTypeTableForGitDest(gitDestRef("wide-target")) + namedTable, _ := manager.watchedTypeTableForGitDest(gitDestRef("sel-target")) + assert.Equal(t, []string{""}, wideTable.Types[0].WatchScopes()) + assert.Equal(t, []string{"team-a"}, namedTable.Types[0].WatchScopes()) + for _, rule := range []configv1alpha3.WatchRule{wide, selectorRule("named", "team-a", now, nil, "")} { + refused, message := manager.CollectionOverlapForWatchRule(rule) + assert.False(t, refused, message) + } +} + +// A refusal follows the configuration: removing the older rule admits the newer one, and the +// running plan swaps the collections with the old stream retired before the new one starts, so the +// target never has two producers over one object. +func TestCollectionOverlap_RemovingTheOlderRuleAdmitsTheNewer(t *testing.T) { + manager, store := makeWatchedTypeManager(t) + older := time.Unix(1000, 0) + named := selectorRule("named", "team-a", older, nil, "") + wide := selectorRule("wide", "test-ns", older.Add(time.Minute), nil, configv1alpha3.SourceNamespaceWildcard) + addSelectorRule(store, named, "team-a") + addSelectorRule(store, wide, "") + + manager.refreshWatchedTypeTables() + refused, message := manager.CollectionOverlapForWatchRule(wide) + require.True(t, refused) + assert.Contains(t, message, "WatchRule test-ns/wide is refused") + assert.Contains(t, message, "configmaps in all namespaces with no objectSelector") + assert.Contains(t, message, "configmaps in team-a with no objectSelector already selected by the older "+ + "WatchRule team-a/named") + before, _ := manager.watchedTypeTableForGitDest(gitDestRef("sel-target")) + + store.Delete(ruleKey(named)) + manager.refreshWatchedTypeTables() + refused, _ = manager.CollectionOverlapForWatchRule(wide) + assert.False(t, refused, "the refusal clears once the overlapping rule is gone") + after, _ := manager.watchedTypeTableForGitDest(gitDestRef("sel-target")) + assert.Equal(t, []string{""}, after.Types[0].WatchScopes()) + + gitDest := gitDestRef("sel-target") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + running, opened := planTestManager(t, gitDest) + require.NoError(t, running.replaceGitTargetWatches(ctx, before)) + first := receiveOpenedWatch(t, opened) + assertNoOpenedWatch(t, opened) // the refused all-namespaces subscription never started + retiring := running.targetWatches[gitDest.Key()].streams[types.CollectionKeyFor(configmapsGVR, "team-a")] + require.NotNil(t, retiring) + + require.NoError(t, running.replaceGitTargetWatches(ctx, after)) + retiring.gate.mu.Lock() + retired := retiring.gate.retired + retiring.gate.mu.Unlock() + assert.True(t, retired, "the named stream is retired before the all-namespaces stream is launched") + second := receiveOpenedWatch(t, opened) + assert.Empty(t, second.namespace) + assertStopped(t, first) + assert.Len(t, running.targetWatches[gitDest.Key()].streams, 1) +} + +// A catalog change can create an overlap no rule edit made: a type the older rule names starts +// being served, and the newer rule's collection for it overlaps. Resolution re-runs on the registry +// change and refuses the newer rule then. +func TestCollectionOverlap_ACatalogChangeCanCreateOne(t *testing.T) { + manager, store := makeWatchedTypeManager(t) + withoutWidgets := newCommonTestDiscovery() + withWidgets := newCommonTestDiscovery() + listWatch := metav1.Verbs{"get", "list", "watch"} + withWidgets.groups = append(withWidgets.groups, testAPIGroup("example.com", "v1")) + withWidgets.resources = append(withWidgets.resources, &metav1.APIResourceList{ + GroupVersion: "example.com/v1", + APIResources: []metav1.APIResource{{Name: "widgets", Kind: "Widget", Namespaced: true, Verbs: listWatch}}, + }) + _, err := manager.resourceCatalog.Refresh(withoutWidgets) + require.NoError(t, err) + + older := time.Unix(1000, 0) + wide := selectorRule("wide", "test-ns", older, nil, configv1alpha3.SourceNamespaceWildcard) + wide.Spec.Rules[0].APIGroups = []string{"example.com"} + wide.Spec.Rules[0].APIVersions = nil + wide.Spec.Rules[0].Resources = []string{"widgets"} + named := selectorRule("named", "team-a", older.Add(time.Minute), nil, "") + named.Spec.Rules[0].APIGroups = []string{"", "example.com"} + named.Spec.Rules[0].APIVersions = nil + named.Spec.Rules[0].Resources = []string{"configmaps", "widgets"} + addSelectorRule(store, wide, "") + addSelectorRule(store, named, "team-a") + + manager.refreshWatchedTypeTables() + refused, message := manager.CollectionOverlapForWatchRule(named) + require.False(t, refused, message) + + _, err = manager.resourceCatalog.Refresh(withWidgets) + require.NoError(t, err) + manager.refreshClusterTypeRegistry(manager.cluster(configPlaneClusterID)) + manager.refreshWatchedTypeTables() + + refused, message = manager.CollectionOverlapForWatchRule(named) + assert.True(t, refused) + assert.Contains(t, message, "widgets.example.com in team-a") + assert.Contains(t, message, "widgets.example.com in all namespaces") + table, _ := manager.watchedTypeTableForGitDest(gitDestRef("sel-target")) + keys := targetWatchKeys(table) + require.Len(t, keys, 1, "the refused rule's configmaps stream goes with it: a refusal is whole-rule") + assert.Equal(t, "widgets", keys[0].GVR.Resource) + assert.Empty(t, keys[0].Namespace) +} diff --git a/internal/watch/desired_state_change_filter.go b/internal/watch/desired_state_change_filter.go new file mode 100644 index 00000000..66609773 --- /dev/null +++ b/internal/watch/desired_state_change_filter.go @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "crypto/sha256" + "encoding/json" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// desiredStateChangeFilter is one stream's memory of the desired state the branch worker last +// accepted for each object it delivers, keyed by UID. It passes CREATE, DELETE, and every UPDATE +// that changes Git-visible content, retained labels and annotations included, and drops an UPDATE +// whose sanitized content equals that baseline: the classic /status-only update. +// +// Dropping it is not about Git content, since a routed no-op finds no diff. A /status update has no +// audit author (the policy drops /status writes), so routing one would close a named author's open +// commit window on the author change and split a save in two. It is also most of the churn. +// +// The stream owns the map, and only the stream goroutine reads or writes it. It survives the +// stream's reconnects and goes with the stream. Two things write it, both only once the worker +// took what they describe: an accepted live event records its own object (accept), and an accepted +// replay snapshot replaces the whole map (adopt), so a baseline older than the replay never +// outlives it. One stream per object, which collection-overlap refusal guarantees per GitTarget, is +// what makes a per-stream baseline the worker's. +// +// A missing baseline and content that cannot be hashed both pass: never drop a change that cannot +// be proven a no-op. +type desiredStateChangeFilter struct { + baselines map[k8stypes.UID]string +} + +func newDesiredStateChangeFilter() *desiredStateChangeFilter { + return &desiredStateChangeFilter{baselines: map[k8stypes.UID]string{}} +} + +// desiredStateChange is one live event's verdict, taken before it is routed and recorded only +// once the worker accepted it. +type desiredStateChange struct { + uid k8stypes.UID + remove bool + hash string + hashed bool +} + +// check reports whether a live event changes no desired state the worker accepted. It reads the map +// and never writes it: recording before the worker took the event let a refused UPDATE, redelivered +// from the unchanged cursor, match its own hash and be skipped. +// +// A nil filter passes everything. +func (f *desiredStateChangeFilter) check( + u *unstructured.Unstructured, event *git.Event, op string, +) (desiredStateChange, bool) { + change := desiredStateChange{uid: u.GetUID()} + if op == string(types.OperationDelete) { + change.remove = true + return change, false + } + change.hash, change.hashed = desiredStateHash(event.Object) + if f == nil || !change.hashed || change.uid == "" { + return change, false + } + baseline, ok := f.baselines[change.uid] + return change, op == string(types.OperationUpdate) && ok && baseline == change.hash +} + +// accept records what the worker just accepted: a DELETE clears the object's baseline, so a +// recreate under the same UID can never match its predecessor, and anything else stores its hash. +func (f *desiredStateChangeFilter) accept(change desiredStateChange) { + if f == nil || change.uid == "" { + return + } + if change.remove { + delete(f.baselines, change.uid) + return + } + if change.hashed { + f.baselines[change.uid] = change.hash + } +} + +// adopt replaces every baseline with an accepted snapshot's. An object the snapshot does not hold +// loses its baseline with it. +func (f *desiredStateChangeFilter) adopt(baselines map[k8stypes.UID]string) { + if f == nil { + return + } + if baselines == nil { + baselines = map[k8stypes.UID]string{} + } + f.baselines = baselines +} + +// desiredStateHash hashes a sanitized object, the content Git holds for it. ok=false means it cannot +// be hashed (nil, or a marshal error). +func desiredStateHash(obj *unstructured.Unstructured) (string, bool) { + if obj == nil { + return "", false + } + raw, err := json.Marshal(obj) + if err != nil { + return "", false + } + sum := sha256.Sum256(raw) + return string(sum[:]), true +} + +// replaySnapshot is one session's snapshot as it is gathered: the desired set the resync carries, +// and each object's desired-state hash by UID, for the stream's filter to adopt once the worker +// accepted the resync. A snapshot that is refused, or a session that ends before it is complete, +// is discarded with both. +type replaySnapshot struct { + desired []manifestanalyzer.DesiredResource + baselines map[k8stypes.UID]string +} + +// add folds one observed object into the snapshot. An object desiredFromObject excludes (a +// Terminating one) gets no baseline either: Git does not hold it. +func (r *replaySnapshot) add(gvr schema.GroupVersionResource, u *unstructured.Unstructured) { + desired, ok := desiredFromObject(gvr, u) + if !ok { + return + } + r.desired = append(r.desired, desired) + if r.baselines == nil { + r.baselines = map[k8stypes.UID]string{} + } + if hash, hashed := desiredStateHash(desired.Object); hashed && u.GetUID() != "" { + r.baselines[u.GetUID()] = hash + } +} + +// snapshotFromList folds a LIST fallback's items exactly as a replay folds its events. +func snapshotFromList(gvr schema.GroupVersionResource, list *unstructured.UnstructuredList) replaySnapshot { + var snapshot replaySnapshot + if list == nil { + return snapshot + } + for i := range list.Items { + snapshot.add(gvr, &list.Items[i]) + } + return snapshot +} diff --git a/internal/watch/desired_state_change_filter_test.go b/internal/watch/desired_state_change_filter_test.go new file mode 100644 index 00000000..b62c7903 --- /dev/null +++ b/internal/watch/desired_state_change_filter_test.go @@ -0,0 +1,533 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/reconcile" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// filterObject is a ConfigMap "" in apps whose desired state is its data value. +func filterObject(uid, rv, value string) *unstructured.Unstructured { + obj := configMapObject(rv) + obj.SetName(uid) + obj.SetUID(k8stypes.UID(uid)) + obj.Object["data"] = map[string]interface{}{"key": value} + return obj +} + +// verdict runs the filter's check on one object, as the live path does. +func verdict(f *desiredStateChangeFilter, u *unstructured.Unstructured, op string) (desiredStateChange, bool) { + opType := watch.Modified + switch op { + case string(types.OperationCreate): + opType = watch.Added + case string(types.OperationDelete): + opType = watch.Deleted + } + event := targetWatchGitEvent(configmapsGVR, u, operationForLiveTargetWatchEvent(opType, u)) + return f.check(u, &event, op) +} + +// offer checks one event and, when it routes, accepts it as a worker that took it would. +func offer(f *desiredStateChangeFilter, u *unstructured.Unstructured, op string) bool { + change, unchanged := verdict(f, u, op) + if !unchanged { + f.accept(change) + } + return !unchanged +} + +func TestDesiredStateChangeFilter_Contract(t *testing.T) { + create, update, del := string(types.OperationCreate), string(types.OperationUpdate), string(types.OperationDelete) + + t.Run("an UPDATE with no baseline passes", func(t *testing.T) { + assert.True(t, offer(newDesiredStateChangeFilter(), filterObject("a", "1", "X"), update)) + }) + t.Run("a nil filter passes everything", func(t *testing.T) { + var f *desiredStateChangeFilter + assert.True(t, offer(f, filterObject("a", "1", "X"), create)) + assert.True(t, offer(f, filterObject("a", "2", "X"), update)) + }) + t.Run("an object without a UID is never filtered", func(t *testing.T) { + f := newDesiredStateChangeFilter() + assert.True(t, offer(f, filterObject("", "1", "X"), create)) + assert.True(t, offer(f, filterObject("", "2", "X"), update)) + }) + t.Run("X, then Y, then X: both changes route", func(t *testing.T) { + f := newDesiredStateChangeFilter() + require.True(t, offer(f, filterObject("a", "1", "X"), create)) + assert.True(t, offer(f, filterObject("a", "2", "Y"), update)) + assert.True(t, offer(f, filterObject("a", "3", "X"), update), "a return to earlier content is a change") + assert.False(t, offer(f, filterObject("a", "4", "X"), update)) + }) + t.Run("a CREATE always routes, even with a matching baseline", func(t *testing.T) { + f := newDesiredStateChangeFilter() + require.True(t, offer(f, filterObject("a", "1", "X"), create)) + assert.True(t, offer(f, filterObject("a", "2", "X"), create)) + }) + t.Run("a refused event is not a baseline", func(t *testing.T) { + f := newDesiredStateChangeFilter() + require.True(t, offer(f, filterObject("a", "1", "X"), create)) + _, unchanged := verdict(f, filterObject("a", "2", "Y"), update) + require.False(t, unchanged) + // The worker refused Y: nothing is accepted. + assert.True(t, offer(f, filterObject("a", "2", "Y"), update), "Y redelivered after its refusal still routes") + }) + t.Run("an accepted DELETE leaves no baseline for its UID", func(t *testing.T) { + f := newDesiredStateChangeFilter() + require.True(t, offer(f, filterObject("a", "1", "X"), create)) + require.True(t, offer(f, filterObject("a", "2", "X"), del)) + assert.True(t, offer(f, filterObject("a", "3", "X"), update)) + }) + t.Run("a Terminating UPDATE is a removal and clears the baseline", func(t *testing.T) { + f := newDesiredStateChangeFilter() + require.True(t, offer(f, filterObject("a", "1", "X"), create)) + terminating := filterObject("a", "2", "X") + terminating.SetDeletionTimestamp(&metav1.Time{Time: time.Unix(0, 0)}) + change, unchanged := verdict(f, terminating, del) + require.False(t, unchanged) + f.accept(change) + assert.Empty(t, f.baselines) + }) + t.Run("an accepted snapshot replaces every baseline", func(t *testing.T) { + f := newDesiredStateChangeFilter() + require.True(t, offer(f, filterObject("a", "1", "X"), create)) + require.True(t, offer(f, filterObject("gone", "1", "X"), create)) + var snapshot replaySnapshot + snapshot.add(configmapsGVR, filterObject("a", "5", "Z")) + f.adopt(snapshot.baselines) + assert.True(t, offer(f, filterObject("a", "6", "X"), update), "the replay's Z replaced the stale X") + assert.True(t, offer(f, filterObject("gone", "6", "X"), update), "a UID the snapshot omits has no baseline") + }) + t.Run("an accepted empty snapshot clears every baseline", func(t *testing.T) { + f := newDesiredStateChangeFilter() + require.True(t, offer(f, filterObject("a", "1", "X"), create)) + f.adopt(snapshotFromList(configmapsGVR, &unstructured.UnstructuredList{}).baselines) + assert.True(t, offer(f, filterObject("a", "2", "X"), update)) + assert.False(t, offer(f, filterObject("a", "3", "X"), update), "and records again afterwards") + }) +} + +// Only Git-visible content is compared: data, spec, and the labels and annotations sanitize keeps. +// Status and the metadata sanitize strips change nothing Git holds. +func TestDesiredStateChangeFilter_ComparesWhatGitHolds(t *testing.T) { + update := string(types.OperationUpdate) + base := func() *unstructured.Unstructured { + obj := filterObject("a", "1", "X") + obj.SetLabels(map[string]string{"team": "a"}) + obj.SetAnnotations(map[string]string{"note": "kept"}) + return obj + } + changes := map[string]func(*unstructured.Unstructured){ + "data": func(o *unstructured.Unstructured) { o.Object["data"] = map[string]interface{}{"key": "Y"} }, + "spec": func(o *unstructured.Unstructured) { o.Object["spec"] = map[string]interface{}{"replicas": int64(2)} }, + "a retained label": func(o *unstructured.Unstructured) { o.SetLabels(map[string]string{"team": "b"}) }, + "a retained annotation": func(o *unstructured.Unstructured) { + o.SetAnnotations(map[string]string{"note": "edited"}) + }, + } + for name, mutate := range changes { + t.Run(name+" routes", func(t *testing.T) { + f := newDesiredStateChangeFilter() + require.True(t, offer(f, base(), string(types.OperationCreate))) + edited := base() + mutate(edited) + assert.True(t, offer(f, edited, update)) + }) + } + noOps := map[string]func(*unstructured.Unstructured){ + "status": func(o *unstructured.Unstructured) { + o.Object["status"] = map[string]interface{}{"phase": "Ready"} + }, + "resourceVersion and generation": func(o *unstructured.Unstructured) { + o.SetResourceVersion("99") + o.SetGeneration(7) + }, + "managedFields": func(o *unstructured.Unstructured) { + o.SetManagedFields([]metav1.ManagedFieldsEntry{{Manager: "kubectl", Operation: "Update"}}) + }, + "a stripped annotation": func(o *unstructured.Unstructured) { + o.SetAnnotations(map[string]string{ + "note": "kept", "kubectl.kubernetes.io/last-applied-configuration": "{}", + }) + }, + } + for name, mutate := range noOps { + t.Run(name+" is unchanged", func(t *testing.T) { + f := newDesiredStateChangeFilter() + require.True(t, offer(f, base(), string(types.OperationCreate))) + edited := base() + mutate(edited) + assert.False(t, offer(f, edited, update)) + }) + } +} + +// admittingResyncs is a branch worker's admission as a replay snapshot sees it: it takes resyncs +// until told to refuse them, answering each on its reply channel as the worker does. +type admittingResyncs struct { + mu sync.Mutex + refuse bool + requests []*git.ResyncRequest +} + +func (a *admittingResyncs) EnqueueResync(request *git.ResyncRequest) bool { + a.mu.Lock() + defer a.mu.Unlock() + if a.refuse { + request.Result <- git.ResyncResult{Err: git.ErrFinalizeQueueFull} + return false + } + a.requests = append(a.requests, request) + request.Result <- git.ResyncResult{} + return true +} + +func (a *admittingResyncs) setRefuse(refuse bool) { + a.mu.Lock() + defer a.mu.Unlock() + a.refuse = refuse +} + +// recordingAuthors names alice as the author of every write to the object "a", and no one else. +type recordingAuthors struct{} + +func (recordingAuthors) ResolveAuthor(_ context.Context, query AuthorQuery) (git.UserInfo, git.AttributionOutcome) { + if query.Name == "a" { + return git.UserInfo{Username: "alice"}, git.AttributionResolved + } + return git.UserInfo{}, git.AttributionUnresolved +} + +// replayMode is how a session gathers its snapshot. +type replayMode string + +const ( + initialEvents replayMode = "initial-events" + listFallback replayMode = "list-fallback" +) + +// filterHarness drives whole watch sessions of one GitTarget's stream against a worker that +// admits live events and replay snapshots until told to refuse them. +type filterHarness struct { + t *testing.T + manager *Manager + live *admittingEnqueuer + resyncs *admittingResyncs + gitDest types.ResourceReference + mode replayMode + opened chan *watch.FakeWatcher + listMu sync.Mutex + listNext *unstructured.UnstructuredList +} + +func newFilterHarness(t *testing.T, mode replayMode) *filterHarness { + t.Helper() + gitDest := types.NewResourceReference("target", "default") + h := &filterHarness{ + t: t, + live: &admittingEnqueuer{}, + resyncs: &admittingResyncs{}, + gitDest: gitDest, + mode: mode, + opened: make(chan *watch.FakeWatcher, 1), + } + h.manager = &Manager{ + Log: logr.Discard(), + EventRouter: &EventRouter{ + Log: logr.Discard(), + gitTargetStreams: map[string]*reconcile.GitTargetEventStream{ + gitDest.Key(): reconcile.NewGitTargetEventStream( + gitDest.Name, gitDest.Namespace, h.live, logr.Discard()), + }, + resyncWorker: func(context.Context, types.ResourceReference) (resyncEnqueuer, error) { + return h.resyncs, nil + }, + }, + targetWatchOpen: func( + _ context.Context, _ schema.GroupVersionResource, _ string, opts metav1.ListOptions, + ) (watch.Interface, error) { + if mode == listFallback && opts.SendInitialEvents != nil { + return nil, errors.New("sendInitialEvents: Forbidden: sendInitialEvents is forbidden") + } + // Buffered, so a session that ends early never strands a send. + fw := watch.NewFakeWithChanSize(64, false) + h.opened <- fw + return fw, nil + }, + targetWatchList: func( + context.Context, schema.GroupVersionResource, string, metav1.ListOptions, + ) (*unstructured.UnstructuredList, error) { + h.listMu.Lock() + defer h.listMu.Unlock() + return h.listNext, nil + }, + } + return h +} + +// session runs one session that replays objects pinned at rv, then delivers live events. complete +// false ends an initial-events replay before its bookmark. It returns the session's error. +func (h *filterHarness) session( + stream targetWatchStream, objects []*unstructured.Unstructured, rv string, complete bool, live ...watch.Event, +) error { + h.t.Helper() + var events []watch.Event + switch h.mode { + case initialEvents: + for _, obj := range objects { + events = append(events, watch.Event{Type: watch.Added, Object: obj}) + } + if complete { + bookmark := &unstructured.Unstructured{} + bookmark.SetResourceVersion(rv) + bookmark.SetAnnotations(map[string]string{metav1.InitialEventsAnnotationKey: "true"}) + events = append(events, watch.Event{Type: watch.Bookmark, Object: bookmark}) + events = append(events, live...) + } + case listFallback: + list := &unstructured.UnstructuredList{} + list.SetResourceVersion(rv) + for _, obj := range objects { + list.Items = append(list.Items, *obj) + } + h.listMu.Lock() + h.listNext = list + h.listMu.Unlock() + events = live + } + return h.run(stream, false, events) +} + +// resume runs one session that resumes from a stored cursor and delivers live events. +func (h *filterHarness) resume(stream targetWatchStream, live ...watch.Event) error { + h.t.Helper() + h.manager.WatchCursorStore = &fakeWatchCursorStore{rv: "1", ok: true} + h.manager.rememberGitTargetUID(h.gitDest.WithUID("uid-1")) + defer func() { h.manager.WatchCursorStore = nil }() + return h.run(stream, true, live) +} + +func (h *filterHarness) run(stream targetWatchStream, resume bool, events []watch.Event) error { + h.t.Helper() + done := make(chan error, 1) + go func() { + done <- h.manager.targetWatchReplayAndStream(context.Background(), logr.Discard(), h.gitDest, stream, resume) + }() + var fw *watch.FakeWatcher + select { + case fw = <-h.opened: + case err := <-done: + return err + case <-time.After(time.Second): + h.t.Fatal("expected the session to open a watch") + } + for _, ev := range events { + fw.Action(ev.Type, ev.Object) + } + fw.Stop() + select { + case err := <-done: + return err + case <-time.After(time.Second): + h.t.Fatal("expected the session to end") + return nil + } +} + +// accepted returns the live events the worker took, as "=" for each. +func (h *filterHarness) accepted() []string { + var out []string + for _, event := range h.live.accepted() { + value := "" + if event.Object != nil { + value, _, _ = unstructured.NestedString(event.Object.Object, "data", "key") + } + out = append(out, event.Identifier.Name+"="+value) + } + return out +} + +func modified(obj *unstructured.Unstructured) watch.Event { + return watch.Event{Type: watch.Modified, Object: obj} +} + +// statusOnly is obj with a status change and nothing Git holds. +func statusOnly(obj *unstructured.Unstructured) *unstructured.Unstructured { + obj.Object["status"] = map[string]interface{}{"observed": obj.GetResourceVersion()} + return obj +} + +func bothReplayModes(t *testing.T, test func(t *testing.T, h *filterHarness)) { + for _, mode := range []replayMode{initialEvents, listFallback} { + t.Run(string(mode), func(t *testing.T) { test(t, newFilterHarness(t, mode)) }) + } +} + +// The replay regression. Git holds X from a live event; the stream reconnects into a replay that +// finds Z, which the worker writes; the object then goes back to X. That return is a change to +// Git, and a baseline still at X would have dropped it and left Git at Z. +func TestDesiredStateChangeFilter_AnAcceptedReplayReplacesTheBaseline(t *testing.T) { + bothReplayModes(t, func(t *testing.T, h *filterHarness) { + stream := testStream(targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}) + require.ErrorIs(t, h.session(stream, nil, "10", true, + watch.Event{Type: watch.Added, Object: filterObject("a", "11", "X")}), errTargetWatchClosed) + require.ErrorIs(t, h.session(stream, []*unstructured.Unstructured{filterObject("a", "20", "Z")}, "20", true, + modified(filterObject("a", "21", "X"))), errTargetWatchClosed) + + assert.Equal(t, []string{"a=X", "a=X"}, h.accepted(), "the return to X must reach the worker") + require.Len(t, h.resyncs.requests, 2) + }) +} + +// The commit-window regression. B is replayed and then only its status changes, while alice is +// editing A. B's update has no author; routed between alice's two edits, it would close her window +// on the author change and split her save into two commits. The replay is B's baseline, so it is +// filtered and alice's edits reach the worker back to back. +func TestDesiredStateChangeFilter_AReplayedStatusUpdateStaysOutOfAnotherAuthorsWindow(t *testing.T) { + bothReplayModes(t, func(t *testing.T, h *filterHarness) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + h.manager.AuthorResolver = recordingAuthors{} + stream := testStream(targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}) + + replayed := []*unstructured.Unstructured{filterObject("a", "10", "A0"), filterObject("b", "10", "B0")} + require.ErrorIs(t, h.session(stream, replayed, "10", true, + modified(filterObject("a", "11", "A1")), + modified(statusOnly(filterObject("b", "12", "B0"))), + modified(filterObject("a", "13", "A2")), + ), errTargetWatchClosed) + + accepted := h.live.accepted() + require.Len(t, accepted, 2, "only alice's edits reach the worker") + for _, event := range accepted { + assert.Equal(t, "a", event.Identifier.Name) + assert.Equal(t, "alice", event.UserInfo.Username) + } + assert.Equal(t, int64(1), unchangedEvents(t, reader, h.gitDest)) + }) +} + +// A snapshot the worker refused, or one the session never finished, is not the worker's state, so +// the stream keeps the baselines it had. The retry that is accepted replaces them. +func TestDesiredStateChangeFilter_ARefusedOrIncompleteReplayInstallsNothing(t *testing.T) { + bothReplayModes(t, func(t *testing.T, h *filterHarness) { + stream := testStream(targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}) + require.ErrorIs(t, h.session(stream, []*unstructured.Unstructured{filterObject("a", "10", "X")}, "10", true), + errTargetWatchClosed) + baseline := stream.changes.baselines["a"] + require.NotEmpty(t, baseline) + + h.resyncs.setRefuse(true) + err := h.session(stream, []*unstructured.Unstructured{filterObject("a", "20", "Z")}, "20", true) + require.ErrorIs(t, err, git.ErrFinalizeQueueFull) + assert.Equal(t, baseline, stream.changes.baselines["a"], "a refused snapshot installs no candidate hash") + h.resyncs.setRefuse(false) + + if h.mode == initialEvents { + require.ErrorIs( + t, + h.session(stream, []*unstructured.Unstructured{filterObject("a", "30", "Z")}, "30", false), + errTargetWatchClosed, + ) + assert.Equal(t, baseline, stream.changes.baselines["a"], "an unfinished replay installs nothing") + } + + require.ErrorIs(t, h.session(stream, []*unstructured.Unstructured{filterObject("a", "40", "Z")}, "40", true, + modified(statusOnly(filterObject("a", "41", "Z"))), + modified(filterObject("a", "42", "X")), + ), errTargetWatchClosed) + assert.Equal(t, []string{"a=X"}, h.accepted(), "the accepted retry is the baseline: Z is unchanged, X routes") + }) +} + +// A replay that no longer holds an object takes its baseline with it: the worker's snapshot swept +// it, so its next update is a change, whatever the stream last saw. +func TestDesiredStateChangeFilter_AReplayThatOmitsAnObjectDropsItsBaseline(t *testing.T) { + bothReplayModes(t, func(t *testing.T, h *filterHarness) { + stream := testStream(targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}) + require.ErrorIs(t, h.session(stream, nil, "10", true, + watch.Event{Type: watch.Added, Object: filterObject("q", "11", "X")}), errTargetWatchClosed) + require.ErrorIs(t, h.session(stream, nil, "20", true, modified(filterObject("q", "21", "X"))), + errTargetWatchClosed) + assert.Equal(t, []string{"q=X", "q=X"}, h.accepted()) + }) +} + +// A reconnect stays within the stream and keeps its accepted baselines; a replacement stream starts +// with none; and no baseline survives an accepted DELETE. +func TestDesiredStateChangeFilter_LifetimeFollowsTheStream(t *testing.T) { + h := newFilterHarness(t, initialEvents) + key := targetWatchKey{GVR: configmapsGVR, Namespace: "apps"} + stream := testStream(key) + require.ErrorIs(t, h.session(stream, []*unstructured.Unstructured{filterObject("a", "10", "X")}, "10", true), + errTargetWatchClosed) + + require.ErrorIs(t, h.resume(stream, modified(statusOnly(filterObject("a", "11", "X")))), errTargetWatchClosed) + assert.Empty(t, h.accepted(), "a reconnect keeps the replay's baseline") + + replacement := testStream(key) + require.ErrorIs(t, h.resume(replacement, modified(statusOnly(filterObject("a", "12", "X")))), + errTargetWatchClosed) + assert.Equal(t, []string{"a=X"}, h.accepted(), "a replacement stream has no predecessor's baseline") + + require.ErrorIs(t, h.resume(stream, + watch.Event{Type: watch.Deleted, Object: filterObject("a", "13", "X")}, + modified(filterObject("a", "14", "X")), + ), errTargetWatchClosed) + assert.Equal(t, []string{"a=X", "a=", "a=X"}, h.accepted(), "no baseline survives the deletion") +} + +// runTargetWatch gives each stream it runs a fresh filter, so a stream started again for the same +// collection never inherits a baseline. +func TestRunTargetWatch_GivesEachStreamItsOwnFilter(t *testing.T) { + h := newFilterHarness(t, initialEvents) + stream := targetWatchStream{key: targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}, gate: &producerGate{}} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + h.manager.runTargetWatch(ctx, logr.Discard(), h.gitDest, stream) + }() + fw := <-h.opened + bookmark := &unstructured.Unstructured{} + bookmark.SetResourceVersion("10") + bookmark.SetAnnotations(map[string]string{metav1.InitialEventsAnnotationKey: "true"}) + fw.Add(filterObject("a", "10", "X")) + fw.Action(watch.Bookmark, bookmark) + fw.Modify(statusOnly(filterObject("a", "11", "X"))) + fw.Modify(filterObject("a", "12", "Y")) + require.Eventually(t, func() bool { return len(h.live.accepted()) == 1 }, time.Second, 10*time.Millisecond) + cancel() + <-done + assert.Equal(t, []string{"a=Y"}, h.accepted(), "the stream's own replay was its baseline") + assert.Nil(t, stream.changes, "the caller's copy is untouched; the filter lives with the run") +} + +func unchangedEvents(t *testing.T, reader *sdkmetric.ManualReader, gitDest types.ResourceReference) int64 { + t.Helper() + count, _ := telemetry.CollectInt64Sum(reader, watchEventsMetric, map[string]string{ + "gittarget_namespace": gitDest.Namespace, "gittarget_name": gitDest.Name, + "group": "", "version": "v1", "resource": "configmaps", "outcome": watchOutcomeUnchanged, + }) + return count +} diff --git a/internal/watch/event_router.go b/internal/watch/event_router.go index 542baccd..8e65bfb0 100644 --- a/internal/watch/event_router.go +++ b/internal/watch/event_router.go @@ -40,6 +40,21 @@ type EventRouter struct { // Registry of GitTargetEventStreams by gitDest key gitTargetStreams map[string]*reconcile.GitTargetEventStream streamsMu sync.RWMutex + + // resyncWorker overrides the worker a scoped resync enters. nil resolves the GitTarget's branch + // worker; tests set it to accept or refuse a snapshot without running one. + resyncWorker func(ctx context.Context, gitDest types.ResourceReference) (resyncEnqueuer, error) +} + +// resyncEnqueuer is the part of a branch worker a scoped resync enters through. +type resyncEnqueuer interface { + EnqueueResync(request *git.ResyncRequest) bool +} + +// intakePauser is the part of a branch worker that says whether it has paused intake. See +// git.BranchWorker.IntakePaused. +type intakePauser interface { + IntakePaused() <-chan struct{} } // NewEventRouter creates a new event router. @@ -95,6 +110,7 @@ func (r *EventRouter) ServiceCommitRequest( // Report where the worker says the request stands, so the controller never has to infer // a phase from having sent the attach. result.Phase = worker.LookupCommitRequestPhase(attach.Namespace, attach.Name, attach.UID) + result.Held = heldBecause(worker, result.Phase) } return result, resolved, nil } @@ -135,7 +151,7 @@ func (r *EventRouter) WithdrawCommitRequest( } phase := worker.LookupCommitRequestPhase(attach.Namespace, attach.Name, attach.UID) if phase.Held() { - return git.FinalizeResult{Branch: branch, Phase: phase}, false, nil + return git.FinalizeResult{Branch: branch, Phase: phase, Held: heldBecause(worker, phase)}, false, nil } worker.EnqueueWithdraw(&attach) // A worker whose loop has exited answers the withdraw at once. @@ -145,6 +161,15 @@ func (r *EventRouter) WithdrawCommitRequest( return git.FinalizeResult{Branch: branch, Phase: phase}, false, nil } +// heldBecause is why a request waiting for its push has not reached the remote: the worker's +// publication report while it cannot publish, empty otherwise. +func heldBecause(worker *git.BranchWorker, phase git.CommitRequestPhase) string { + if phase != git.PhaseWaitingForPush { + return "" + } + return worker.Publication().Message() +} + // commitRequestWorker finds the worker to ask about a CommitRequest, and the branch it serves. // // The worker that holds the request comes first, whatever its GitTarget says now: a shared worker @@ -240,7 +265,6 @@ func (r *EventRouter) resolveWorkerForGitDest( // than the gather deletes managed documents outside it. heal marks a drift-correcting resync the // worker defers while a commit window is open. enqueued is false when the worker's queue was full // and dropped the request (its failure is still delivered on resultCh for the drain to record). -// refreshRemote asks the worker to fetch the latest remote tip before it inspects the folder. // resourceVersion is the LIST's collection version the desired set is pinned to. func (r *EventRouter) enqueueScopedResync( ctx context.Context, @@ -250,9 +274,8 @@ func (r *EventRouter) enqueueScopedResync( desired []manifestanalyzer.DesiredResource, resourceVersion string, heal bool, - refreshRemote bool, ) (chan git.ResyncResult, bool, error) { - worker, err := r.resolveWorkerForGitDest(ctx, gitDest) + worker, err := r.resyncTarget(ctx, gitDest) if err != nil { return nil, false, err } @@ -265,12 +288,37 @@ func (r *EventRouter) enqueueScopedResync( Scope: &scope, SourceCollection: sourceCollection, Heal: heal, - RefreshRemote: refreshRemote, Result: resultCh, }) return resultCh, enqueued, nil } +// resyncTarget is the branch worker a GitTarget's scoped resyncs enter. +func (r *EventRouter) resyncTarget(ctx context.Context, gitDest types.ResourceReference) (resyncEnqueuer, error) { + if r.resyncWorker != nil { + return r.resyncWorker(ctx, gitDest) + } + worker, err := r.resolveWorkerForGitDest(ctx, gitDest) + if err != nil { + return nil, err + } + return worker, nil +} + +// branchIntakePaused reports a GitTarget whose branch worker has paused intake: it returns a channel +// closed when intake reopens, or nil while it is open or the worker cannot be resolved, in which +// case nothing is known to wait for. +func (r *EventRouter) branchIntakePaused(ctx context.Context, gitDest types.ResourceReference) <-chan struct{} { + worker, err := r.resyncTarget(ctx, gitDest) + if err != nil { + return nil + } + if pauser, ok := worker.(intakePauser); ok { + return pauser.IntakePaused() + } + return nil +} + // resyncScopeForWatchKey is the single conversion from a watch key to the resync scope its // replay must be swept under. It exists so the two halves of the invariant — the namespace a // stream gathered, and the namespace its sweep may touch — are derived from ONE value and diff --git a/internal/watch/event_router_test.go b/internal/watch/event_router_test.go index 9a54e55a..5b500447 100644 --- a/internal/watch/event_router_test.go +++ b/internal/watch/event_router_test.go @@ -107,7 +107,6 @@ func TestEnqueueScopedResync_ReportsMissingWorker(t *testing.T) { nil, "12", false, - false, ) require.Error(t, err) @@ -141,7 +140,6 @@ func TestEnqueueScopedResync_ReportsGoneGitTargetAsTerminal(t *testing.T) { nil, "12", false, - false, ) require.Error(t, err) diff --git a/internal/watch/helpers_test.go b/internal/watch/helpers_test.go index 32be9687..526474ca 100644 --- a/internal/watch/helpers_test.go +++ b/internal/watch/helpers_test.go @@ -27,7 +27,7 @@ func itemScope(namespaces ...string) [][]string { // testStream is a running stream, for the tests that drive one stream directly. func testStream(key targetWatchKey) targetWatchStream { - return targetWatchStream{key: key, gate: &producerGate{}} + return targetWatchStream{key: key, gate: &producerGate{}, changes: newDesiredStateChangeFilter()} } func TestNormalizeResource(t *testing.T) { diff --git a/internal/watch/live_content_dedup_test.go b/internal/watch/live_content_dedup_test.go deleted file mode 100644 index 86440192..00000000 --- a/internal/watch/live_content_dedup_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package watch - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - - "github.com/ConfigButler/gitops-reverser/internal/git" - "github.com/ConfigButler/gitops-reverser/internal/types" -) - -// dedupGVR is the deployments GVR used across the dedup table. -func dedupGVR() schema.GroupVersionResource { - return schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} -} - -// callSkip drives skipUnchangedLiveUpdate for one object identity, with a sanitized -// content marker (empty for delete, where the writer leaves Object nil). -func callSkip(m *Manager, gitDest types.ResourceReference, uid, content, op string) bool { - u := &unstructured.Unstructured{Object: map[string]interface{}{ - "metadata": map[string]interface{}{"uid": uid}, - }} - event := &git.Event{ - Identifier: types.NewResourceIdentifier("apps", "v1", "deployments", "ns", "d"), - Operation: op, - } - if op != string(types.OperationDelete) { - event.Object = &unstructured.Unstructured{Object: map[string]interface{}{ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": map[string]interface{}{"name": "d", "namespace": "ns"}, - "spec": map[string]interface{}{"marker": content}, - }} - } - return m.skipUnchangedLiveUpdate(gitDest, dedupGVR(), u, event, op) -} - -func TestSkipUnchangedLiveUpdate(t *testing.T) { - m := &Manager{} - dest := types.NewResourceReference("gt", "ns") - - create := string(types.OperationCreate) - update := string(types.OperationUpdate) - del := string(types.OperationDelete) - - // A CREATE always routes and seeds the cache. - assert.False(t, callSkip(m, dest, "uid-1", "A", create), "CREATE must always route") - - // A status-only UPDATE sanitizes to the same content → skipped (the bug fix). - assert.True(t, callSkip(m, dest, "uid-1", "A", update), "no-op UPDATE must be skipped") - assert.True(t, callSkip(m, dest, "uid-1", "A", update), "repeated no-op UPDATE stays skipped") - - // A real UPDATE (content changed) routes and refreshes the cache. - assert.False(t, callSkip(m, dest, "uid-1", "B", update), "a content change must route") - assert.True(t, callSkip(m, dest, "uid-1", "B", update), "the new content then dedups") - - // DELETE always routes and clears the cache, so a recreate is never deduped away. - assert.False(t, callSkip(m, dest, "uid-1", "", del), "DELETE must always route") - assert.False(t, callSkip(m, dest, "uid-1", "B", create), "recreate after delete must route") -} - -// A first-seen UPDATE (no prior CREATE in this session) routes: we cannot prove it is a -// no-op without a baseline, so we fail open. -func TestSkipUnchangedLiveUpdate_FirstSeenUpdateRoutes(t *testing.T) { - m := &Manager{} - dest := types.NewResourceReference("gt", "ns") - assert.False(t, callSkip(m, dest, "uid-x", "A", string(types.OperationUpdate)), - "a first-seen UPDATE has no baseline and must route") -} - -// The same object mirrored to two GitTargets dedups independently: a no-op for one -// stream must not suppress routing to the other. -func TestSkipUnchangedLiveUpdate_PerGitTargetIsolation(t *testing.T) { - m := &Manager{} - destA := types.NewResourceReference("gt-a", "ns") - destB := types.NewResourceReference("gt-b", "ns") - - create := string(types.OperationCreate) - assert.False(t, callSkip(m, destA, "uid-1", "A", create)) - // destB has never seen this object: its CREATE still routes. - assert.False(t, callSkip(m, destB, "uid-1", "A", create), "a different GitTarget dedups independently") -} diff --git a/internal/watch/manager.go b/internal/watch/manager.go index 4c24dcd6..cf0f51e2 100644 --- a/internal/watch/manager.go +++ b/internal/watch/manager.go @@ -102,15 +102,6 @@ type Manager struct { opts metav1.ListOptions, ) (*unstructured.UnstructuredList, error) - // liveContentDedup caches, per (gitDest, object), the hash of the last sanitized - // content routed to a branch worker. A live UPDATE whose sanitized content is - // unchanged (the classic /status-only churn, which carries no git-writable change) - // is dropped before routing, so it cannot split an open commit window by arriving - // unattributed against a named window author. Keyed by gitDest+gvr+namespace+uid; - // entries are cleared on delete. Cross-session by design: a reconnect keeps deduping - // against what git already holds. See routeLiveTargetWatchEvent. - liveContentDedup sync.Map - // SourceClusters resolves a GitTarget's source cluster — a ClusterProvider NAME — into a // rest.Config, reading the kubeconfig Secret the provider names from the config plane. It is // required for any GitTarget to mirror, single-cluster installs included: a source cluster is @@ -182,6 +173,10 @@ type Manager struct { // only writer and its only reader, so cancelling a stream and starting one no longer happen // under a lock that the woken goroutine then has to contend for. See owner.go. targetWatches map[string]*targetWatchSet + // targetWatchesRunning counts the goroutines running a target watch. Cancelling a stream + // returns before its goroutine has finished its last session, so a caller that must know the + // streams are gone, a test resetting process-wide state, waits on it. + targetWatchesRunning sync.WaitGroup // watchLifetime is the parent of every target watch's context: the manager's own lifetime, // set once by Start. diff --git a/internal/watch/object_selector_test.go b/internal/watch/object_selector_test.go index 34079720..271e7105 100644 --- a/internal/watch/object_selector_test.go +++ b/internal/watch/object_selector_test.go @@ -202,13 +202,13 @@ func TestResolveWatchedTypeTables_RefusesTheNewerOfTwoOverlappingSelectors(t *te assert.Equal(t, []string{""}, table.Types[0].WatchScopes()) assert.Equal(t, map[string]string{"": teamA}, table.Types[0].NamespaceScopes) - refused, message := manager.ObjectSelectorConflictForWatchRule( + refused, message := manager.CollectionOverlapForWatchRule( selectorRule("named", "team-b", older.Add(time.Minute), labels("team", "b"), "")) assert.True(t, refused) assert.Contains(t, message, "WatchRule test-ns/cluster-wide") assert.Contains(t, message, `objectSelector "team in (b)"`) - refused, _ = manager.ObjectSelectorConflictForWatchRule( + refused, _ = manager.CollectionOverlapForWatchRule( selectorRule("cluster-wide", "test-ns", older, labels("team", "a"), "*")) assert.False(t, refused, "the older rule keeps its collection") } @@ -224,91 +224,12 @@ func TestResolveWatchedTypeTables_DisjointNamespacesMayUseDifferentSelectors(t * require.Len(t, table.Types, 1) assert.Equal(t, map[string]string{"team-a": teamA, "team-b": "team in (b)"}, table.Types[0].NamespaceScopes) for _, name := range []string{"a", "b"} { - refused, message := manager.ObjectSelectorConflictForWatchRule( + refused, message := manager.CollectionOverlapForWatchRule( selectorRule(name, "team-"+name, now, nil, "")) assert.False(t, refused, message) } } -// Equivalent selectors written differently are one canonical selector, so they never conflict. -func TestResolveWatchedTypeTables_EquivalentSelectorsDoNotConflict(t *testing.T) { - manager, store := makeWatchedTypeManager(t) - now := time.Unix(1000, 0) - addSelectorRule(store, selectorRule("labels", "test-ns", now, labels("team", "a"), "*"), "") - addSelectorRule(store, selectorRule("expression", "team-a", now.Add(time.Minute), &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - {Key: "team", Operator: metav1.LabelSelectorOpIn, Values: []string{"a", "a"}}, - }, - }, ""), "team-a") - - manager.refreshWatchedTypeTables() - table, _ := manager.watchedTypeTableForGitDest(gitDestRef("sel-target")) - require.Len(t, table.Types, 1) - assert.Equal(t, []string{"", "team-a"}, table.Types[0].WatchScopes()) - refused, message := manager.ObjectSelectorConflictForWatchRule( - selectorRule("expression", "team-a", now.Add(time.Minute), nil, "")) - assert.False(t, refused, message) -} - -func TestRefuseSelectorConflicts(t *testing.T) { - rec := typesetRecordForTest(configmapsGVR) - older := selectingRule{ - kind: ruleKindWatchRule, - namespace: "ns", - name: "b", - createdAt: metav1.NewTime(time.Unix(1, 0)), - } - newer := selectingRule{ - kind: ruleKindWatchRule, - namespace: "ns", - name: "a", - createdAt: metav1.NewTime(time.Unix(2, 0)), - } - sameAgeLowerName := selectingRule{kind: ruleKindWatchRule, namespace: "ns", name: "a", createdAt: older.createdAt} - - t.Run("the older rule wins regardless of order", func(t *testing.T) { - admitted, refused := refuseSelectorConflicts([]watchSelection{ - {record: rec, namespace: "x", labelSelector: "b=1", rule: newer}, - {record: rec, namespace: "x", labelSelector: "a=1", rule: older}, - }) - require.Len(t, admitted, 1) - assert.Equal(t, older, admitted[0].rule) - assert.Contains(t, refused, newer) - }) - t.Run("equal age falls back to the lower name", func(t *testing.T) { - _, refused := refuseSelectorConflicts([]watchSelection{ - {record: rec, namespace: "x", labelSelector: "b=1", rule: older}, - {record: rec, namespace: "x", labelSelector: "a=1", rule: sameAgeLowerName}, - }) - assert.Contains(t, refused, older) - assert.NotContains(t, refused, sameAgeLowerName) - }) - t.Run("a rule conflicting with itself is refused", func(t *testing.T) { - admitted, refused := refuseSelectorConflicts([]watchSelection{ - {record: rec, namespace: "", labelSelector: "a=1", rule: older}, - {record: rec, namespace: "x", labelSelector: "", rule: older}, - }) - assert.Empty(t, admitted) - assert.Contains(t, refused[older], "its items select overlapping collections differently") - }) - t.Run("the same selector over overlapping scopes is allowed", func(t *testing.T) { - admitted, refused := refuseSelectorConflicts([]watchSelection{ - {record: rec, namespace: "", labelSelector: "a=1", rule: older}, - {record: rec, namespace: "x", labelSelector: "a=1", rule: newer}, - }) - assert.Len(t, admitted, 2) - assert.Empty(t, refused) - }) - t.Run("other types never conflict", func(t *testing.T) { - admitted, refused := refuseSelectorConflicts([]watchSelection{ - {record: rec, namespace: "x", labelSelector: "a=1", rule: older}, - {record: typesetRecordForTest(resolverGVR), namespace: "x", labelSelector: "", rule: newer}, - }) - assert.Len(t, admitted, 2) - assert.Empty(t, refused) - }) -} - func TestCompileWatchRule_RefusesAnInvalidObjectSelector(t *testing.T) { store := rulestore.NewStore() rule := watchRuleForTarget("bad", "sel-target", "team-a") diff --git a/internal/watch/owner_test.go b/internal/watch/owner_test.go index c167aee7..c78886ba 100644 --- a/internal/watch/owner_test.go +++ b/internal/watch/owner_test.go @@ -458,7 +458,6 @@ func TestOwner_AStreamOutlivesThePassThatStartedIt(t *testing.T) { keysByCollection([]targetWatchKey{{GVR: configmapsGVR, Namespace: "apps"}}), map[types.CollectionKey]uint64{}, []types.CollectionKey{types.CollectionKeyFor(configmapsGVR, "apps")}, - false, ) require.Len(t, started, 1) streamCtx = started[0].ctx diff --git a/internal/watch/prune_declaration_test.go b/internal/watch/prune_declaration_test.go index e1997aea..6799c545 100644 --- a/internal/watch/prune_declaration_test.go +++ b/internal/watch/prune_declaration_test.go @@ -145,6 +145,7 @@ func TestReplaceGitTargetWatches_ForceReplaysAnUnchangedSet(t *testing.T) { }, } manager.rememberGitTargetUID(gitDest.WithUID("uid-1")) + t.Cleanup(manager.targetWatchesRunning.Wait) table := WatchedTypeTable{ GitDest: gitDest, diff --git a/internal/watch/refused_admission_test.go b/internal/watch/refused_admission_test.go new file mode 100644 index 00000000..a610b19d --- /dev/null +++ b/internal/watch/refused_admission_test.go @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/watch" + + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/reconcile" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// admittingEnqueuer is a branch worker's admission as a live event sees it: it takes events +// until told to refuse them, the way a full queue or a closed admission gate does. +type admittingEnqueuer struct { + mu sync.Mutex + refuse bool + events []git.Event + refused int +} + +func (a *admittingEnqueuer) Enqueue(event git.Event) bool { + a.mu.Lock() + defer a.mu.Unlock() + if a.refuse { + a.refused++ + return false + } + a.events = append(a.events, event) + return true +} + +func (a *admittingEnqueuer) setRefuse(refuse bool) { + a.mu.Lock() + defer a.mu.Unlock() + a.refuse = refuse +} + +func (a *admittingEnqueuer) accepted() []git.Event { + a.mu.Lock() + defer a.mu.Unlock() + return append([]git.Event(nil), a.events...) +} + +// dedupConfigMap is the demo ConfigMap at one resourceVersion with one data value, carrying a +// UID so the dedup cache keys it the way a live object is keyed. +func dedupConfigMap(rv, value string) *unstructured.Unstructured { + obj := configMapObject(rv) + obj.SetUID("cm-uid") + obj.Object["data"] = map[string]interface{}{"key": value} + return obj +} + +func admissionTestManager(gitDest types.ResourceReference) (*Manager, *admittingEnqueuer, *fakeWatchCursorStore) { + enqueuer := &admittingEnqueuer{} + stream := reconcile.NewGitTargetEventStream(gitDest.Name, gitDest.Namespace, enqueuer, logr.Discard()) + store := &fakeWatchCursorStore{} + manager := &Manager{ + Log: logr.Discard(), + EventRouter: &EventRouter{ + Log: logr.Discard(), + gitTargetStreams: map[string]*reconcile.GitTargetEventStream{gitDest.Key(): stream}, + }, + WatchCursorStore: store, + } + manager.rememberGitTargetUID(gitDest.WithUID("uid-1")) + return manager, enqueuer, store +} + +// The producer half of admission backpressure: a refused UPDATE is redelivered by a cursor resume, +// and only that redelivery's ACCEPTANCE may move the cursor past it. Before the fix the refused +// attempt had already stored its content hash, so the redelivered frame matched it, was skipped as +// a no-op, and advanced the cursor past a change Git never received. +func TestProcessLiveTargetWatchEvent_ARefusedUpdateIsOfferedAgainOnCursorResume(t *testing.T) { + gitDest := types.NewResourceReference("target", "default") + manager, enqueuer, store := admissionTestManager(gitDest) + stream := testStream(targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}) + process := func(ev watch.Event) error { + return manager.processLiveTargetWatchEvent(context.Background(), logr.Discard(), gitDest, stream, ev) + } + + require.NoError(t, process(watch.Event{Type: watch.Added, Object: dedupConfigMap("10", "A")})) + require.Equal(t, "10", store.lastRecordedRV()) + + enqueuer.setRefuse(true) + require.Error(t, process(watch.Event{Type: watch.Modified, Object: dedupConfigMap("11", "B")}), + "a refused event ends the session") + assert.Equal(t, "10", store.lastRecordedRV(), "the cursor stays before the refused change") + + // The reconnect resumes from 10 and the API server delivers the same frame again. + enqueuer.setRefuse(false) + require.NoError(t, process(watch.Event{Type: watch.Modified, Object: dedupConfigMap("11", "B")})) + accepted := enqueuer.accepted() + require.Len(t, accepted, 2, "the redelivered UPDATE must reach the worker, not be deduped away") + assert.Equal(t, "UPDATE", accepted[1].Operation) + assert.Equal(t, "11", store.lastRecordedRV()) + + // Accepted content still suppresses a /status-only UPDATE that sanitizes to it. + require.NoError(t, process(watch.Event{Type: watch.Modified, Object: dedupConfigMap("12", "B")})) + assert.Len(t, enqueuer.accepted(), 2, "a no-op UPDATE against accepted content is still skipped") + assert.Equal(t, "12", store.lastRecordedRV(), "and a skipped no-op still advances the cursor") +} + +// An event a cancelled stream never enqueued is neither a baseline nor a cursor position. +func TestProcessLiveTargetWatchEvent_ACancelledStreamNeitherDedupsNorAdvances(t *testing.T) { + gitDest := types.NewResourceReference("target", "default") + manager, enqueuer, store := admissionTestManager(gitDest) + key := targetWatchKey{GVR: configmapsGVR, Namespace: "apps"} + old := testStream(key) + + require.NoError(t, manager.processLiveTargetWatchEvent(context.Background(), logr.Discard(), gitDest, old, + watch.Event{Type: watch.Added, Object: dedupConfigMap("10", "A")})) + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + require.NoError(t, manager.processLiveTargetWatchEvent(cancelled, logr.Discard(), gitDest, old, + watch.Event{Type: watch.Modified, Object: dedupConfigMap("11", "B")})) + assert.Len(t, enqueuer.accepted(), 1, "a cancelled stream enqueues nothing") + assert.Equal(t, "10", store.lastRecordedRV(), "and records no cursor past what it did not enqueue") + + replacement := testStream(key) + require.NoError(t, manager.processLiveTargetWatchEvent(context.Background(), logr.Discard(), gitDest, + replacement, watch.Event{Type: watch.Modified, Object: dedupConfigMap("11", "B")})) + assert.Len(t, enqueuer.accepted(), 2, "the next stream's B is a change: the cancelled one never delivered it") +} + +// A stream resumes from a stored cursor only after one of its own sessions completed a replay. +// The first session here ends mid-replay, with a previous stream's cursor (41) still stored: the +// reconnect must replay again, because resuming from 41 would skip this stream's snapshot and its +// sweep. Once that replay completes (cursor 50), the next reconnect resumes from it. +func TestRunTargetWatch_ResumesOnlyAfterItsOwnReplayCompleted(t *testing.T) { + gitDest := types.NewResourceReference("target", "default") + key := targetWatchKey{GVR: configmapsGVR, Namespace: "apps"} + store := &fakeWatchCursorStore{rv: "41", ok: true} + opened := make(chan openedWatch, 4) + manager := &Manager{ + Log: logr.Discard(), + WatchCursorStore: store, + targetWatchOpen: func( + _ context.Context, + _ schema.GroupVersionResource, + namespace string, + opts metav1.ListOptions, + ) (watch.Interface, error) { + fw := watch.NewFake() + opened <- openedWatch{namespace: namespace, opts: opts, watch: fw} + return fw, nil + }, + } + manager.rememberGitTargetUID(gitDest.WithUID("uid-1")) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { + defer close(done) + manager.runTargetWatch(ctx, logr.Discard(), gitDest, testStream(key)) + }() + + first := receiveOpenedWatch(t, opened) + require.NotNil(t, first.opts.SendInitialEvents, "a new stream replays") + first.watch.Add(configMapObject("45")) + first.watch.Stop() + + second := receiveReconnect(t, opened) + require.NotNil(t, second.opts.SendInitialEvents, + "a stream whose replay never completed must replay again, not resume a previous stream's cursor") + assert.Empty(t, second.opts.ResourceVersion) + + bookmark := &unstructured.Unstructured{} + bookmark.SetResourceVersion("50") + bookmark.SetAnnotations(map[string]string{metav1.InitialEventsAnnotationKey: "true"}) + second.watch.Action(watch.Bookmark, bookmark) + require.Eventually(t, func() bool { return store.lastRecordedRV() == "50" }, time.Second, 10*time.Millisecond) + store.mu.Lock() + store.rv = "50" + store.mu.Unlock() + second.watch.Stop() + + third := receiveReconnect(t, opened) + assert.Nil(t, third.opts.SendInitialEvents, "after its own replay the stream resumes") + assert.Equal(t, "50", third.opts.ResourceVersion) + + cancel() + <-done +} + +// receiveReconnect waits out the reconnect backoff for the next open. +func receiveReconnect(t *testing.T, opened <-chan openedWatch) openedWatch { + t.Helper() + select { + case got := <-opened: + return got + case <-time.After(2 * targetWatchBackoff): + t.Fatal("expected the target watch to reconnect") + return openedWatch{} + } +} + +// pausingResyncs is a branch worker that has paused intake: it refuses every snapshot until it +// reopens, and says so through IntakePaused, the way the real worker does. +type pausingResyncs struct { + mu sync.Mutex + paused chan struct{} +} + +func (p *pausingResyncs) EnqueueResync(request *git.ResyncRequest) bool { + p.mu.Lock() + defer p.mu.Unlock() + if p.paused != nil { + request.Result <- git.ResyncResult{Err: git.ErrFinalizeQueueFull} + return false + } + request.Result <- git.ResyncResult{} + return true +} + +func (p *pausingResyncs) IntakePaused() <-chan struct{} { + p.mu.Lock() + defer p.mu.Unlock() + return p.paused +} + +func (p *pausingResyncs) pause() { + p.mu.Lock() + defer p.mu.Unlock() + p.paused = make(chan struct{}) +} + +// wake wakes the waiting producers; pausedAgain pauses again before they get to look. +func (p *pausingResyncs) wake(pausedAgain bool) { + p.mu.Lock() + defer p.mu.Unlock() + close(p.paused) + p.paused = nil + if pausedAgain { + p.paused = make(chan struct{}) + } +} + +// A stream whose branch has paused intake waits for the branch to reopen instead of reconnecting on +// its backoff: reconnecting would gather a snapshot the branch refuses again, every two seconds, +// for as long as the outage lasts. A wake-up finds the branch paused again and keeps waiting; the +// reopening reconnects at once. +func TestRunTargetWatch_APausedBranchWaitsForIntakeToReopen(t *testing.T) { + setTargetWatchBackoff(t, 10*time.Millisecond) + gitDest := types.NewResourceReference("target", "default") + worker := &pausingResyncs{} + worker.pause() + opened := make(chan openedWatch, 8) + manager := &Manager{ + Log: logr.Discard(), + WatchCursorStore: &fakeWatchCursorStore{}, + EventRouter: &EventRouter{ + Log: logr.Discard(), + resyncWorker: func(context.Context, types.ResourceReference) (resyncEnqueuer, error) { + return worker, nil + }, + }, + targetWatchOpen: func( + _ context.Context, _ schema.GroupVersionResource, namespace string, opts metav1.ListOptions, + ) (watch.Interface, error) { + fw := watch.NewFakeWithChanSize(8, false) + opened <- openedWatch{namespace: namespace, opts: opts, watch: fw} + return fw, nil + }, + } + manager.rememberGitTargetUID(gitDest.WithUID("uid-1")) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + key := targetWatchKey{GVR: configmapsGVR, Namespace: "apps"} + go func() { + defer close(done) + manager.runTargetWatch(ctx, logr.Discard(), gitDest, testStream(key)) + }() + + first := receiveOpenedWatch(t, opened) + completeInitialEvents(first.watch, "50") // the paused branch refuses the replay's snapshot + assertNoReconnectWithin(t, opened, 300*time.Millisecond, "a paused branch is not offered the snapshot again") + waiting := streamStatusFor(manager, gitDest, key) + assert.Equal(t, StreamStateReplaying, waiting.state, + "a stream waiting for its branch is not current, and not broken: no WatchError to stall the target") + assert.Equal(t, StreamReasonBranchIntakePaused, waiting.reason) + + worker.wake(true) + assertNoReconnectWithin(t, opened, 300*time.Millisecond, "a wake-up that finds the branch paused again waits on") + + worker.wake(false) + second := receiveReconnect(t, opened) + assert.NotNil(t, second.opts.SendInitialEvents, "the reopened branch gets the replay it refused") + + cancel() + <-done +} + +// completeInitialEvents ends an initial-events replay with its bookmark at rv. +func completeInitialEvents(fw *watch.FakeWatcher, rv string) { + bookmark := &unstructured.Unstructured{} + bookmark.SetResourceVersion(rv) + bookmark.SetAnnotations(map[string]string{metav1.InitialEventsAnnotationKey: "true"}) + fw.Action(watch.Bookmark, bookmark) +} + +func assertNoReconnectWithin(t *testing.T, opened <-chan openedWatch, d time.Duration, msg string) { + t.Helper() + select { + case got := <-opened: + got.watch.Stop() + t.Fatalf("reconnected: %s", msg) + case <-time.After(d): + } +} + +// setTargetWatchBackoff shortens the reconnect backoff for one test. +func setTargetWatchBackoff(t *testing.T, d time.Duration) { + t.Helper() + original := targetWatchBackoff + targetWatchBackoff = d + t.Cleanup(func() { targetWatchBackoff = original }) +} diff --git a/internal/watch/source_namespace_planning_test.go b/internal/watch/source_namespace_planning_test.go index 218b0fe7..822ed62e 100644 --- a/internal/watch/source_namespace_planning_test.go +++ b/internal/watch/source_namespace_planning_test.go @@ -4,9 +4,11 @@ package watch import ( "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" "github.com/ConfigButler/gitops-reverser/internal/rulestore" @@ -126,26 +128,32 @@ func TestCollectWatchRuleSelections_WildcardIsOneClusterWideScope(t *testing.T) "a wildcard compiles to the all-namespaces collection, read once") } -// A cluster-wide collection is a PEER of a named-namespace collection on the same type, never a -// replacement. Collapsing the two once widened the named rule's stream to every namespace its -// credential could read (CollectionKey's own doc comment records the bug). Two streams over -// overlapping objects is the correct outcome here. -func TestCollectWatchRuleSelections_WildcardIsAPeerOfANamedNamespace(t *testing.T) { +// A cluster-wide collection is never collapsed into a named-namespace one on the same type: +// collapsing them once widened the named rule's stream to every namespace its credential could read +// (CollectionKey's own doc comment records the bug). Nor may one GitTarget stream both, because +// the target would then receive every object in the named namespace twice. The newer rule is +// refused; the older keeps its own, unwidened collection. +func TestCollectWatchRuleSelections_WildcardAndANamedNamespaceRefuseTheNewer(t *testing.T) { manager, store := makeWatchedTypeManager(t) - addRule(store, - watchRuleWithSource("wild", "shared-target", configv1alpha3.SourceNamespaceWildcard), - itemScope("")) - addRule(store, - watchRuleWithSource("named", "shared-target", "repo-config"), - itemScope("repo-config")) + older := time.Unix(1000, 0) + named := watchRuleWithSource("named", "shared-target", "repo-config") + named.CreationTimestamp = metav1.NewTime(older) + wild := watchRuleWithSource("wild", "shared-target", configv1alpha3.SourceNamespaceWildcard) + wild.CreationTimestamp = metav1.NewTime(older.Add(time.Minute)) + addRule(store, wild, itemScope("")) + addRule(store, named, itemScope("repo-config")) manager.refreshWatchedTypeTables() table, ok := manager.watchedTypeTableForGitDest(gitDestRef("shared-target")) require.True(t, ok) require.Len(t, table.Types, 1) - assert.Equal(t, []string{"", "repo-config"}, table.Types[0].WatchScopes(), - "the named scope survives beside the cluster-wide one; collapsing them loses its filter") + assert.Equal(t, []string{"repo-config"}, table.Types[0].WatchScopes(), + "the older named scope keeps its own collection and is not widened") + refused, message := manager.CollectionOverlapForWatchRule(wild) + assert.True(t, refused) + assert.Contains(t, message, "configmaps in all namespaces") + assert.Contains(t, message, "configmaps in repo-config") } // TestCollectWatchRuleSelections_LegacyRuleStillWatchesItsOwnNamespace is the upgrade guarantee at diff --git a/internal/watch/stream_readiness.go b/internal/watch/stream_readiness.go index 6104b836..aa99ef43 100644 --- a/internal/watch/stream_readiness.go +++ b/internal/watch/stream_readiness.go @@ -35,6 +35,8 @@ const ( StreamReasonAllStreamsReady = "AllStreamsReady" StreamReasonReplaying = "Replaying" StreamReasonNoResolvedTypes = "NoResolvedTypes" + // StreamReasonBranchIntakePaused is a stream waiting for its branch worker to reopen intake. + StreamReasonBranchIntakePaused = "BranchIntakePaused" ) const pendingStreamSampleLimit = 5 diff --git a/internal/watch/target_watch.go b/internal/watch/target_watch.go index 288b0baa..11da8554 100644 --- a/internal/watch/target_watch.go +++ b/internal/watch/target_watch.go @@ -4,14 +4,13 @@ package watch import ( "context" - "crypto/sha256" - "encoding/json" "errors" "fmt" "sort" "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/go-logr/logr" @@ -23,16 +22,22 @@ import ( "k8s.io/utils/ptr" "github.com/ConfigButler/gitops-reverser/internal/git" - "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" "github.com/ConfigButler/gitops-reverser/internal/queue" "github.com/ConfigButler/gitops-reverser/internal/sanitize" "github.com/ConfigButler/gitops-reverser/internal/types" ) -const ( - targetWatchBackoff = 2 * time.Second - targetWatchBufferCapacity = 1024 -) +const targetWatchBufferCapacity = 1024 + +// targetWatchBackoff is the wait before a stream's next session. A var, not a const, so a test can +// shorten it. +// +//nolint:gochecknoglobals // a test seam +var targetWatchBackoff = 2 * time.Second + +// intakePauseRecheck bounds how long a stream waits on a paused branch without looking again. The +// branch wakes its producers when it reopens, so this is only a fallback. +const intakePauseRecheck = time.Minute var ( errTargetWatchClosed = errors.New("target watch result channel closed") @@ -165,10 +170,6 @@ type targetWatchStream struct { key targetWatchKey // gate is shared with the runningTargetWatch that stops this stream; see producerGate. gate *producerGate - // refreshRemote asks the worker to inspect the remote tip before applying this stream's - // replay. It is set on forced GitTarget rechecks, where a human may have fixed or broken the - // folder directly in Git and the local checkout is the stale thing being tested. - refreshRemote bool // revision is the incarnation of this stream's COLLECTION, issued by the render-fidelity gate // when the stream was started. It is captured at start, not read when a replay result is // ready: a cancelled stream that read the current revision on its way out would report its @@ -176,6 +177,24 @@ type targetWatchStream struct { // stream retired by a served-version change lands on the live collection's scope rather than // missing it, so the capture is what keeps it out. revision uint64 + // replayed is set once one of this stream's sessions has completed a replay and recorded the + // cursor that replay earned. runTargetWatch gives it to every stream it runs; nil reads unset. + replayed *atomic.Bool + // changes is this stream's desired-state change filter. runTargetWatch gives every stream a fresh + // one, so a replacement stream starts with no baselines; nil passes every event. + changes *desiredStateChangeFilter +} + +// markReplayed records that this stream's replay completed and its cursor is the stream's own. +func (s targetWatchStream) markReplayed() { + if s.replayed != nil { + s.replayed.Store(true) + } +} + +// hasReplayed reports whether a session of this stream has completed a replay. +func (s targetWatchStream) hasReplayed() bool { + return s.replayed != nil && s.replayed.Load() } // sourceCollection is what this stream stamps on the work it queues: the collection that produced it. @@ -288,7 +307,7 @@ func (m *Manager) replaceGitTargetWatches( collections := collectionsForWatchKeys(keys) m.resetTargetStreamStates(table.GitDest, collections, starting) revisions, fidelityChanged := m.reconcileTargetRenderFidelity(table.GitDest, collections, starting) - started := m.startTargetWatchStreams(ctx, set, keysByCollection(keys), revisions, starting, force) + started := m.startTargetWatchStreams(ctx, set, keysByCollection(keys), revisions, starting) m.retainTargetRetentionScopes(table.GitDest, streamRevisions(collections, revisions)) if fidelityChanged { @@ -296,12 +315,10 @@ func (m *Manager) replaceGitTargetWatches( } logTargetWatchPlanDiff(log, previous, desired, diff) for _, stream := range started { - go m.runTargetWatch(stream.ctx, log, table.GitDest, stream.stream) + m.targetWatchesRunning.Go(func() { m.runTargetWatch(stream.ctx, log, table.GitDest, stream.stream) }) } - // Name every declared stream, not just the count. A GVR appearing twice — once - // cluster-wide ("") and once under a named namespace — means the same object is - // delivered on two streams, which is legitimate scoping but doubles the events for - // objects in that namespace. That is invisible in a bare count. + // Name every declared stream, not just the count: which collections a target watches is + // invisible in a bare count. log.V(1).Info("watch-first target watch set reconciled", "watchCount", len(keys), "streams", describeWatchKeys(keys)) return nil @@ -335,7 +352,6 @@ func (m *Manager) startTargetWatchStreams( byCollection map[types.CollectionKey]targetWatchKey, revisions map[types.CollectionKey]uint64, starting []types.CollectionKey, - refreshRemote bool, ) []startingTargetWatch { out := make([]startingTargetWatch, 0, len(starting)) for _, collection := range starting { @@ -349,10 +365,9 @@ func (m *Manager) startTargetWatchStreams( out = append(out, startingTargetWatch{ ctx: streamCtx, stream: targetWatchStream{ - key: watchKey, - gate: gate, - refreshRemote: refreshRemote, - revision: revisions[collection], + key: watchKey, + gate: gate, + revision: revisions[collection], }, }) } @@ -466,9 +481,9 @@ func (m *Manager) forgetGitTargetWatches(gitDest types.ResourceReference) { // snapshots of one boundary, each sweeping the documents the other gathered. So the version is // chosen once, deterministically. // -// A cluster-wide scope ("") stays a peer of any named namespace on the same type, never a -// replacement for it: collapsing THOSE widened the named rule's stream -// (pr2-stream-scope-collapse.md). They are different collections, and both stream. +// A cluster-wide scope ("") is never collapsed into a named namespace on the same type: that +// widened the named rule's stream (pr2-stream-scope-collapse.md). The resolver refuses a rule that +// would put both on one GitTarget (refuseCollectionOverlaps), so no table carries the pair. func targetWatchStreams(table WatchedTypeTable) map[targetWatchKey]struct{} { chosen := map[types.CollectionKey]targetWatchKey{} chosenPreferred := map[types.CollectionKey]bool{} @@ -544,14 +559,20 @@ func (m *Manager) runTargetWatch( releaseFacts := m.followFactsForWatch(gitDest, stream.key) defer releaseFacts() - // Starting a stream issues its collection a fresh fidelity revision, so its first session must - // replay even when a durable cursor exists: resuming would leave that scope pending under - // the new revision forever. Later reconnects may resume from their cursors because they stay - // within the same stream, and so within the same revision. - resumeFromCursor := false + // Starting a stream issues its collection a fresh fidelity revision, so it must replay even + // when a durable cursor exists: resuming would leave that scope pending under the new + // revision forever. Later reconnects may resume from their cursors because they stay within + // the same stream, and so within the same revision. + // + // "Later" means after a replay of THIS stream completed, not after its first session. A + // session can end before its replay is accepted (the branch worker refused the snapshot under + // admission backpressure, or the watch failed mid-replay), and the cursor stored then is + // still a previous stream's: resuming from it would skip this stream's snapshot and its + // sweep, and leave the scope pending all the same. + stream.replayed = &atomic.Bool{} + stream.changes = newDesiredStateChangeFilter() for ctx.Err() == nil { - err := m.targetWatchReplayAndStream(ctx, log, gitDest, stream, resumeFromCursor) - resumeFromCursor = true + err := m.targetWatchReplayAndStream(ctx, log, gitDest, stream, stream.hasReplayed()) recordWatchSessionEnded(ctx, stream.key.GVR, sessionEndReason(ctx, err)) if ctx.Err() != nil { return @@ -566,18 +587,65 @@ func (m *Manager) runTargetWatch( return } if err != nil { - if state, reason, mark := targetStreamStateForSessionEnd(err); mark { + if m.branchPaused(ctx, gitDest) { + // The branch refused what this session offered because it cannot publish. That is + // the branch's outage, which its GitTarget reports, and not a broken watch: graded + // as a wait, so a Git outage never stalls the target as a WatchError. + m.markTargetStreamState(gitDest, stream.key.Collection(), StreamStateReplaying, + StreamReasonBranchIntakePaused, "waiting for the branch worker to reopen intake") + } else if state, reason, mark := targetStreamStateForSessionEnd(err); mark { m.markTargetStreamState(gitDest, stream.key.Collection(), state, reason, err.Error()) } log.Info("target watch session ended; reconnecting", "gvr", stream.key.GVR.String(), "namespace", stream.key.Namespace, "err", err.Error()) } - if !sleepOrDone(ctx, targetWatchBackoff) { + if !m.waitToReconnect(ctx, log, gitDest, stream.key) { return } } } +// branchPaused reports whether the GitTarget's branch worker has paused intake. +func (m *Manager) branchPaused(ctx context.Context, gitDest types.ResourceReference) bool { + return m.EventRouter != nil && m.EventRouter.branchIntakePaused(ctx, gitDest) != nil +} + +// waitToReconnect waits before a stream's next session, and reports false once the stream is +// cancelled. While the GitTarget's branch has paused intake it waits for the branch to reopen +// rather than for the backoff: a reconnect would gather a snapshot, or deliver an event, that the +// branch refuses again, for as long as the outage lasts. A wake-up looks again, because the branch +// can pause again before this stream gets there. The cursor stays where the last accepted event +// left it, so the reopened stream resumes or replays exactly as after any other session end. +func (m *Manager) waitToReconnect( + ctx context.Context, + log logr.Logger, + gitDest types.ResourceReference, + key targetWatchKey, +) bool { + logged := false + for m.EventRouter != nil { + paused := m.EventRouter.branchIntakePaused(ctx, gitDest) + if paused == nil { + break + } + if !logged { + log.Info("target watch waits for its branch to reopen intake", + "gitDest", gitDest.String(), "gvr", key.GVR.String(), "namespace", key.Namespace) + logged = true + } + recheck := time.NewTimer(intakePauseRecheck) + select { + case <-ctx.Done(): + recheck.Stop() + return false + case <-paused: + case <-recheck.C: + } + recheck.Stop() + } + return sleepOrDone(ctx, targetWatchBackoff) +} + // targetStreamStateForSessionEnd grades one watch session ending, and reports whether that // grading is worth publishing at all. // @@ -717,7 +785,7 @@ func (m *Manager) pumpTargetWatchSession( replaying bool, replayStarted time.Time, ) error { - var replay []manifestanalyzer.DesiredResource + var replay replaySnapshot for { select { case <-ctx.Done(): @@ -842,14 +910,15 @@ func (m *Manager) targetWatchListAndStream( ) return fmt.Errorf("list target watch snapshot %s/%q: %w", stream.key.GVR.String(), stream.key.Namespace, err) } - desired := desiredFromList(stream.key.GVR, list) + snapshot := snapshotFromList(stream.key.GVR, list) resourceVersion := list.GetResourceVersion() - if err := m.enqueueReplayResync(ctx, log, gitDest, stream, desired, resourceVersion); err != nil { + if err := m.enqueueReplayResync(ctx, log, gitDest, stream, snapshot, resourceVersion); err != nil { return err } if err := m.recordTargetWatchCursor(ctx, gitDest, stream.key, resourceVersion); err != nil { return err } + stream.markReplayed() // Recorded HERE, not where the fallback was chosen. watch_recovery_total counts recoveries that // COMPLETED — a target whose state has been rebuilt and is now streaming — so incrementing it // at the decision would have counted an attempt that may still fail on the LIST below, under a @@ -857,7 +926,7 @@ func (m *Manager) targetWatchListAndStream( m.recordWatchRecovery(gitDest, stream.key.GVR.Group, stream.key.GVR.Resource, recoveryModeListFallback) log.Info("target watch list fallback complete", "gitDest", gitDest.String(), "gvr", stream.key.GVR.String(), "namespace", stream.key.Namespace, - "count", len(desired), "resourceVersion", resourceVersion) + "count", len(snapshot.desired), "resourceVersion", resourceVersion) m.markTargetStreamState( gitDest, stream.key.Collection(), @@ -875,7 +944,7 @@ func (m *Manager) handleTargetWatchSessionEvent( stream targetWatchStream, ev watch.Event, replaying bool, - replay *[]manifestanalyzer.DesiredResource, + replay *replaySnapshot, ) (bool, error) { if !replaying { rv, err := m.routeLiveTargetWatchEvent(ctx, log, gitDest, stream, ev) @@ -894,7 +963,8 @@ func (m *Manager) handleTargetWatchSessionEvent( if err := m.recordTargetWatchCursor(ctx, gitDest, stream.key, rv); err != nil { return true, err } - *replay = nil + stream.markReplayed() + *replay = replaySnapshot{} m.markTargetStreamState( gitDest, stream.key.Collection(), @@ -910,7 +980,7 @@ func (m *Manager) foldTargetReplayEvent( gitDest types.ResourceReference, stream targetWatchStream, ev watch.Event, - replay *[]manifestanalyzer.DesiredResource, + replay *replaySnapshot, ) (bool, string, error) { switch ev.Type { case watch.Bookmark: @@ -923,16 +993,14 @@ func (m *Manager) foldTargetReplayEvent( } log.Info("target watch replay complete", "gitDest", gitDest.String(), "gvr", stream.key.GVR.String(), "namespace", stream.key.Namespace, - "count", len(*replay), "resourceVersion", u.GetResourceVersion()) + "count", len(replay.desired), "resourceVersion", u.GetResourceVersion()) return true, u.GetResourceVersion(), nil case watch.Added, watch.Modified: u, ok := ev.Object.(*unstructured.Unstructured) if !ok { return false, "", fmt.Errorf("target replay event carried %T for %s", ev.Object, stream.key.GVR.String()) } - if desired, ok := desiredFromObject(stream.key.GVR, u); ok { - *replay = append(*replay, desired) - } + replay.add(stream.key.GVR, u) return false, "", nil case watch.Deleted: return false, "", nil @@ -948,7 +1016,7 @@ func (m *Manager) enqueueReplayResync( log logr.Logger, gitDest types.ResourceReference, stream targetWatchStream, - desired []manifestanalyzer.DesiredResource, + snapshot replaySnapshot, resourceVersion string, ) error { if m.EventRouter == nil { @@ -968,8 +1036,8 @@ func (m *Manager) enqueueReplayResync( // A retired stream's snapshot is dropped here, at the producer: see producerGate. if !stream.gate.enqueue(ctx, func() { resultCh, enqueued, err = m.EventRouter.enqueueScopedResync( - ctx, gitDest, resyncScopeForWatchKey(stream.key), stream.sourceCollection(), desired, resourceVersion, - false, stream.refreshRemote) + ctx, gitDest, resyncScopeForWatchKey(stream.key), stream.sourceCollection(), snapshot.desired, + resourceVersion, false) }) { return nil } @@ -992,9 +1060,13 @@ func (m *Manager) enqueueReplayResync( return fmt.Errorf("target replay resync for %s on %s dropped: %w", stream.key.GVR.String(), gitDest.String(), git.ErrFinalizeQueueFull) } + // The worker holds this snapshot now, ahead of every live event the stream routes after it, so + // its content is the baseline for those events. A refused or never-completed snapshot never + // gets here, and the stream keeps the baselines of what the worker did accept. + stream.changes.adopt(snapshot.baselines) log.V(1).Info("target replay resync enqueued", "gitDest", gitDest.String(), "gvr", stream.key.GVR.String(), - "resourceVersion", resourceVersion, "count", len(desired)) + "resourceVersion", resourceVersion, "count", len(snapshot.desired)) return nil } @@ -1111,17 +1183,16 @@ func (m *Manager) routeLiveTargetWatchEvent( op := operationForLiveTargetWatchEvent(ev.Type, u) event := targetWatchGitEvent(stream.key.GVR, u, op) // Stamp the producing collection and stream incarnation before the event leaves the stream. - // It is the only place both are known: downstream, a cluster-wide and a namespaced - // stream deliver the same object and the collection can no longer be recovered from it. + // It is the only place both are known: an object does not name the collection that delivered + // it (all namespaces or one, which selector), so downstream it cannot be recovered. event.SourceCollection = stream.sourceCollection() // Carry the source cluster so the git writer resolves this document's GVK->GVR // against the cluster it was watched on, never a union of all clusters. event.SourceCluster = m.clusterIDForGitTarget(gitDest) - // Drop a no-op UPDATE before it reaches the worker: a /status-only change - // sanitizes to identical git content but ships unattributed (its /status audit - // is dropped), so routing it would split an open commit window on the author - // flip. CREATE/DELETE always route and refresh/clear the dedup cache. - if m.skipUnchangedLiveUpdate(gitDest, stream.key.GVR, u, &event, op) { + // Drop an UPDATE that changes no desired state before attribution and admission: see + // desiredStateChangeFilter. Its baseline changes only below, once the worker took the event. + change, unchanged := stream.changes.check(u, &event, op) + if unchanged { recordWatchEvent(ctx, gitDest, stream.key.GVR, watchOutcomeUnchanged) log.V(1).Info("target watch skipped unchanged update (no git content change)", "gitDest", gitDest.String(), "gvr", stream.key.GVR.String(), @@ -1139,8 +1210,13 @@ func (m *Manager) routeLiveTargetWatchEvent( }) { // Counted, because a census with an unrecorded exit is not a census: the totals would // quietly stop adding up. It is not loss — a restart replays and resyncs this object. + // + // No resourceVersion either: the event never reached the worker, so the cursor must + // not pass it. Nothing in this stream resumes from it, but the cursor is durable and + // keyed by collection, and a later stream for the same collection whose first replay is + // refused would resume from it (see runTargetWatch). recordWatchEvent(ctx, gitDest, stream.key.GVR, watchOutcomeShutdown) - return rv, nil + return "", nil } if err := routeErr; err != nil { recordWatchEvent(ctx, gitDest, stream.key.GVR, watchOutcomeRouteFailed) @@ -1148,6 +1224,7 @@ func (m *Manager) routeLiveTargetWatchEvent( "gitDest", gitDest.String(), "gvr", stream.key.GVR.String(), "err", err.Error()) return rv, err } + stream.changes.accept(change) recordWatchEvent(ctx, gitDest, stream.key.GVR, watchOutcomeRouted) return rv, nil case watch.Error: @@ -1231,67 +1308,6 @@ func (m *Manager) attachAuthor( } } -// skipUnchangedLiveUpdate reports whether a live event carries no git-writable change -// from the last event routed for the same object, and maintains the dedup cache: -// - DELETE clears the entry and never skips (a removal always routes). -// - CREATE/UPDATE store the sanitized-content hash; an UPDATE whose hash equals the -// stored one is a no-op (e.g. a /status-only change) and is skipped. -// -// Only UPDATE is ever skipped — a CREATE always routes and seeds the cache, so a later -// /status-only UPDATE dedups against it. When the content cannot be hashed the event is -// routed and the cache is left untouched (fail open, never drop a real change). -func (m *Manager) skipUnchangedLiveUpdate( - gitDest types.ResourceReference, - gvr schema.GroupVersionResource, - u *unstructured.Unstructured, - event *git.Event, - op string, -) bool { - key := liveContentDedupKey(gitDest, gvr, u) - if op == string(types.OperationDelete) { - m.liveContentDedup.Delete(key) - return false - } - hash, ok := sanitizedContentHash(event) - if !ok { - return false - } - if op == string(types.OperationUpdate) { - if prev, loaded := m.liveContentDedup.Load(key); loaded { - if prevHash, isStr := prev.(string); isStr && prevHash == hash { - return true - } - } - } - m.liveContentDedup.Store(key, hash) - return false -} - -// liveContentDedupKey identifies one object within one GitTarget stream. It includes -// gitDest so the same object mirrored to two GitTargets dedups independently, and the -// uid so a delete-and-recreate (new uid) is never deduped against its predecessor. -func liveContentDedupKey( - gitDest types.ResourceReference, gvr schema.GroupVersionResource, u *unstructured.Unstructured, -) string { - return gitDest.String() + "|" + gvr.String() + "|" + string(u.GetUID()) -} - -// sanitizedContentHash hashes an event's git-writable content so two events that -// materialize identically (a spec write and a later /status update) compare equal. -// ok=false means the content cannot be hashed (nil object or marshal error); the caller -// then routes without deduping. -func sanitizedContentHash(event *git.Event) (string, bool) { - if event.Object == nil { - return "", false - } - raw, err := json.Marshal(event.Object) - if err != nil { - return "", false - } - sum := sha256.Sum256(raw) - return string(sum[:]), true -} - // targetWatchGitEvent adapts one observed object into the Event the pipeline carries. // // The resourceVersion and generation are stamped here BECAUSE this is where the unsanitized @@ -1487,22 +1503,6 @@ func bufferTargetWatchEvents(ctx context.Context, in <-chan watch.Event, out cha } } -func desiredFromList( - gvr schema.GroupVersionResource, - list *unstructured.UnstructuredList, -) []manifestanalyzer.DesiredResource { - if list == nil { - return nil - } - desired := make([]manifestanalyzer.DesiredResource, 0, len(list.Items)) - for i := range list.Items { - if item, ok := desiredFromObject(gvr, &list.Items[i]); ok { - desired = append(desired, item) - } - } - return desired -} - func targetWatchExpired(ev watch.Event) bool { if ev.Type != watch.Error || ev.Object == nil { return false diff --git a/internal/watch/target_watch_test.go b/internal/watch/target_watch_test.go index 0b7794e3..796b4399 100644 --- a/internal/watch/target_watch_test.go +++ b/internal/watch/target_watch_test.go @@ -19,7 +19,6 @@ import ( "k8s.io/apimachinery/pkg/watch" "github.com/ConfigButler/gitops-reverser/internal/git" - "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" "github.com/ConfigButler/gitops-reverser/internal/queue" "github.com/ConfigButler/gitops-reverser/internal/reconcile" "github.com/ConfigButler/gitops-reverser/internal/telemetry" @@ -49,7 +48,8 @@ func TestTargetWatchKeys_UsesOneWatchPerScope(t *testing.T) { // A cluster-wide selection and a named-namespace selection on the SAME GVR are two streams. // Collapsing them to one all-namespaces watch gave the named rule events from every namespace -// the credential could read. +// the credential could read. The resolver refuses that pair on one GitTarget (collection_overlap.go); +// the planner still never collapses it. func TestTargetWatchKeys_NamedAndClusterWideScopesStayDistinctStreams(t *testing.T) { table := WatchedTypeTable{ GitDest: types.NewResourceReference("target", "default"), @@ -129,6 +129,7 @@ func TestReplaceGitTargetWatches_ReusesUnchangedSetAndRestartsOnSpecChange(t *te }, } manager.rememberGitTargetUID(gitDest.WithUID("uid-1")) + t.Cleanup(manager.targetWatchesRunning.Wait) first := WatchedTypeTable{ GitDest: gitDest, @@ -193,9 +194,8 @@ func TestRouteLiveTargetWatchEvent_ForwardsObjectEventsAsCommitter(t *testing.T) assert.Empty(t, event.Object.GetResourceVersion(), "live events are sanitized before entering Git") } -// The source collection is stamped where it is known. Downstream it cannot be reconstructed: a -// cluster-wide and a namespaced stream deliver the same object, so the producing collection is not -// recoverable from the event itself. +// The source collection is stamped where it is known. Downstream it cannot be reconstructed: an +// object does not name the collection (all namespaces or one, which selector) that delivered it. func TestRouteLiveTargetWatchEvent_StampsTheProducingCollection(t *testing.T) { gitDest := types.NewResourceReference("target", "default") enqueuer := &recordingEnqueuer{} @@ -378,7 +378,7 @@ func TestHandleTargetWatchSessionEvent_CompletesReplayWithoutRouter(t *testing.T manager := &Manager{} gitDest := types.NewResourceReference("target", "default") key := targetWatchKey{GVR: configmapsGVR, Namespace: "apps"} - var replay []manifestanalyzer.DesiredResource + var replay replaySnapshot replaying, err := manager.handleTargetWatchSessionEvent( context.Background(), @@ -391,7 +391,7 @@ func TestHandleTargetWatchSessionEvent_CompletesReplayWithoutRouter(t *testing.T ) require.NoError(t, err) assert.True(t, replaying) - require.Len(t, replay, 1) + require.Len(t, replay.desired, 1) bookmark := &unstructured.Unstructured{} bookmark.SetResourceVersion("11") @@ -407,7 +407,7 @@ func TestHandleTargetWatchSessionEvent_CompletesReplayWithoutRouter(t *testing.T ) require.NoError(t, err) assert.False(t, replaying) - assert.Nil(t, replay) + assert.Empty(t, replay.desired) } func TestTargetWatchReplayAndStream_ReturnsWhenContextCancels(t *testing.T) { @@ -610,19 +610,19 @@ func TestFoldTargetReplayEvent_AccumulatesUntilInitialEventsBookmark(t *testing. manager := &Manager{} gitDest := types.NewResourceReference("target", "default") key := targetWatchKey{GVR: configmapsGVR, Namespace: "apps"} - var desired []manifestanalyzer.DesiredResource + var replay replaySnapshot done, rv, err := manager.foldTargetReplayEvent( logr.Discard(), gitDest, testStream(key), watch.Event{Type: watch.Added, Object: configMapObject("10")}, - &desired, + &replay, ) require.NoError(t, err) assert.False(t, done) assert.Empty(t, rv) - require.Len(t, desired, 1) + require.Len(t, replay.desired, 1) bookmark := &unstructured.Unstructured{} bookmark.SetResourceVersion("11") @@ -632,7 +632,7 @@ func TestFoldTargetReplayEvent_AccumulatesUntilInitialEventsBookmark(t *testing. gitDest, testStream(key), watch.Event{Type: watch.Bookmark, Object: bookmark}, - &desired, + &replay, ) require.NoError(t, err) assert.True(t, done) @@ -948,6 +948,9 @@ func planTestManager(t *testing.T, gitDest types.ResourceReference) (*Manager, c }, } manager.rememberGitTargetUID(gitDest.WithUID("uid-1")) + // A test cancels its streams on return; wait for their goroutines too, or their last session + // still records metrics while the next test resets the global exporter. + t.Cleanup(manager.targetWatchesRunning.Wait) return manager, opened } @@ -1122,7 +1125,7 @@ func TestRouteLiveTargetWatchEvent_ACancelledStreamStopsEnqueuing(t *testing.T) ) require.NoError(t, err) - assert.Equal(t, "12", rv, "the cursor still advances; only the enqueue is dropped") + assert.Empty(t, rv, "an event that was never enqueued must not advance the cursor") assert.Empty(t, enqueuer.events) } diff --git a/internal/watch/terminating_snapshot_test.go b/internal/watch/terminating_snapshot_test.go index 5b201e8a..e275a1b1 100644 --- a/internal/watch/terminating_snapshot_test.go +++ b/internal/watch/terminating_snapshot_test.go @@ -40,21 +40,22 @@ func TestFoldTargetReplayEvent_SkipsTerminatingObjects(t *testing.T) { t.Run(string(eventType), func(t *testing.T) { manager := &Manager{} key := targetWatchKey{GVR: configmapsGVR, Namespace: "apps"} - var desired []manifestanalyzer.DesiredResource + var replay replaySnapshot done, rv, err := manager.foldTargetReplayEvent( logr.Discard(), types.NewResourceReference("target", "default"), testStream(key), watch.Event{Type: eventType, Object: terminatingConfigMapObject("7")}, - &desired, + &replay, ) require.NoError(t, err) assert.False(t, done) assert.Empty(t, rv) - assert.Empty(t, desired, + assert.Empty(t, replay.desired, "a Terminating object must not be folded into the replay's desired set") + assert.Empty(t, replay.baselines, "nor given a baseline Git does not hold") }) } } @@ -70,7 +71,7 @@ func TestDesiredFromList_ExcludesTerminatingItems(t *testing.T) { list := &unstructured.UnstructuredList{Items: []unstructured.Unstructured{*live, *terminating}} - desired := desiredFromList(configmapsGVR, list) + desired := snapshotFromList(configmapsGVR, list).desired require.Len(t, desired, 1, "only the live object belongs in the desired set") assert.Equal(t, "still-here", desired[0].Resource.Name) diff --git a/internal/watch/watched_type_resolver.go b/internal/watch/watched_type_resolver.go index da1c572c..acdadd55 100644 --- a/internal/watch/watched_type_resolver.go +++ b/internal/watch/watched_type_resolver.go @@ -46,8 +46,8 @@ type watchedTypeStore struct { // the previous cluster's GVR table. clusterFP uint64 resolved bool - // refusals is the last resolution's selector-conflict refusals, keyed by rule; see - // refuseSelectorConflicts. It is published with the tables it was decided for, so a rule's + // refusals is the last resolution's collection-overlap refusals, keyed by rule; see + // refuseCollectionOverlaps. It is published with the tables it was decided for, so a rule's // status never reports a refusal the running plan does not reflect. refusals map[selectingRule]string } @@ -267,8 +267,8 @@ func (m *Manager) clusterMappingFingerprint() uint64 { // an empty table so a transient discovery gap does not look like rule removal. The per-target // table is stamped with its own cluster's registry generation. // -// A rule whose collections overlap another rule's with a different object selector contributes -// nothing to its target's table, and is returned among the refusals instead. +// A rule whose collections overlap a different collection of its target contributes nothing to +// that target's table, and is returned among the refusals instead. func (m *Manager) resolveWatchedTypeTables() (map[string]WatchedTypeTable, map[selectingRule]string) { if m.RuleStore == nil { return map[string]WatchedTypeTable{}, nil @@ -304,7 +304,7 @@ func (m *Manager) resolveWatchedTypeTables() (map[string]WatchedTypeTable, map[s tables := make(map[string]WatchedTypeTable, len(byTarget)) var refusals map[selectingRule]string for key, ts := range byTarget { - admitted, refused := refuseSelectorConflicts(ts.selections) + admitted, refused := refuseCollectionOverlaps(ts.selections) for rule, reason := range refused { if refusals == nil { refusals = map[selectingRule]string{} @@ -354,10 +354,11 @@ func (m *Manager) collectWatchRuleSelections( // metadata.namespace. So changing it never moves anything in Git. // // A WILDCARD item emits the empty key deliberately: "*" is one cluster-wide list - // and watch, which for a namespaced GVR is the all-namespaces collection. That - // collection is a PEER of any named-namespace collection on the same type, never a - // replacement — collapsing the two once widened a named rule's stream (CollectionKey - // in internal/types/collection.go). An omitted item still resolves to a concrete name. + // and watch, which for a namespaced GVR is the all-namespaces collection. It is never + // collapsed into a named-namespace collection on the same type — that once widened a + // named rule's stream (CollectionKey in internal/types/collection.go) — and one + // GitTarget cannot hold both: refuseCollectionOverlaps refuses the newer rule. An + // omitted item still resolves to a concrete name. for _, namespace := range rr.SourceNamespaces { ts.selections = append(ts.selections, watchSelection{ record: rec, namespace: namespace, labelSelector: rr.LabelSelector, rule: selecting, @@ -551,7 +552,7 @@ func (m *Manager) rulesFingerprint() uint64 { // watchRuleFingerprint hashes everything about a compiled WatchRule that can change what it // watches, including the rule's identity and age: they decide which of two conflicting rules keeps -// its collections (refuseSelectorConflicts). Each item's src= component MUST be that item's RESOLVED source-namespace SET, not the +// its collections (refuseCollectionOverlaps). Each item's src= component MUST be that item's RESOLVED source-namespace SET, not the // WatchRule object's own namespace and not the requested value. // // It is now derivable from the rule spec alone: a wildcard resolves to the one cluster-wide collection @@ -591,24 +592,23 @@ func clusterWatchRuleFingerprint(rule rulestore.CompiledClusterRule) string { return b.String() } -// ObjectSelectorConflictForWatchRule reports whether the rule is refused because one of its -// collections overlaps another rule's on the same GitTarget with a different object selector, and -// why. It reads the resolution the running plan was built from, refreshing it first. -func (m *Manager) ObjectSelectorConflictForWatchRule(rule configv1alpha3.WatchRule) (bool, string) { - return m.objectSelectorConflict(selectingRule{ +// CollectionOverlapForWatchRule reports whether the rule is refused because one of its +// collections overlaps a different collection on the same GitTarget, and why. It reads the resolution the running plan was built from, refreshing it first. +func (m *Manager) CollectionOverlapForWatchRule(rule configv1alpha3.WatchRule) (bool, string) { + return m.overlapRefusal(selectingRule{ kind: ruleKindWatchRule, namespace: rule.Namespace, name: rule.Name, createdAt: rule.CreationTimestamp, }) } -// ObjectSelectorConflictForClusterWatchRule is ObjectSelectorConflictForWatchRule for a +// CollectionOverlapForClusterWatchRule is CollectionOverlapForWatchRule for a // ClusterWatchRule. -func (m *Manager) ObjectSelectorConflictForClusterWatchRule(rule configv1alpha3.ClusterWatchRule) (bool, string) { - return m.objectSelectorConflict(selectingRule{ +func (m *Manager) CollectionOverlapForClusterWatchRule(rule configv1alpha3.ClusterWatchRule) (bool, string) { + return m.overlapRefusal(selectingRule{ kind: ruleKindClusterWatchRule, name: rule.Name, createdAt: rule.CreationTimestamp, }) } -func (m *Manager) objectSelectorConflict(rule selectingRule) (bool, string) { +func (m *Manager) overlapRefusal(rule selectingRule) (bool, string) { m.refreshWatchedTypeTables() m.watchedTypes.mu.Lock() defer m.watchedTypes.mu.Unlock() diff --git a/internal/watch/watched_type_table.go b/internal/watch/watched_type_table.go index 468a2f47..39f83d4b 100644 --- a/internal/watch/watched_type_table.go +++ b/internal/watch/watched_type_table.go @@ -30,7 +30,7 @@ type WatchedType struct { // cluster-wide collection: a cluster-scoped resource, or a namespaced resource a WatchRule // follows across every namespace. One selector per scope is enough because two collections on // one type and namespace overlap, and the resolver refuses a second selector over an - // overlapping collection (see refuseSelectorConflicts). + // overlapping collection (see refuseCollectionOverlaps). NamespaceScopes map[string]string } @@ -101,8 +101,8 @@ type watchedTypeAccum struct { // buildWatchedTypeTable folds a GitTarget's selected followable records into its // watched-type table, deduplicating each record's namespace scopes. Identity -// and followability are already settled by the registry, and selector conflicts by -// refuseSelectorConflicts before this runs, so this is a pure fold with no catalog lookup and no +// and followability are already settled by the registry, and collection overlaps by +// refuseCollectionOverlaps before this runs, so this is a pure fold with no catalog lookup and no // conflict decision. func buildWatchedTypeTable( gitDest types.ResourceReference, diff --git a/test/e2e/object_selector_e2e_test.go b/test/e2e/object_selector_e2e_test.go index 713c30f7..851c454b 100644 --- a/test/e2e/object_selector_e2e_test.go +++ b/test/e2e/object_selector_e2e_test.go @@ -264,7 +264,7 @@ metadata: const conflicting = "selector-conflicting-rule" applySelectorRule(conflicting, alwaysTarget, selectedBy("other")) verifyResourceCondition("watchrule", conflicting, testNs, "ResourcesResolved", "False", - "ObjectSelectorConflict", "") + "CollectionOverlap", "") verifyResourceCondition("watchrule", alwaysRule, testNs, "ResourcesResolved", "True", "", "") By("the older rule's mirror keeps following its own selection") diff --git a/test/e2e/source_namespace_e2e_test.go b/test/e2e/source_namespace_e2e_test.go index 24993099..94983b78 100644 --- a/test/e2e/source_namespace_e2e_test.go +++ b/test/e2e/source_namespace_e2e_test.go @@ -35,11 +35,13 @@ var _ = Describe("WatchRule source namespace", Label("manager"), Ordered, func() delegatingCP = "srcns-delegating" nonDelegatingCP = "srcns-non-delegating" grantedTarget = "srcns-granted" + wildcardTarget = "srcns-wildcard" refusedTarget = "srcns-refused" grantedRule = "srcns-granted-rule" refusedRule = "srcns-refused-rule" wildcardRule = "srcns-wildcard-rule" grantedPath = "e2e/srcns-granted" + wildcardPath = "e2e/srcns-wildcard" refusedPath = "e2e/srcns-refused" ) @@ -90,18 +92,23 @@ var _ = Describe("WatchRule source namespace", Label("manager"), Ordered, func() Expect(applyInClusterClusterProvider(nonDelegatingCP, configNS, false)).Error(). NotTo(HaveOccurred(), "failed to apply non-delegating ClusterProvider") - By("creating one GitTarget behind the delegating provider and one behind the refusing one") + By("creating two GitTargets behind the delegating provider and one behind the refusing one") // The targets are identical. The only thing that differs is which ClusterProvider they // mirror through, which is now the whole of the source-namespace policy: there is no - // per-target allow-list any more. + // per-target allow-list any more. The wildcard gets a target of its own because one + // GitTarget cannot watch a type both in all namespaces and in a named one. Expect(applyGitTargetForClusterProvider( configNS, grantedTarget, providerName, grantedPath, delegatingCP)).Error(). NotTo(HaveOccurred(), "failed to apply granted GitTarget") + Expect(applyGitTargetForClusterProvider( + configNS, wildcardTarget, providerName, wildcardPath, delegatingCP)).Error(). + NotTo(HaveOccurred(), "failed to apply wildcard GitTarget") Expect(applyGitTargetForClusterProvider( configNS, refusedTarget, providerName, refusedPath, nonDelegatingCP)).Error(). NotTo(HaveOccurred(), "failed to apply refused GitTarget") verifyResourceCondition("gittarget", grantedTarget, configNS, "Validated", "True", "Succeeded", "") + verifyResourceCondition("gittarget", wildcardTarget, configNS, "Validated", "True", "Succeeded", "") verifyResourceCondition("gittarget", refusedTarget, configNS, "Validated", "True", "Succeeded", "") }) @@ -155,14 +162,25 @@ var _ = Describe("WatchRule source namespace", Label("manager"), Ordered, func() }) It("reaches every namespace the credential can read through one cluster-wide watch", func() { - By("creating a WatchRule whose item asks for every namespace") + By("refusing a wildcard beside the named source namespace on the same target") + // The granted target already watches ConfigMaps in sourceNS. All namespaces would deliver + // those objects a second time, so the newer rule is refused as a whole, naming both scopes. + Expect(applyWildcardConfigMapWatchRule( + wildcardRule, configNS, grantedTarget)).Error(). + NotTo(HaveOccurred(), "failed to apply overlapping wildcard WatchRule") + verifyResourceCondition("watchrule", wildcardRule, configNS, + "ResourcesResolved", "False", "CollectionOverlap", "configmaps in all namespaces") + verifyResourceCondition("watchrule", grantedRule, configNS, "ResourcesResolved", "True", "", "") + + By("pointing the WatchRule at a target of its own") // ConfigMaps only, deliberately. A cluster-wide "*" mirrors everything the credential can // read, and adding secrets here would file every service-account token in the cluster into // the fixture repository. That the widening reaches that far is the POINT of the change; // one type is enough to observe it. Expect(applyWildcardConfigMapWatchRule( - wildcardRule, configNS, grantedTarget)).Error(). + wildcardRule, configNS, wildcardTarget)).Error(). NotTo(HaveOccurred(), "failed to apply wildcard WatchRule") + verifyResourceCondition("watchrule", wildcardRule, configNS, "ResourcesResolved", "True", "", "") By("asserting the wildcard is authorized") // The gate is what this spec is about. Whole-target Ready is deliberately NOT asserted: a @@ -187,17 +205,17 @@ var _ = Describe("WatchRule source namespace", Label("manager"), Ordered, func() Expect(err).NotTo(HaveOccurred()) By("asserting both arrive, each under its own namespace's folder") - wantWildcard := path.Join(grantedPath, fmt.Sprintf("%s/configmaps/%s.yaml", wildcardNS, wildcardCM)) - wantOutside := path.Join(grantedPath, fmt.Sprintf("%s/configmaps/%s.yaml", outsideNS, outsideCM)) + wantWildcard := path.Join(wildcardPath, fmt.Sprintf("%s/configmaps/%s.yaml", wildcardNS, wildcardCM)) + wantOutside := path.Join(wildcardPath, fmt.Sprintf("%s/configmaps/%s.yaml", outsideNS, outsideCM)) Eventually(func(g Gomega) { pullLatestRepoState(g, srcnsRepo.CheckoutDir) g.Expect(filepath.Join(srcnsRepo.CheckoutDir, wantWildcard)).To(BeAnExistingFile(), `"*" must reach %q, which no rule item names. Recent commits:\n%s`, - wildcardNS, recentCommitDiagnostics(srcnsRepo.CheckoutDir, grantedPath)) + wildcardNS, recentCommitDiagnostics(srcnsRepo.CheckoutDir, wildcardPath)) g.Expect(filepath.Join(srcnsRepo.CheckoutDir, wantOutside)).To(BeAnExistingFile(), `"*" is bounded by the source credential's RBAC and nothing else, so %q arrives `+ "too. Recent commits:\n%s", - outsideNS, recentCommitDiagnostics(srcnsRepo.CheckoutDir, grantedPath)) + outsideNS, recentCommitDiagnostics(srcnsRepo.CheckoutDir, wildcardPath)) }, 3*time.Minute, 5*time.Second).Should(Succeed()) })