From 9cb983acee9efd104776b6684a221a5f06061b14 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 15:29:53 -0700 Subject: [PATCH 1/4] refactor(taskfile): adopt lib v2.6.0 - The proton engine names its image: drop IMAGE from the toolbox include (C2). - The toolbox defines no report-engine: drop it from the excludes (C6). - check.yml is the caller of lib's check workflow, with offline: true (C7). - The vars input of sync.yml reaches every task, and LIST_FLOOR and OWNERS there change prune: say so (C26). - github runs daily, and DESC takes the owners from OWNERS (C27). The pins move to v2.6.0, with render.txt, once the tag exists. Until then lib's check workflow at the v2.2.0 pin has no offline input. --- .github/workflows/check.yml | 11 ++--------- .github/workflows/sync.yml | 2 +- Taskfile.yml | 7 +++---- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index e29b496..09c15db 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,14 +1,7 @@ -# One runner, one image pull: the render diff and the offline check. name: check on: {pull_request: {}} permissions: {contents: read} jobs: check: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: katoptra/lib/.github/actions/toolbox@fe8162386f612ddb4e5c886632fa7a8d74c54d9f # v2.2.0 - with: {op: 'false'} - - run: task check - - run: task run -- task offline + uses: katoptra/lib/.github/workflows/check.yml@fe8162386f612ddb4e5c886632fa7a8d74c54d9f # v2.2.0 + with: {offline: true} diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 41d1b2f..5a834f2 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -3,7 +3,7 @@ on: workflow_dispatch: inputs: vars: - description: 'KEY=value pairs for the pipeline; none are read today' + description: 'KEY=value pairs for all the tasks. LIST_FLOOR and OWNERS here change prune.' type: string default: '' permissions: diff --git a/Taskfile.yml b/Taskfile.yml index 56b1c42..e46f8d5 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,4 +1,4 @@ -# github: a nightly mirror of every repository under OWNERS into Proton Drive. +# github: a daily mirror of every repository under OWNERS into Proton Drive. # # Each repository becomes one git bundle, /.bundle under the destination # folder: a mirror clone's every ref in one file that `git clone name.bundle` restores. @@ -25,11 +25,10 @@ includes: toolbox: taskfile: https://raw.githubusercontent.com/katoptra/lib/v2/toolbox.yml flatten: true - excludes: [report-engine, report-mirror] + excludes: [report-mirror] vars: NAME: github - DESC: a nightly mirror of every repository under jshvn and katoptra into Proton Drive - IMAGE: ghcr.io/katoptra/toolbox:proton-v2 + DESC: a daily mirror of every repository under {{.OWNERS}} into Proton Drive MENU: >- "" "${c}write -- changes Proton Drive${r}" From 2880a630f1f3d71bc99f7e2627e40be18393b700 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 15:35:29 -0700 Subject: [PATCH 2/4] docs(comments): rewrite the Taskfile, workflow and op.env comments in STE - The Taskfile header takes the shared mirror shape (C35): the upstream, the folder, the cadence and the bundle; the includes and the README of lib; the items that are only this mirror's; the secrets by name. The bundle paragraph stays after it. - Every Taskfile comment, the actions: write comment of sync.yml (now on its own lines) and the op.env comments are in ASD-STE100 (C78). No fact goes, and no op:// line changes. The LIST_FLOOR comment adds that the vars input of the sync workflow can change the value (C26). --- .github/workflows/sync.yml | 5 +++- Taskfile.yml | 55 ++++++++++++++++++++++++-------------- op.env | 5 ++-- 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 5a834f2..41560ea 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -8,7 +8,10 @@ on: default: '' permissions: contents: read - actions: write # the called workflow chains a run that left .run/chain; this one never does, but a called workflow cannot raise this + # The sync workflow of lib starts the next run if a run writes .run/chain. The runs of + # this mirror do not write that file. But the workflow of lib cannot have more + # permissions than this workflow gives it. + actions: write concurrency: {group: sync, cancel-in-progress: false} jobs: sync: diff --git a/Taskfile.yml b/Taskfile.yml index e46f8d5..8d3c1f3 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,21 +1,33 @@ -# github: a daily mirror of every repository under OWNERS into Proton Drive. +# This mirror keeps a copy of each GitHub repository of the owners in OWNERS, in one +# Proton Drive folder. Each copy is one git bundle, and the mirror updates the copies +# daily. katoptra/lib supplies the toolbox and the proton engine, and this file includes +# the two at v2 with `flatten: true`. The README of lib gives the items that each of the +# two supplies. The other items in this file are only for this mirror: +# - OWNERS: the owners of the repositories +# - LIST_FLOOR: the minimum number of repositories in a listing +# - list: the verb that gets the repositories of each owner from the GitHub API +# - stage and prune: the two hooks that the engine lets a mirror write +# - report-mirror: the rows of the mirror in the run summary +# - offline: the check that two bundles of one repository have the same bytes, a +# condition that is necessary for this mirror # -# Each repository becomes one git bundle, /.bundle under the destination -# folder: a mirror clone's every ref in one file that `git clone name.bundle` restores. -# A bundle's bytes are the same on every night the repository did not change, so the -# proton engine's upload skips it, and a night it changed becomes a new revision in -# Proton's version history. Nothing is kept between runs but the engine's CLI session. +# op.env contains the names of the secrets. `op run` gets their values from the vault, +# and each secret goes into the image by its name. # -# The toolbox and the proton engine are katoptra/lib's, included below at v2 and -# flattened into this file. What is here is the mirror's own: which owners, the floor -# under a listing, the two hooks the engine leaves to a mirror (stage, prune), its rows -# of the report, and the offline check of the assumption everything rests on. +# A bundle is one file with all the refs of a mirror clone, at /.bundle in +# the destination folder. `git clone name.bundle` gets the repository back from it. If a +# repository does not change, its bundle has the same bytes in each run. Thus, the upload +# of the proton engine does not send it again. If a repository changes, its bundle +# becomes a new revision in the version history of Proton. Between runs, the mirror keeps +# only the CLI session of the engine. version: '3' vars: OWNERS: jshvn katoptra - # A listing under this many repositories is a broken token or a broken API, never an - # account that emptied itself; stage refuses it, so prune can never trash on one. + # If a listing has less than this number of repositories, a token or the API does not + # operate correctly. An account that removed its repositories is not the cause. stage + # rejects such a listing. Thus, prune cannot move bundles to the trash because of it. + # The vars input of the sync workflow can change this value. LIST_FLOOR: 20 API: https://api.github.com CURL: curl -fsS --connect-timeout 15 --max-time 120 --retry 3 --retry-connrefused @@ -49,8 +61,10 @@ tasks: set: [pipefail] cmds: - mkdir -p {{.RUN}} && rm -f {{.RUN}}/repos.txt && touch {{.RUN}}/repos.txt - # A user token lists that user's own repositories and an org token the org's; the - # endpoint follows the owner's kind. Each is read a page at a time until a short one. + # With a user token, the API gives the repositories of that user. With an + # organization token, the API gives the repositories of the organization. The verb + # selects the endpoint for the type of the owner. It reads the list of each owner one + # page at a time, until a page has less than 100 repositories. - for: {var: OWNERS} cmd: | o={{.ITEM}}; tok=$(printenv "MIRROR_GITHUB_TOKEN_$(printf %s "$o" | tr 'a-z-' 'A-Z_')") || { echo "list: no token for $o in the environment" >&2; exit 1; } @@ -73,17 +87,18 @@ tasks: test "$n" -ge {{.LIST_FLOOR}} || { echo "list: $n repositories is under the floor of {{.LIST_FLOOR}}; refusing to take a short listing for the account" >&2; exit 1; } echo "list: $n repositories under {{.OWNERS}}" - # ---- the engine's hooks ---- + # ---- the hooks of the engine ---- stage: desc: list, then clone each repository as a mirror and bundle it into STAGING//.bundle cmds: - {task: list} - rm -rf {{.STAGING}} {{.RUN}}/work && mkdir -p {{.STAGING}} {{.RUN}}/work - # The token reaches git as a header through its environment, never on a command line, - # and git's stderr goes to a file: a clone error names the repository, and the log is - # public. A repository with no refs yet has nothing to bundle. Bundles are - # byte-for-byte reproducible for an unchanged repository (offline checks this), which - # is what lets the engine skip them. + # The token goes to git as a header in the environment of git, not on a command + # line. A clone error shows the name of the repository, and all persons can read the + # run log. Thus, the stderr of git goes to a file. stage makes no bundle for an empty + # repository, which has no refs. If a repository does not change, its bundle has the + # same bytes each time (offline examines this). Thus, the engine does not upload the + # bundle again. - | i=0 while IFS=/ read -r o name; do diff --git a/op.env b/op.env index d5ae549..618f0a8 100644 --- a/op.env +++ b/op.env @@ -1,5 +1,6 @@ -# op:// references only; `op run --env-file=op.env` resolves them at run time. -# One vault, addressed by UUID so a rename cannot break it; item github. See katoptra/lib, Secrets. +# This file contains only op:// references. `op run --env-file=op.env` gets their values +# at run time. The secrets are in one vault, addressed by UUID. Thus, the references stay +# correct when the vault gets a new name. The item is github. Refer to katoptra/lib, Secrets. MIRROR_GITHUB_TOKEN_JSHVN=op://y6y6b6l2zjbpv7szc5ym5sprne/github/github/token_jshvn MIRROR_GITHUB_TOKEN_KATOPTRA=op://y6y6b6l2zjbpv7szc5ym5sprne/github/github/token_katoptra MIRROR_PROTON_DESTINATION=op://y6y6b6l2zjbpv7szc5ym5sprne/github/proton/destination From b1dc4aa569331e3fd193f144161946528fdbfc2a Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 16:07:33 -0700 Subject: [PATCH 3/4] docs(comments): use approved words and say where stderr goes - The secrets go into the image "with" their names: BY is only a mathematical function word in STE (review m1). - The list comment says "listing", the word the header and the LIST_FLOOR comment use for the same thing (review m2). - The stage comment states first that all persons can read the run log, then where each item goes: the token into the environment of git, not a command line, and the stderr of git into .run/git.err, not the run log (review m3). --- Taskfile.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 8d3c1f3..a19fd93 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -12,7 +12,7 @@ # condition that is necessary for this mirror # # op.env contains the names of the secrets. `op run` gets their values from the vault, -# and each secret goes into the image by its name. +# and each secret goes into the image with its name. # # A bundle is one file with all the refs of a mirror clone, at /.bundle in # the destination folder. `git clone name.bundle` gets the repository back from it. If a @@ -63,8 +63,8 @@ tasks: - mkdir -p {{.RUN}} && rm -f {{.RUN}}/repos.txt && touch {{.RUN}}/repos.txt # With a user token, the API gives the repositories of that user. With an # organization token, the API gives the repositories of the organization. The verb - # selects the endpoint for the type of the owner. It reads the list of each owner one - # page at a time, until a page has less than 100 repositories. + # selects the endpoint for the type of the owner. It reads the listing of each owner + # one page at a time, until a page has less than 100 repositories. - for: {var: OWNERS} cmd: | o={{.ITEM}}; tok=$(printenv "MIRROR_GITHUB_TOKEN_$(printf %s "$o" | tr 'a-z-' 'A-Z_')") || { echo "list: no token for $o in the environment" >&2; exit 1; } @@ -93,12 +93,12 @@ tasks: cmds: - {task: list} - rm -rf {{.STAGING}} {{.RUN}}/work && mkdir -p {{.STAGING}} {{.RUN}}/work - # The token goes to git as a header in the environment of git, not on a command - # line. A clone error shows the name of the repository, and all persons can read the - # run log. Thus, the stderr of git goes to a file. stage makes no bundle for an empty - # repository, which has no refs. If a repository does not change, its bundle has the - # same bytes each time (offline examines this). Thus, the engine does not upload the - # bundle again. + # All persons can read the run log. Thus, the token goes to git as a header in the + # environment of git, not on a command line. The stderr of git goes to .run/git.err, + # not to the run log, because a clone error shows the name of the repository. stage + # makes no bundle for an empty repository, which has no refs. If a repository does + # not change, its bundle has the same bytes each time (offline examines this). Thus, + # the engine does not upload the bundle again. - | i=0 while IFS=/ read -r o name; do From 7c7f71da44cef99d93fd5cec2e6393140e4ccd48 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 16:49:59 -0700 Subject: [PATCH 4/4] chore(render): accept lib v2.6.0 and pin its workflows --- .github/workflows/check.yml | 2 +- .github/workflows/sync.yml | 2 +- render.txt | 15 ++++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 09c15db..0c650f2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -3,5 +3,5 @@ on: {pull_request: {}} permissions: {contents: read} jobs: check: - uses: katoptra/lib/.github/workflows/check.yml@fe8162386f612ddb4e5c886632fa7a8d74c54d9f # v2.2.0 + uses: katoptra/lib/.github/workflows/check.yml@36d75e6b21473400b5b15003025d9a2e81d4eeea # v2.6.0 with: {offline: true} diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 41560ea..51d6f37 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -15,6 +15,6 @@ permissions: concurrency: {group: sync, cancel-in-progress: false} jobs: sync: - uses: katoptra/lib/.github/workflows/sync.yml@fe8162386f612ddb4e5c886632fa7a8d74c54d9f # v2.2.0 + uses: katoptra/lib/.github/workflows/sync.yml@36d75e6b21473400b5b15003025d9a2e81d4eeea # v2.6.0 with: {vars: '${{ inputs.vars }}', timeout-minutes: 90} secrets: inherit diff --git a/render.txt b/render.txt index 1230802..294c5e3 100644 --- a/render.txt +++ b/render.txt @@ -16,7 +16,7 @@ rm -f "$k"; exit $rc task: [session] tar -xf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json && chmod 600 /work/.run/session/*.json task: [session] rm -f /work/.run/session.tar /work/.run/session.tar.age task: [session] sha256sum /work/.run/session/auth-session.json > /work/.run/session.sha -task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem list -j "$(dirname "$MIRROR_PROTON_DESTINATION")" > /work/.run/parent.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem list -j "$(dirname "$MIRROR_PROTON_DESTINATION")" > /work/.run/parent.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc task: [session-push] echo "session: token rotated, sealing it back to the bucket" task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; } @@ -106,7 +106,7 @@ done < /work/.run/repos.txt task: [stage] find /work/staging -type f -printf '%P\t%s\n' | sort > /work/.run/staged.txt task: [stage] echo "stage: $(wc -l < /work/.run/staged.txt) bundles, $(awk -F'\t' '{ s += $2 } END { printf "%.1f", s / 1e6 }' /work/.run/staged.txt) MB" task: [upload] test -n "$(ls -A /work/staging 2>/dev/null)" || { echo "upload: staging is empty; stage put nothing there" >&2; exit 1; } -task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem upload -f create-new-revision -d merge -t --json /work/staging/* "$MIRROR_PROTON_DESTINATION" > /work/.run/upload.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem upload -f create-new-revision -d merge -t --json /work/staging/* "$MIRROR_PROTON_DESTINATION" > /work/.run/upload.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc task: [session-push] echo "session: token rotated, sealing it back to the bucket" task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; } @@ -130,8 +130,9 @@ task: [confirm] files=$(find /work/staging -type f | wc -l); folders=$(find /wor python3 - "$files" "$folders" /work/.run/upload.json /work/.run/confirm.txt <<'PY' import json, sys files, folders = int(sys.argv[1]), int(sys.argv[2]) -# The CLI writes progress and its summary as one JSON object per line; the last -# line carrying transferredItems is the summary. +# While the upload continues, the CLI writes one JSON object on each line. Its +# summary is also one JSON object on a line. The last line with transferredItems +# is the summary. summary = None for line in open(sys.argv[3]): try: @@ -150,7 +151,7 @@ print("confirm: " + verdict) sys.exit(0 if ok else 1) PY -task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem list -j "$MIRROR_PROTON_DESTINATION/jshvn" > /work/.run/proton-jshvn.json || echo "[]" > /work/.run/proton-jshvn.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem list -j "$MIRROR_PROTON_DESTINATION/jshvn" > /work/.run/proton-jshvn.json || echo "[]" > /work/.run/proton-jshvn.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc task: [session-push] echo "session: token rotated, sealing it back to the bucket" task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; } @@ -170,7 +171,7 @@ task: [pd] rc=$(cat /work/.run/pd.rc) case " 0 " in *" $rc "*) exit 0 ;; esac echo "proton-drive exited $rc; its stderr, $(wc -l < /work/.run/pd.err) lines, is in .run/pd.err" >&2; exit $rc -task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem list -j "$MIRROR_PROTON_DESTINATION/katoptra" > /work/.run/proton-katoptra.json || echo "[]" > /work/.run/proton-katoptra.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem list -j "$MIRROR_PROTON_DESTINATION/katoptra" > /work/.run/proton-katoptra.json || echo "[]" > /work/.run/proton-katoptra.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc task: [session-push] echo "session: token rotated, sealing it back to the bucket" task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; } @@ -206,7 +207,7 @@ for owner in sys.argv[1:]: PY task: [prune] echo "prune: $(wc -l < /work/.run/prune.txt) bundles to trash" -task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem trash 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem trash 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc task: [session-push] echo "session: token rotated, sealing it back to the bucket" task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; }