diff --git a/.gitea/workflows/backend-quality-gate.yml b/.gitea/workflows/backend-quality-gate.yml index 71ae55c1..ea5adb05 100644 --- a/.gitea/workflows/backend-quality-gate.yml +++ b/.gitea/workflows/backend-quality-gate.yml @@ -65,25 +65,72 @@ jobs: NODE_TOOL_IMAGE: node:22-bookworm-slim steps: - name: Checkout exact Gitea revision + env: + MIRROR_PATH: /root/.cache/jyotisha-mirror.git run: | set -euo pipefail [[ "$GITEA_SHA" =~ ^[0-9a-f]{40}$ ]] git init . git remote remove origin 2>/dev/null || true git remote add origin https://git.copse.top/root/Jyotisha.git + bounded_git() { + timeout 300 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 "$@" + } fetch_succeeded=false - for attempt in 1 2 3; do - if timeout 300 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 \ - fetch --depth=1 --no-tags origin "$GITEA_SHA"; then - fetch_succeeded=true - break + # act_runner's hostexecutor discards this workspace after every run but + # keeps the host filesystem, so a bare mirror at MIRROR_PATH amortises + # the 105 MB tree across runs; the exact SHA is then fetched from local + # disk in seconds instead of 4-8 minutes per job over the WAN. The + # mirror is an accelerator, never a dependency: every failure below + # falls through to the bounded remote fetch that has always been used. + sync_mirror() { + if [ -d "$MIRROR_PATH" ] && [ "$(git -C "$MIRROR_PATH" rev-parse --is-bare-repository 2>/dev/null)" = true ]; then + # We hold the host lock, so any git lock file left by a cancelled job is stale. + find "$MIRROR_PATH" -name '*.lock' -type f -delete 2>/dev/null || true + if ! git -C "$MIRROR_PATH" cat-file -e "$GITEA_SHA^{commit}" 2>/dev/null; then + bounded_git -C "$MIRROR_PATH" fetch --prune origin || return 1 + fi + else + rm -rf "$MIRROR_PATH" + timeout 900 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 \ + clone --quiet --mirror https://git.copse.top/root/Jyotisha.git "$MIRROR_PATH" || { rm -rf "$MIRROR_PATH"; return 1; } fi - if [ "$attempt" -eq 3 ]; then - echo "exact staging gate checkout failed after $attempt bounded attempts" >&2 - exit 1 + git -C "$MIRROR_PATH" cat-file -e "$GITEA_SHA^{commit}" + } + if mkdir -p "$(dirname "$MIRROR_PATH")" 2>/dev/null && exec 9>"$MIRROR_PATH.lock" 2>/dev/null; then + if flock -w 900 9; then + if sync_mirror; then + git remote set-url origin "$MIRROR_PATH" + if timeout 300 git fetch --no-tags origin "$GITEA_SHA"; then + fetch_succeeded=true + else + echo "mirror fetch of $GITEA_SHA failed; falling back to remote fetch" >&2 + fi + git remote set-url origin https://git.copse.top/root/Jyotisha.git + else + echo "mirror sync at $MIRROR_PATH failed; falling back to remote fetch" >&2 + fi + flock -u 9 + else + echo "mirror lock $MIRROR_PATH.lock is busy; falling back to remote fetch" >&2 fi - sleep $((attempt * 10)) - done + exec 9>&- + else + echo "mirror path $MIRROR_PATH is unavailable; falling back to remote fetch" >&2 + fi + if [ "$fetch_succeeded" != true ]; then + for attempt in 1 2 3; do + if bounded_git fetch --depth=1 --no-tags origin "$GITEA_SHA"; then + fetch_succeeded=true + break + fi + if [ "$attempt" -eq 3 ]; then + echo "exact staging gate checkout failed after $attempt bounded attempts" >&2 + exit 1 + fi + sleep $((attempt * 10)) + done + fi [[ "$fetch_succeeded" == true ]] git checkout --detach --force "$GITEA_SHA" git clean -ffdx @@ -272,25 +319,72 @@ jobs: IMAGE_REPOSITORY: crpi-d1feco6itet73spp.cn-hongkong.personal.cr.aliyuncs.com/copse/jyotisha steps: - name: Checkout exact Gitea revision + env: + MIRROR_PATH: /root/.cache/jyotisha-mirror.git run: | set -euo pipefail [[ "$GITEA_SHA" =~ ^[0-9a-f]{40}$ ]] git init . git remote remove origin 2>/dev/null || true git remote add origin https://git.copse.top/root/Jyotisha.git + bounded_git() { + timeout 300 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 "$@" + } fetch_succeeded=false - for attempt in 1 2 3; do - if timeout 300 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 \ - fetch --depth=1 --no-tags origin "$GITEA_SHA"; then - fetch_succeeded=true - break + # act_runner's hostexecutor discards this workspace after every run but + # keeps the host filesystem, so a bare mirror at MIRROR_PATH amortises + # the 105 MB tree across runs; the exact SHA is then fetched from local + # disk in seconds instead of 4-8 minutes per job over the WAN. The + # mirror is an accelerator, never a dependency: every failure below + # falls through to the bounded remote fetch that has always been used. + sync_mirror() { + if [ -d "$MIRROR_PATH" ] && [ "$(git -C "$MIRROR_PATH" rev-parse --is-bare-repository 2>/dev/null)" = true ]; then + # We hold the host lock, so any git lock file left by a cancelled job is stale. + find "$MIRROR_PATH" -name '*.lock' -type f -delete 2>/dev/null || true + if ! git -C "$MIRROR_PATH" cat-file -e "$GITEA_SHA^{commit}" 2>/dev/null; then + bounded_git -C "$MIRROR_PATH" fetch --prune origin || return 1 + fi + else + rm -rf "$MIRROR_PATH" + timeout 900 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 \ + clone --quiet --mirror https://git.copse.top/root/Jyotisha.git "$MIRROR_PATH" || { rm -rf "$MIRROR_PATH"; return 1; } fi - if [ "$attempt" -eq 3 ]; then - echo "exact staging gate checkout failed after $attempt bounded attempts" >&2 - exit 1 + git -C "$MIRROR_PATH" cat-file -e "$GITEA_SHA^{commit}" + } + if mkdir -p "$(dirname "$MIRROR_PATH")" 2>/dev/null && exec 9>"$MIRROR_PATH.lock" 2>/dev/null; then + if flock -w 900 9; then + if sync_mirror; then + git remote set-url origin "$MIRROR_PATH" + if timeout 300 git fetch --no-tags origin "$GITEA_SHA"; then + fetch_succeeded=true + else + echo "mirror fetch of $GITEA_SHA failed; falling back to remote fetch" >&2 + fi + git remote set-url origin https://git.copse.top/root/Jyotisha.git + else + echo "mirror sync at $MIRROR_PATH failed; falling back to remote fetch" >&2 + fi + flock -u 9 + else + echo "mirror lock $MIRROR_PATH.lock is busy; falling back to remote fetch" >&2 fi - sleep $((attempt * 10)) - done + exec 9>&- + else + echo "mirror path $MIRROR_PATH is unavailable; falling back to remote fetch" >&2 + fi + if [ "$fetch_succeeded" != true ]; then + for attempt in 1 2 3; do + if bounded_git fetch --depth=1 --no-tags origin "$GITEA_SHA"; then + fetch_succeeded=true + break + fi + if [ "$attempt" -eq 3 ]; then + echo "exact staging gate checkout failed after $attempt bounded attempts" >&2 + exit 1 + fi + sleep $((attempt * 10)) + done + fi [[ "$fetch_succeeded" == true ]] git checkout --detach --force "$GITEA_SHA" git clean -ffdx diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index 132e7e83..4eacf244 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -1464,3 +1464,49 @@ test("is-docs-only-range.sh decides from local history and refuses non-ancestor rmSync(root, { recursive: true, force: true }); } }); + +test("gate checkouts fetch the exact SHA from a host-persistent mirror and fall back to the bounded remote fetch", () => { + const workflow = read(giteaQualityWorkflow); + const checkouts = [...workflow.matchAll(/- name: Checkout exact Gitea revision[\s\S]*?(?=\n\s+- name: )/g)].map((match) => match[0]); + assert.equal(checkouts.length, 2); + + for (const step of checkouts) { + assert.match(step, /MIRROR_PATH: \/root\/\.cache\/jyotisha-mirror\.git/); + assert.match(step, /bounded_git\(\) \{\n\s+timeout 300 git -c http\.connectTimeout=15 -c http\.lowSpeedLimit=1 -c http\.lowSpeedTime=60 "\$@"/); + // validate and publish (or two overlapping runs) may touch the mirror at once. + assert.match(step, /exec 9>"\$MIRROR_PATH\.lock"/); + assert.match(step, /flock -w 900 9/); + assert.match(step, /flock -u 9/); + assert.match(step, /clone --quiet --mirror https:\/\/git\.copse\.top\/root\/Jyotisha\.git "\$MIRROR_PATH"/); + assert.match(step, /bounded_git -C "\$MIRROR_PATH" fetch --prune origin \|\| return 1/); + assert.match(step, /git -C "\$MIRROR_PATH" cat-file -e "\$GITEA_SHA\^\{commit\}"/); + assert.match(step, /find "\$MIRROR_PATH" -name '\*\.lock' -type f -delete/); + // The workspace fetches from local disk, then origin points back at Gitea + // for every later step. + assert.match( + step, + /git remote set-url origin "\$MIRROR_PATH"\n\s+if timeout 300 git fetch --no-tags origin "\$GITEA_SHA"; then\n\s+fetch_succeeded=true\n[\s\S]*?git remote set-url origin https:\/\/git\.copse\.top\/root\/Jyotisha\.git/, + ); + // Every mirror failure mode falls through to the pre-existing bounded remote fetch. + assert.equal((step.match(/falling back to remote fetch/g) ?? []).length, 4); + assert.match( + step, + /if \[ "\$fetch_succeeded" != true \]; then\n\s+for attempt in 1 2 3; do\n\s+if bounded_git fetch --depth=1 --no-tags origin "\$GITEA_SHA"; then/, + ); + assert.equal((step.match(/exit 1/g) ?? []).length, 1, "only the exhausted remote fetch may fail the checkout"); + assertOrder(step, [ + "git remote add origin https://git.copse.top/root/Jyotisha.git", + "sync_mirror() {", + 'flock -w 900 9', + 'git remote set-url origin "$MIRROR_PATH"', + 'if [ "$fetch_succeeded" != true ]; then', + 'fetch --depth=1 --no-tags origin "$GITEA_SHA"', + "exact staging gate checkout failed after $attempt bounded attempts", + '[[ "$fetch_succeeded" == true ]]', + 'git checkout --detach --force "$GITEA_SHA"', + "git clean -ffdx", + 'test "$(git rev-parse HEAD)" = "$GITEA_SHA"', + "git status --porcelain --untracked-files=all", + ]); + } +});