root/Jyotisha
1
0
Fork 0
Code Issues 1 Pull Requests 3 Actions Packages Projects Releases Wiki Activity

ci(gate): check out the exact SHA from a host-persistent mirror with remote fallback

Browse Source 
Both gate jobs spent 4-8 minutes (validate) plus 3-5 minutes (publish) on
`git fetch --depth=1` of a 105 MB tree over the WAN, and run 2270 burned
15 minutes on three timed-out attempts. act_runner's hostexecutor discards
the workspace between runs but keeps the host filesystem, so keep a bare
mirror at /root/.cache/jyotisha-mirror.git: take an flock on it (validate,
publish, or overlapping runs may race), clone it once or `fetch --prune`
only when it lacks the requested commit, fetch the exact SHA from local
disk, then point origin back at Gitea for every later step.

The mirror is an accelerator, not a dependency: an unwritable path, a busy
lock, a failed clone/fetch, a non-repository directory, or a stale git lock
file all fall through to the unchanged bounded three-attempt remote fetch,
and the SHA-format, checkout, `git clean -ffdx`, HEAD-equals-SHA, and
clean-tree assertions are untouched. Rehearsed locally against a local
remote for fresh, reused, prune-refresh, unwritable, corrupt, stale-lock,
and lock-contention cases.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VawU7Xfd5jS9wUEXz1XYmS
This commit is contained in:
Jesse_Chen
2026-09-01 23:20:13 +00:00
parent 534f5e617c
commit 68414a8223
2 changed files with 160 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/workflows/backend-quality-gate.yml
+114 -20
View File
@@ -65,25 +65,72 @@ jobs:
NODE_TOOL_IMAGE: node:22-bookworm-slim
steps:
- name: Checkout exact Gitea revision
env:
MIRROR_PATH: /root/.cache/jyotisha-mirror.git
run: |
set -euo pipefail
[[ "$GITEA_SHA" =~ ^[0-9a-f]{40}$ ]]
git init .
git remote remove origin 2>/dev/null || true
git remote add origin https://git.copse.top/root/Jyotisha.git
bounded_git() {
timeout 300 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 "$@"
}
fetch_succeeded=false
for attempt in 1 2 3; do
if timeout 300 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 \
fetch --depth=1 --no-tags origin "$GITEA_SHA"; then
fetch_succeeded=true
break
# act_runner's hostexecutor discards this workspace after every run but
# keeps the host filesystem, so a bare mirror at MIRROR_PATH amortises
# the 105 MB tree across runs; the exact SHA is then fetched from local
# disk in seconds instead of 4-8 minutes per job over the WAN. The
# mirror is an accelerator, never a dependency: every failure below
# falls through to the bounded remote fetch that has always been used.
sync_mirror() {
if [ -d "$MIRROR_PATH" ] && [ "$(git -C "$MIRROR_PATH" rev-parse --is-bare-repository 2>/dev/null)" = true ]; then
# We hold the host lock, so any git lock file left by a cancelled job is stale.
find "$MIRROR_PATH" -name '*.lock' -type f -delete 2>/dev/null || true
if ! git -C "$MIRROR_PATH" cat-file -e "$GITEA_SHA^{commit}" 2>/dev/null; then
bounded_git -C "$MIRROR_PATH" fetch --prune origin || return 1
fi
else
rm -rf "$MIRROR_PATH"
timeout 900 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 \
clone --quiet --mirror https://git.copse.top/root/Jyotisha.git "$MIRROR_PATH" || { rm -rf "$MIRROR_PATH"; return 1; }
fi
if [ "$attempt" -eq 3 ]; then
echo "exact staging gate checkout failed after $attempt bounded attempts" >&2
exit 1
git -C "$MIRROR_PATH" cat-file -e "$GITEA_SHA^{commit}"
}
if mkdir -p "$(dirname "$MIRROR_PATH")" 2>/dev/null && exec 9>"$MIRROR_PATH.lock" 2>/dev/null; then
if flock -w 900 9; then
if sync_mirror; then
git remote set-url origin "$MIRROR_PATH"
if timeout 300 git fetch --no-tags origin "$GITEA_SHA"; then
fetch_succeeded=true
else
echo "mirror fetch of $GITEA_SHA failed; falling back to remote fetch" >&2
fi
git remote set-url origin https://git.copse.top/root/Jyotisha.git
else
echo "mirror sync at $MIRROR_PATH failed; falling back to remote fetch" >&2
fi
flock -u 9
else
echo "mirror lock $MIRROR_PATH.lock is busy; falling back to remote fetch" >&2
fi
sleep $((attempt * 10))
done
exec 9>&-
else
echo "mirror path $MIRROR_PATH is unavailable; falling back to remote fetch" >&2
fi
if [ "$fetch_succeeded" != true ]; then
for attempt in 1 2 3; do
if bounded_git fetch --depth=1 --no-tags origin "$GITEA_SHA"; then
fetch_succeeded=true
break
fi
if [ "$attempt" -eq 3 ]; then
echo "exact staging gate checkout failed after $attempt bounded attempts" >&2
exit 1
fi
sleep $((attempt * 10))
done
fi
[[ "$fetch_succeeded" == true ]]
git checkout --detach --force "$GITEA_SHA"
git clean -ffdx
@@ -272,25 +319,72 @@ jobs:
IMAGE_REPOSITORY: crpi-d1feco6itet73spp.cn-hongkong.personal.cr.aliyuncs.com/copse/jyotisha
steps:
- name: Checkout exact Gitea revision
env:
MIRROR_PATH: /root/.cache/jyotisha-mirror.git
run: |
set -euo pipefail
[[ "$GITEA_SHA" =~ ^[0-9a-f]{40}$ ]]
git init .
git remote remove origin 2>/dev/null || true
git remote add origin https://git.copse.top/root/Jyotisha.git
bounded_git() {
timeout 300 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 "$@"
}
fetch_succeeded=false
for attempt in 1 2 3; do
if timeout 300 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 \
fetch --depth=1 --no-tags origin "$GITEA_SHA"; then
fetch_succeeded=true
break
# act_runner's hostexecutor discards this workspace after every run but
# keeps the host filesystem, so a bare mirror at MIRROR_PATH amortises
# the 105 MB tree across runs; the exact SHA is then fetched from local
# disk in seconds instead of 4-8 minutes per job over the WAN. The
# mirror is an accelerator, never a dependency: every failure below
# falls through to the bounded remote fetch that has always been used.
sync_mirror() {
if [ -d "$MIRROR_PATH" ] && [ "$(git -C "$MIRROR_PATH" rev-parse --is-bare-repository 2>/dev/null)" = true ]; then
# We hold the host lock, so any git lock file left by a cancelled job is stale.
find "$MIRROR_PATH" -name '*.lock' -type f -delete 2>/dev/null || true
if ! git -C "$MIRROR_PATH" cat-file -e "$GITEA_SHA^{commit}" 2>/dev/null; then
bounded_git -C "$MIRROR_PATH" fetch --prune origin || return 1
fi
else
rm -rf "$MIRROR_PATH"
timeout 900 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 \
clone --quiet --mirror https://git.copse.top/root/Jyotisha.git "$MIRROR_PATH" || { rm -rf "$MIRROR_PATH"; return 1; }
fi
if [ "$attempt" -eq 3 ]; then
echo "exact staging gate checkout failed after $attempt bounded attempts" >&2
exit 1
git -C "$MIRROR_PATH" cat-file -e "$GITEA_SHA^{commit}"
}
if mkdir -p "$(dirname "$MIRROR_PATH")" 2>/dev/null && exec 9>"$MIRROR_PATH.lock" 2>/dev/null; then
if flock -w 900 9; then
if sync_mirror; then
git remote set-url origin "$MIRROR_PATH"
if timeout 300 git fetch --no-tags origin "$GITEA_SHA"; then
fetch_succeeded=true
else
echo "mirror fetch of $GITEA_SHA failed; falling back to remote fetch" >&2
fi
git remote set-url origin https://git.copse.top/root/Jyotisha.git
else
echo "mirror sync at $MIRROR_PATH failed; falling back to remote fetch" >&2
fi
flock -u 9
else
echo "mirror lock $MIRROR_PATH.lock is busy; falling back to remote fetch" >&2
fi
sleep $((attempt * 10))
done
exec 9>&-
else
echo "mirror path $MIRROR_PATH is unavailable; falling back to remote fetch" >&2
fi
if [ "$fetch_succeeded" != true ]; then
for attempt in 1 2 3; do
if bounded_git fetch --depth=1 --no-tags origin "$GITEA_SHA"; then
fetch_succeeded=true
break
fi
if [ "$attempt" -eq 3 ]; then
echo "exact staging gate checkout failed after $attempt bounded attempts" >&2
exit 1
fi
sleep $((attempt * 10))
done
fi
[[ "$fetch_succeeded" == true ]]
git checkout --detach --force "$GITEA_SHA"
git clean -ffdx
frontend/tests/staging-backend-workflows.test.ts
+46
View File
@@ -1464,3 +1464,49 @@ test("is-docs-only-range.sh decides from local history and refuses non-ancestor
rmSync(root, { recursive: true, force: true });
}
});
test("gate checkouts fetch the exact SHA from a host-persistent mirror and fall back to the bounded remote fetch", () => {
const workflow = read(giteaQualityWorkflow);
const checkouts = [...workflow.matchAll(/- name: Checkout exact Gitea revision[\s\S]*?(?=\n\s+- name: )/g)].map((match) => match[0]);
assert.equal(checkouts.length, 2);
for (const step of checkouts) {
assert.match(step, /MIRROR_PATH: \/root\/\.cache\/jyotisha-mirror\.git/);
assert.match(step, /bounded_git\(\) \{\n\s+timeout 300 git -c http\.connectTimeout=15 -c http\.lowSpeedLimit=1 -c http\.lowSpeedTime=60 "\$@"/);
// validate and publish (or two overlapping runs) may touch the mirror at once.
assert.match(step, /exec 9>"\$MIRROR_PATH\.lock"/);
assert.match(step, /flock -w 900 9/);
assert.match(step, /flock -u 9/);
assert.match(step, /clone --quiet --mirror https:\/\/git\.copse\.top\/root\/Jyotisha\.git "\$MIRROR_PATH"/);
assert.match(step, /bounded_git -C "\$MIRROR_PATH" fetch --prune origin \|\| return 1/);
assert.match(step, /git -C "\$MIRROR_PATH" cat-file -e "\$GITEA_SHA\^\{commit\}"/);
assert.match(step, /find "\$MIRROR_PATH" -name '\*\.lock' -type f -delete/);
// The workspace fetches from local disk, then origin points back at Gitea
// for every later step.
assert.match(
step,
/git remote set-url origin "\$MIRROR_PATH"\n\s+if timeout 300 git fetch --no-tags origin "\$GITEA_SHA"; then\n\s+fetch_succeeded=true\n[\s\S]*?git remote set-url origin https:\/\/git\.copse\.top\/root\/Jyotisha\.git/,
);
// Every mirror failure mode falls through to the pre-existing bounded remote fetch.
assert.equal((step.match(/falling back to remote fetch/g) ?? []).length, 4);
assert.match(
step,
/if \[ "\$fetch_succeeded" != true \]; then\n\s+for attempt in 1 2 3; do\n\s+if bounded_git fetch --depth=1 --no-tags origin "\$GITEA_SHA"; then/,
);
assert.equal((step.match(/exit 1/g) ?? []).length, 1, "only the exhausted remote fetch may fail the checkout");
assertOrder(step, [
"git remote add origin https://git.copse.top/root/Jyotisha.git",
"sync_mirror() {",
'flock -w 900 9',
'git remote set-url origin "$MIRROR_PATH"',
'if [ "$fetch_succeeded" != true ]; then',
'fetch --depth=1 --no-tags origin "$GITEA_SHA"',
"exact staging gate checkout failed after $attempt bounded attempts",
'[[ "$fetch_succeeded" == true ]]',
'git checkout --detach --force "$GITEA_SHA"',
"git clean -ffdx",
'test "$(git rev-parse HEAD)" = "$GITEA_SHA"',
"git status --porcelain --untracked-files=all",
]);
}
});