diff --git a/.gitea/workflows/backend-quality-gate.yml b/.gitea/workflows/backend-quality-gate.yml index 71a0f45a..d2d2bbf4 100644 --- a/.gitea/workflows/backend-quality-gate.yml +++ b/.gitea/workflows/backend-quality-gate.yml @@ -262,8 +262,12 @@ jobs: [[ "$web_digest" =~ ^sha256:[0-9a-f]{64}$ ]] install -d -m 700 artifacts/staging-images umask 077 - printf 'git_sha=%s\napi_digest=%s\nweb_digest=%s\n' \ - "$GITEA_SHA" "$api_digest" "$web_digest" \ + git archive --format=tar --output artifacts/staging-images/controller.tar \ + "$GITEA_SHA" deploy frontend/scripts/staging-image-manifest.mjs + controller_sha256="$(sha256sum artifacts/staging-images/controller.tar | awk '{print $1}')" + [[ "$controller_sha256" =~ ^[0-9a-f]{64}$ ]] + printf 'git_sha=%s\napi_digest=%s\nweb_digest=%s\ncontroller_sha256=%s\n' \ + "$GITEA_SHA" "$api_digest" "$web_digest" "$controller_sha256" \ > artifacts/staging-images/manifest.env node frontend/scripts/staging-image-manifest.mjs \ artifacts/staging-images/manifest.env "$GITEA_SHA" "$IMAGE_REPOSITORY" >/dev/null @@ -282,7 +286,7 @@ jobs: --workdir "$workdir" \ --env HOME=/tmp \ --env "INPUT_NAME=staging-image-manifest-$GITEA_SHA-$GITEA_RUN_ATTEMPT" \ - --env INPUT_PATH=artifacts/staging-images/manifest.env \ + --env INPUT_PATH=artifacts/staging-images/ \ --env INPUT_OVERWRITE=false \ --env ACTIONS_RUNTIME_TOKEN \ --env ACTIONS_RESULTS_URL \ diff --git a/.gitea/workflows/deploy-staging.yml b/.gitea/workflows/deploy-staging.yml index db6f873e..c10d0e2c 100644 --- a/.gitea/workflows/deploy-staging.yml +++ b/.gitea/workflows/deploy-staging.yml @@ -76,42 +76,64 @@ jobs: fi [[ "$gate_run_id" =~ ^[0-9]+$ ]] || { echo "no successful exact-SHA staging quality gate run found" >&2; exit 1; } - staging_head="$(git ls-remote https://git.copse.top/root/Jyotisha.git refs/heads/staging | awk '{print $1}')" - [[ "$staging_head" =~ ^[0-9a-f]{40}$ ]] + read_ref_sha() { + local branch="$1" + curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/$branch" | + jq -er --arg ref "refs/heads/$branch" ' + select(type == "array" and length == 1) | + .[0] | select(.ref == $ref) | .object.sha | + select(test("^[0-9a-f]{40}$")) + ' + } + staging_head="$(read_ref_sha staging)" + controller_sha="$(read_ref_sha main)" + [[ "$controller_sha" == "$staging_head" ]] || { echo "reviewed main and staging controller heads differ" >&2; exit 1; } if [[ "$allow_rollback" == false && "$REQUESTED_SHA" != "$staging_head" ]]; then echo "stale staging revision refused; use explicit manual rollback only when intended" >&2 exit 1 fi + if [[ "$allow_rollback" == true && "$REQUESTED_SHA" != "$controller_sha" ]]; then + comparison="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/compare/$REQUESTED_SHA...$controller_sha")" + jq -e --arg base "$REQUESTED_SHA" --arg head "$controller_sha" ' + (.commits // []) as $commits | + def parents($sha): [$commits[] | select(.sha == $sha) | (.parents // [])[] | .sha]; + def reaches($sha; $seen): + if $sha == $base then true + elif ($seen | index($sha)) != null then false + else any(parents($sha)[]; . as $parent | reaches($parent; $seen + [$sha])) end; + (.total_commits | type) == "number" and + .total_commits == ($commits | length) and ($commits | length) > 0 and + ([$commits[].sha] | length == (unique | length)) and reaches($head; []) + ' <<<"$comparison" >/dev/null || { echo "rollback revision is not in reviewed main history" >&2; exit 1; } + fi + + controller_gate_run_id="$gate_run_id" + if [[ "$controller_sha" != "$REQUESTED_SHA" ]]; then + controller_runs="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$controller_sha&branch=staging&event=push&status=success&limit=100")" + controller_run="$(jq -cer --arg sha "$controller_sha" ' + [.workflow_runs[] | select( + (.path | split("@")[0] | endswith("backend-quality-gate.yml")) and + .head_sha == $sha and .head_branch == "staging" and + .event == "push" and .conclusion == "success" + )] | sort_by(.id) | reverse | first + ' <<<"$controller_runs")" + controller_gate_run_id="$(jq -er '.id' <<<"$controller_run")" + fi + [[ "$controller_gate_run_id" =~ ^[0-9]+$ ]] { echo "sha=$REQUESTED_SHA" echo "gate_run_id=$gate_run_id" + echo "controller_sha=$controller_sha" + echo "controller_gate_run_id=$controller_gate_run_id" echo "allow_rollback=$allow_rollback" } >>"$GITHUB_OUTPUT" - - name: Checkout trusted main controller - env: - DEPLOY_SHA: ${{ steps.revision.outputs.sha }} - run: | - set -euo pipefail - git init . - git remote remove origin 2>/dev/null || true - git remote add origin https://git.copse.top/root/Jyotisha.git - fetch_succeeded=false - for attempt in 1 2 3; do - if timeout 120 git fetch --no-tags origin main "$DEPLOY_SHA"; then - fetch_succeeded=true - break - fi - if [ "$attempt" -eq 3 ]; then - echo "trusted main fetch failed after $attempt bounded attempts" >&2 - exit 1 - fi - sleep $((attempt * 10)) - done - [[ "$fetch_succeeded" == true ]] - git checkout --detach --force origin/main - git merge-base --is-ancestor "$DEPLOY_SHA" HEAD || { echo "staging revision is not in trusted main history" >&2; exit 1; } - - name: Prepare pinned Node tooling env: NODE_TOOL_SOURCE_IMAGE: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/library/node:22-bookworm-slim@sha256:ef343465b6a14bbdf2ab52f6e100ec0659a792464fcf72c462370d88b3df909c @@ -152,44 +174,103 @@ jobs: node --version npm --version - - name: Download gate-produced image manifest + - name: Download target and controller gate artifacts env: - GATE_RUN_ID: ${{ steps.revision.outputs.gate_run_id }} + TARGET_GATE_RUN_ID: ${{ steps.revision.outputs.gate_run_id }} DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + CONTROLLER_GATE_RUN_ID: ${{ steps.revision.outputs.controller_gate_run_id }} + CONTROLLER_SHA: ${{ steps.revision.outputs.controller_sha }} run: | set -euo pipefail - artifact_prefix="staging-image-manifest-$DEPLOY_SHA-" - artifacts="$(curl --fail --silent --show-error \ - --header "Authorization: token $GITEA_TOKEN" \ - "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs/$GATE_RUN_ID/artifacts?limit=100")" - selected_artifact="$(jq -cer --arg prefix "$artifact_prefix" ' - [(.artifacts // [])[] - | select(.expired == false and (.name | startswith($prefix))) - | . + {attempt: ((.name | ltrimstr($prefix)) | tonumber?)} - | select(.attempt != null and .attempt >= 1) - ] | sort_by(.attempt, .id) | reverse | first - ' <<<"$artifacts")" - artifact_name="$(jq -er '.name' <<<"$selected_artifact")" - artifact_id="$(jq -er '.id' <<<"$selected_artifact")" - artifact_attempt="${artifact_name#"$artifact_prefix"}" - [[ "$artifact_name" == "$artifact_prefix"* ]] - [[ "$artifact_attempt" =~ ^[1-9][0-9]*$ ]] - [[ "$artifact_id" =~ ^[0-9]+$ ]] - install -d -m 700 artifacts/staging-image - curl --fail --silent --show-error --location \ - --header "Authorization: token $GITEA_TOKEN" \ - "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/artifacts/$artifact_id/zip" \ - --output "${RUNNER_TEMP}/staging-image-manifest.zip" - unzip -q "${RUNNER_TEMP}/staging-image-manifest.zip" -d artifacts/staging-image - [[ -f artifacts/staging-image/manifest.env ]] + download_bundle() { + local run_id="$1" sha="$2" destination="$3" zip_path="$4" + local prefix artifacts selected name id attempt + prefix="staging-image-manifest-$sha-" + artifacts="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs/$run_id/artifacts?limit=100")" + selected="$(jq -cer --arg prefix "$prefix" ' + [(.artifacts // [])[] + | select(.expired == false and (.name | startswith($prefix))) + | . + {attempt: ((.name | ltrimstr($prefix)) | tonumber?)} + | select(.attempt != null and .attempt >= 1) + ] | sort_by(.attempt, .id) | reverse | first + ' <<<"$artifacts")" + name="$(jq -er '.name' <<<"$selected")" + id="$(jq -er '.id' <<<"$selected")" + attempt="${name#"$prefix"}" + [[ "$name" == "$prefix"* && "$attempt" =~ ^[1-9][0-9]*$ && "$id" =~ ^[0-9]+$ ]] + install -d -m 700 "$destination" + curl --fail --silent --show-error --location --connect-timeout 15 --max-time 120 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/artifacts/$id/zip" \ + --output "$zip_path" + python3 - "$zip_path" "$destination" <<'PY' + import pathlib, stat, sys, zipfile + archive = pathlib.Path(sys.argv[1]) + destination = pathlib.Path(sys.argv[2]) + allowed = {"manifest.env", "controller.tar"} + with zipfile.ZipFile(archive) as bundle: + entries = bundle.infolist() + names = [entry.filename for entry in entries] + if len(names) != len(set(names)) or not names or not set(names).issubset(allowed): + raise SystemExit("invalid staging artifact bundle") + if sum(entry.file_size for entry in entries) > 3 * 1024 * 1024: + raise SystemExit("staging artifact bundle is too large") + for entry in entries: + path = pathlib.PurePosixPath(entry.filename) + mode = entry.external_attr >> 16 + if path.is_absolute() or ".." in path.parts or path.name != entry.filename: + raise SystemExit("unsafe staging artifact path") + if mode and not stat.S_ISREG(mode): + raise SystemExit("unsafe staging artifact type") + target = destination / entry.filename + with bundle.open(entry) as source, target.open("xb") as output: + output.write(source.read()) + PY + [[ -f "$destination/manifest.env" ]] + } + rm -rf artifacts/staging-image artifacts/controller + download_bundle "$TARGET_GATE_RUN_ID" "$DEPLOY_SHA" artifacts/staging-image "${RUNNER_TEMP}/staging-target.zip" + download_bundle "$CONTROLLER_GATE_RUN_ID" "$CONTROLLER_SHA" artifacts/controller "${RUNNER_TEMP}/staging-controller.zip" - - name: Validate immutable image manifest + - name: Validate gate-attested controller and immutable image manifest id: images env: DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + CONTROLLER_SHA: ${{ steps.revision.outputs.controller_sha }} run: | set -euo pipefail - node frontend/scripts/staging-image-manifest.mjs \ + controller_manifest=artifacts/controller/manifest.env + controller_tar=artifacts/controller/controller.tar + [[ -f "$controller_tar" ]] + [[ "$(wc -l < "$controller_manifest" | tr -d ' ')" == 4 ]] + manifest_controller_sha="$(awk -F= '$1 == "git_sha" {print $2}' "$controller_manifest")" + expected_controller_digest="$(awk -F= '$1 == "controller_sha256" {print $2}' "$controller_manifest")" + [[ "$manifest_controller_sha" == "$CONTROLLER_SHA" ]] + [[ "$expected_controller_digest" =~ ^[0-9a-f]{64}$ ]] + printf '%s %s\n' "$expected_controller_digest" "$controller_tar" | sha256sum --check --status + python3 - "$controller_tar" <<'PY' + import pathlib, sys, tarfile + archive = pathlib.Path(sys.argv[1]) + required = {"deploy/run-staging-deploy.sh", "frontend/scripts/staging-image-manifest.mjs"} + with tarfile.open(archive, "r:") as bundle: + members = bundle.getmembers() + names = [member.name for member in members] + if len(names) != len(set(names)) or not required.issubset(names): + raise SystemExit("invalid staging controller bundle") + if sum(member.size for member in members) > 2 * 1024 * 1024: + raise SystemExit("staging controller bundle is too large") + for member in members: + path = pathlib.PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts or not (member.isdir() or member.isfile()): + raise SystemExit("unsafe staging controller bundle") + PY + install -d -m 700 artifacts/controller/extracted + tar -xf "$controller_tar" -C artifacts/controller/extracted + node artifacts/controller/extracted/frontend/scripts/staging-image-manifest.mjs \ + "$controller_manifest" "$CONTROLLER_SHA" "$IMAGE_REPOSITORY" >/dev/null + node artifacts/controller/extracted/frontend/scripts/staging-image-manifest.mjs \ artifacts/staging-image/manifest.env "$DEPLOY_SHA" "$IMAGE_REPOSITORY" >>"$GITHUB_OUTPUT" - name: Deploy exact image digests under pinned SSH identity @@ -217,7 +298,12 @@ jobs: remote="$DEPLOY_USER@$DEPLOY_HOST" require_current_staging_head() { [[ "$ALLOW_ROLLBACK" == true ]] && return - current_head="$(git ls-remote https://git.copse.top/root/Jyotisha.git refs/heads/staging | awk '{print $1}')" + current_head="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/staging" | + jq -er 'select(type == "array" and length == 1) | .[0] | + select(.ref == "refs/heads/staging") | .object.sha | + select(test("^[0-9a-f]{40}$"))')" [[ "$current_head" == "$DEPLOY_SHA" ]] || { echo "staging advanced during deployment; refusing stale mutation" >&2; exit 1; } } cleanup() { @@ -230,15 +316,26 @@ jobs: incoming="$(ssh "${ssh_options[@]}" "$remote" "mktemp -d /tmp/jyotisha-staging.XXXXXXXXXX")" [[ "$incoming" == /tmp/jyotisha-staging.* ]] ssh "${ssh_options[@]}" "$remote" "install -d -m 700 '$incoming/.docker'" - tar -cf "${RUNNER_TEMP}/deploy.tar" deploy - scp -i "$key_path" -P "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path" "${RUNNER_TEMP}/deploy.tar" "$remote:$incoming/deploy.tar" - ssh "${ssh_options[@]}" "$remote" "tar -xf '$incoming/deploy.tar' -C '$incoming' && rm -f -- '$incoming/deploy.tar'" + scp -i "$key_path" -P "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path" artifacts/controller/controller.tar "$remote:$incoming/controller.tar" + ssh "${ssh_options[@]}" "$remote" "tar -xf '$incoming/controller.tar' -C '$incoming' && rm -f -- '$incoming/controller.tar'" previous_sha="$(ssh "${ssh_options[@]}" "$remote" "state='$DEPLOY_PATH/.state/deployed-revision'; if [ -f \"\$state\" ]; then cat \"\$state\"; else id=\$(sudo -n docker ps -aq --filter 'label=com.docker.compose.project=jyotisha-staging' --filter 'label=com.docker.compose.service=web' | head -n 1); if [ -n \"\$id\" ]; then sudo -n docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \"\$id\" | sed -n 's/^GITHUB_SHA=//p' | head -n 1; else printf not-deployed; fi; fi")" [[ "$previous_sha" == not-deployed || "$previous_sha" =~ ^[0-9a-f]{40}$ ]] || exit 1 forward_verified=false if [[ "$previous_sha" != not-deployed && "$previous_sha" != "$DEPLOY_SHA" && "$ALLOW_ROLLBACK" != true ]]; then - git cat-file -e "${previous_sha}^{commit}" 2>/dev/null || git fetch origin "$previous_sha" - git merge-base --is-ancestor "$previous_sha" "$DEPLOY_SHA" || { echo "automatic staging rollback or divergent deploy refused" >&2; exit 1; } + comparison="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/compare/$previous_sha...$DEPLOY_SHA")" + jq -e --arg base "$previous_sha" --arg head "$DEPLOY_SHA" ' + (.commits // []) as $commits | + def parents($sha): [$commits[] | select(.sha == $sha) | (.parents // [])[] | .sha]; + def reaches($sha; $seen): + if $sha == $base then true + elif ($seen | index($sha)) != null then false + else any(parents($sha)[]; . as $parent | reaches($parent; $seen + [$sha])) end; + (.total_commits | type) == "number" and + .total_commits == ($commits | length) and ($commits | length) > 0 and + ([$commits[].sha] | length == (unique | length)) and reaches($head; []) + ' <<<"$comparison" >/dev/null || { echo "automatic staging rollback or divergent deploy refused" >&2; exit 1; } forward_verified=true fi require_current_staging_head diff --git a/.gitea/workflows/migrate-staging-database.yml b/.gitea/workflows/migrate-staging-database.yml index 67f2f1c1..bd2df30e 100644 --- a/.gitea/workflows/migrate-staging-database.yml +++ b/.gitea/workflows/migrate-staging-database.yml @@ -41,8 +41,21 @@ jobs: run: | set -euo pipefail [[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "deploy_sha must be a lowercase full commit SHA" >&2; exit 1; } - staging_head="$(git ls-remote https://git.copse.top/root/Jyotisha.git refs/heads/staging | awk '{print $1}')" + read_ref_sha() { + local branch="$1" + curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/$branch" | + jq -er --arg ref "refs/heads/$branch" ' + select(type == "array" and length == 1) | + .[0] | select(.ref == $ref) | .object.sha | + select(test("^[0-9a-f]{40}$")) + ' + } + staging_head="$(read_ref_sha staging)" + main_head="$(read_ref_sha main)" [[ "$staging_head" == "$DEPLOY_SHA" ]] || { echo "migration requires current staging head" >&2; exit 1; } + [[ "$main_head" == "$DEPLOY_SHA" ]] || { echo "staging migration revision must equal reviewed main head" >&2; exit 1; } runs="$(curl --fail --silent --show-error \ --header "Authorization: token $GITEA_TOKEN" \ "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$DEPLOY_SHA&branch=staging&event=push&status=success&limit=100")" @@ -60,30 +73,6 @@ jobs: echo "gate_run_id=$gate_run_id" } >>"$GITHUB_OUTPUT" - - name: Checkout trusted main controller - env: - DEPLOY_SHA: ${{ steps.revision.outputs.sha }} - run: | - set -euo pipefail - git init . - git remote remove origin 2>/dev/null || true - git remote add origin https://git.copse.top/root/Jyotisha.git - fetch_succeeded=false - for attempt in 1 2 3; do - if timeout 120 git fetch --no-tags origin main "$DEPLOY_SHA"; then - fetch_succeeded=true - break - fi - if [ "$attempt" -eq 3 ]; then - echo "trusted main fetch failed after $attempt bounded attempts" >&2 - exit 1 - fi - sleep $((attempt * 10)) - done - [[ "$fetch_succeeded" == true ]] - git checkout --detach --force origin/main - git merge-base --is-ancestor "$DEPLOY_SHA" HEAD || { echo "staging revision is not in trusted main history" >&2; exit 1; } - - name: Prepare pinned Node tooling env: NODE_TOOL_SOURCE_IMAGE: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/library/node:22-bookworm-slim@sha256:ef343465b6a14bbdf2ab52f6e100ec0659a792464fcf72c462370d88b3df909c @@ -152,17 +141,65 @@ jobs: --header "Authorization: token $GITEA_TOKEN" \ "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/artifacts/$artifact_id/zip" \ --output "${RUNNER_TEMP}/staging-image-manifest.zip" - unzip -q "${RUNNER_TEMP}/staging-image-manifest.zip" -d artifacts/staging-image + python3 - "${RUNNER_TEMP}/staging-image-manifest.zip" artifacts/staging-image <<'PY' + import pathlib, stat, sys, zipfile + archive = pathlib.Path(sys.argv[1]) + destination = pathlib.Path(sys.argv[2]) + allowed = {"manifest.env", "controller.tar"} + with zipfile.ZipFile(archive) as bundle: + entries = bundle.infolist() + names = [entry.filename for entry in entries] + if len(names) != len(set(names)) or set(names) != allowed: + raise SystemExit("invalid staging artifact bundle") + if sum(entry.file_size for entry in entries) > 3 * 1024 * 1024: + raise SystemExit("staging artifact bundle is too large") + for entry in entries: + path = pathlib.PurePosixPath(entry.filename) + mode = entry.external_attr >> 16 + if path.is_absolute() or ".." in path.parts or path.name != entry.filename: + raise SystemExit("unsafe staging artifact path") + if mode and not stat.S_ISREG(mode): + raise SystemExit("unsafe staging artifact type") + target = destination / entry.filename + with bundle.open(entry) as source, target.open("xb") as output: + output.write(source.read()) + PY [[ -f artifacts/staging-image/manifest.env ]] + [[ -f artifacts/staging-image/controller.tar ]] - - name: Validate digest-pinned migration image + - name: Validate gate-attested controller and digest-pinned migration image id: image env: DEPLOY_SHA: ${{ steps.revision.outputs.sha }} run: | set -euo pipefail - node frontend/scripts/staging-image-manifest.mjs \ - artifacts/staging-image/manifest.env "$DEPLOY_SHA" "$IMAGE_REPOSITORY" >>"$GITHUB_OUTPUT" + manifest=artifacts/staging-image/manifest.env + controller_tar=artifacts/staging-image/controller.tar + [[ "$(wc -l < "$manifest" | tr -d ' ')" == 4 ]] + manifest_sha="$(awk -F= '$1 == "git_sha" {print $2}' "$manifest")" + expected_controller_digest="$(awk -F= '$1 == "controller_sha256" {print $2}' "$manifest")" + [[ "$manifest_sha" == "$DEPLOY_SHA" && "$expected_controller_digest" =~ ^[0-9a-f]{64}$ ]] + printf '%s %s\n' "$expected_controller_digest" "$controller_tar" | sha256sum --check --status + python3 - "$controller_tar" <<'PY' + import pathlib, sys, tarfile + archive = pathlib.Path(sys.argv[1]) + required = {"deploy/run-staging-migration.sh", "frontend/scripts/staging-image-manifest.mjs"} + with tarfile.open(archive, "r:") as bundle: + members = bundle.getmembers() + names = [member.name for member in members] + if len(names) != len(set(names)) or not required.issubset(names): + raise SystemExit("invalid staging controller bundle") + if sum(member.size for member in members) > 2 * 1024 * 1024: + raise SystemExit("staging controller bundle is too large") + for member in members: + path = pathlib.PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts or not (member.isdir() or member.isfile()): + raise SystemExit("unsafe staging controller bundle") + PY + install -d -m 700 artifacts/staging-image/extracted + tar -xf "$controller_tar" -C artifacts/staging-image/extracted + node artifacts/staging-image/extracted/frontend/scripts/staging-image-manifest.mjs \ + "$manifest" "$DEPLOY_SHA" "$IMAGE_REPOSITORY" >>"$GITHUB_OUTPUT" - name: Apply digest-pinned migration under host lock env: @@ -186,7 +223,12 @@ jobs: ssh_options=(-i "$key_path" -p "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path") remote="$DEPLOY_USER@$DEPLOY_HOST" require_current_staging_head() { - current_head="$(git ls-remote https://git.copse.top/root/Jyotisha.git refs/heads/staging | awk '{print $1}')" + current_head="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/staging" | + jq -er 'select(type == "array" and length == 1) | .[0] | + select(.ref == "refs/heads/staging") | .object.sha | + select(test("^[0-9a-f]{40}$"))')" [[ "$current_head" == "$DEPLOY_SHA" ]] || { echo "staging advanced during migration; refusing stale mutation" >&2; exit 1; } } cleanup() { @@ -199,15 +241,26 @@ jobs: incoming="$(ssh "${ssh_options[@]}" "$remote" "mktemp -d /tmp/jyotisha-staging.XXXXXXXXXX")" [[ "$incoming" == /tmp/jyotisha-staging.* ]] ssh "${ssh_options[@]}" "$remote" "install -d -m 700 '$incoming/.docker'" - tar -cf "${RUNNER_TEMP}/deploy.tar" deploy - scp -i "$key_path" -P "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path" "${RUNNER_TEMP}/deploy.tar" "$remote:$incoming/deploy.tar" - ssh "${ssh_options[@]}" "$remote" "tar -xf '$incoming/deploy.tar' -C '$incoming' && rm -f -- '$incoming/deploy.tar'" + scp -i "$key_path" -P "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path" artifacts/staging-image/controller.tar "$remote:$incoming/controller.tar" + ssh "${ssh_options[@]}" "$remote" "tar -xf '$incoming/controller.tar' -C '$incoming' && rm -f -- '$incoming/controller.tar'" previous_sha="$(ssh "${ssh_options[@]}" "$remote" "state='$DEPLOY_PATH/.state/deployed-revision'; if [ -f \"\$state\" ]; then cat \"\$state\"; else id=\$(sudo -n docker ps -aq --filter 'label=com.docker.compose.project=jyotisha-staging' --filter 'label=com.docker.compose.service=web' | head -n 1); if [ -n \"\$id\" ]; then sudo -n docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \"\$id\" | sed -n 's/^GITHUB_SHA=//p' | head -n 1; else printf not-deployed; fi; fi")" [[ "$previous_sha" == not-deployed || "$previous_sha" =~ ^[0-9a-f]{40}$ ]] || exit 1 forward_verified=false if [[ "$previous_sha" != not-deployed && "$previous_sha" != "$DEPLOY_SHA" ]]; then - git cat-file -e "${previous_sha}^{commit}" 2>/dev/null || git fetch origin "$previous_sha" - git merge-base --is-ancestor "$previous_sha" "$DEPLOY_SHA" || { echo "migration rollback or divergence refused" >&2; exit 1; } + comparison="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/compare/$previous_sha...$DEPLOY_SHA")" + jq -e --arg base "$previous_sha" --arg head "$DEPLOY_SHA" ' + (.commits // []) as $commits | + def parents($sha): [$commits[] | select(.sha == $sha) | (.parents // [])[] | .sha]; + def reaches($sha; $seen): + if $sha == $base then true + elif ($seen | index($sha)) != null then false + else any(parents($sha)[]; . as $parent | reaches($parent; $seen + [$sha])) end; + (.total_commits | type) == "number" and + .total_commits == ($commits | length) and ($commits | length) > 0 and + ([$commits[].sha] | length == (unique | length)) and reaches($head; []) + ' <<<"$comparison" >/dev/null || { echo "migration rollback or divergence refused" >&2; exit 1; } forward_verified=true fi require_current_staging_head diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index e81df7d0..7ac6d6ff 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -2227,16 +2227,16 @@ ## BUG-129 | staging trusted-main checkout 无界 fetch 导致自动部署长期占用 mutation queue -- 状态:resolved +- 状态:investigating - 首次发现:2026-08-06 - 最近更新:2026-08-06 - 影响面:Gitea staging deploy/migration 控制器的 trusted-main checkout;production 与 staging 应用数据面未受影响。 -- 用户现象:exact-SHA quality gate run `1473` 成功后,自动 deploy run `1474` 在 `git fetch --no-tags origin main "$DEPLOY_SHA"` 长时间没有日志进展;期间 staging 公网与 `.state/deployed-revision` 均保持上一健康 SHA,SSH/远端 mutation 尚未开始。fetch 后续自行恢复,run 最终于 18 分钟完成并成功部署 `02cc483b7c303e6cc0f26fb31462c50adb007f12`,5 个容器 restart count 均为 0。 -- 触发条件:Gitea runner 在空仓库中抓取 reviewed `main` 和目标 SHA 时传输长时间停滞;底层远端/网络原因未闭环。checkout step 没有命令级 timeout/retry,只依赖 30 分钟 job 总超时。 -- 根因:已确认的控制面根因是发布控制器对镜像拉取有 bounded retry,却对 trusted-main Git fetch 采用无界单次调用;无论底层传输为何停滞,该调用都会长期占用 `staging-mutation` queue,并推迟后续受控重试。底层 Git 传输停滞原因保持 blocked,不作推断。 -- 修复:Gitea deploy 与 migration 的 trusted-main fetch 改为最多 3 次、每次 120 秒的 bounded retry,重试间隔递增;耗尽后明确 fail closed。仍只 checkout `origin/main`,仍要求目标 SHA 是 reviewed main ancestor,不改变 exact-SHA artifact、forward-only 或 manual rollback 边界。 -- 验证:原 run `1474` 最终成功部署 `02cc483b7c303e6cc0f26fb31462c50adb007f12`,公网 health、主机 state 与 `main/staging` 精确一致,5 个容器 restart count 均为 0;本地 staging workflow contracts 31/31、修改后 YAML 解析、mandatory pre-work 和 diff 检查通过;Gitea PR quality gate run `1475` 成功。修复后 exact-SHA staging 控制面部署仍按正常发布流程单独核对,不将原 run 的自行恢复冒充为 bounded retry 已在真实故障中触发。 -- 防复发:所有 release-controller 网络调用必须同时具备命令级上限和失败闭合;不得仅依赖 job 总 timeout。回归测试必须同时覆盖 deploy 和 migration 的 attempt 数、单次 timeout、最终错误和 ancestry check。 +- 用户现象:exact-SHA quality gate run `1473` 成功后,自动 deploy run `1474` 在 `git fetch --no-tags origin main "$DEPLOY_SHA"` 长时间没有日志进展;fetch 后续自行恢复,run 最终于 18 分钟成功部署 `02cc483b7c303e6cc0f26fb31462c50adb007f12`。第一轮 bounded-retry 修复合入后,run `1480` 的 3 次 120 秒 fetch 全部在服务端压缩 16,093 个对象时耗尽并 fail closed;SSH/远端 mutation 未开始,公网/state 继续健康运行 `02cc483b7c303e6cc0f26fb31462c50adb007f12`。 +- 触发条件:空仓库命令 `git fetch --no-tags origin main "$DEPLOY_SHA"` 同时请求分支和目标 SHA,导致 Gitea 为每次尝试枚举/压缩完整历史对象;runner 与服务端之间的传输无法在 120 秒内完成。 +- 根因:原控制器既没有命令级 timeout,也错误地为正常前向发布抓取 full-history dual ref。第一轮修复只增加 bounded retry,解决了无界占用,但旧回归测试只断言 timeout/attempt/ancestry,未限制传输对象范围,因而未拦住连续三次重新打包完整历史。 +- 修复:不再让 mutation runner 做任何 Git object fetch。成功 staging gate 从其已验证的 exact SHA 生成仅含 tracked `deploy/` 与严格 manifest validator 的 `controller.tar`,将 tar SHA-256 写入四字段 manifest,并与 immutable image digests 一起上传。deploy/migration 从 exact successful gate artifact 下载 bundle,强制校验 controller SHA、tar hash、路径、重复项、类型和 2 MiB 上限后才解包;正常发布使用当前 `main == staging` controller,手工旧版 rollback 也不得执行旧 controller。refs 与 forward/rollback 关系通过有界 Gitea API 和完整 commit-DAG 路径证明,字段缺失、分页不完整、头不一致或证据冲突均 fail closed。 +- 验证:第一轮 bounded retry 的本地 workflow contracts 31/31、PR gates `1475/1477` 与 staging gate `1479` 成功;run `1480` 证明 3 次 120 秒耗尽后无半部署。bundle 修复本地 manifest/workflow contracts 34/34、三份 YAML、shell、真实 25-entry/122,880-byte controller tar hash/安全检查、mutation Git-object-op=0、mandatory pre-work 和 diff 检查通过。完整 PR gate、staging gate 和 exact-SHA deploy 待完成;完成前不得再次标记 resolved。 +- 防复发:所有 release-controller 网络调用必须有命令级上限和失败闭合;mutation workflow 禁止 `git fetch/ls-remote/cat-file/merge-base/checkout/init`。控制器必须来自 exact successful gate 的 hash-bound artifact,正常与 rollback 均使用当前 reviewed controller;测试必须覆盖 artifact identity、tar safety、commit-DAG proof 和旧 Git object 路径为零。 - 相关记录:BUG-128、ERR-094、ERR-095 -- 复发自:无 -- 修复版本:`52b467cbe56f820a6acc19bb3fe3ebbf4ad55776`;待合并后由 staging exact-SHA 发布采用 +- 复发自:BUG-129 第一轮修复未覆盖对象范围 +- 修复版本:待 gate-attested controller bundle 与 staging 验收 diff --git a/docs/research/pre_work_error_ledger.md b/docs/research/pre_work_error_ledger.md index ca331832..bb549935 100644 --- a/docs/research/pre_work_error_ledger.md +++ b/docs/research/pre_work_error_ledger.md @@ -145,11 +145,11 @@ A failed exact-SHA staging deploy displayed the multiline staging SSH private ke Prevention: store `STAGING_SSH_PRIVATE_KEY` only as one unwrapped base64 line; workflows decode it into a mode-`0600` temporary key, validate it with `ssh-keygen`, and delete the temporary directory on every exit. Contract tests must reject direct multiline `SSH_PRIVATE_KEY` injection or `printf` of a decoded secret variable. A leaked staging key must be rotated and revoked before any rerun; production keys remain a separate boundary and were not involved in this incident. -## ERR-095 | Gitea trusted-main fetch can stall until the whole deploy job times out | mitigated 2026-08-06 +## ERR-095 | Gitea trusted-main full-history fetch stalls or exhausts every bounded attempt | investigating 2026-08-06 -After exact-SHA staging gate `1473` succeeded, automatic deploy `1474` stopped making log progress for an extended period in the empty-repository `git fetch --no-tags origin main "$DEPLOY_SHA"` step before any SSH or staging mutation. Public and state SHAs remained on the previous healthy release during the stall. The fetch later recovered and the 18-minute run successfully deployed the exact SHA, but the checkout had no command-level bound; without recovery, the 30-minute job timeout was the only escape and the shared mutation queue would remain occupied. The underlying Git transport stall is not proven. +After exact-SHA staging gate `1473` succeeded, automatic deploy `1474` stopped making log progress for an extended period in the empty-repository `git fetch --no-tags origin main "$DEPLOY_SHA"` step before any SSH or staging mutation. The fetch later recovered and the 18-minute run successfully deployed the exact SHA. The first mitigation added three 120-second attempts, but run `1480` proved every attempt still asked Gitea to enumerate/compress 16,093 full-history objects and then timed out. It failed closed before SSH; public and state SHAs remained on the prior healthy release. The first fix bounded queue occupation but did not reduce the transfer, and its regression test did not reject the full-history dual-ref form. -Prevention: wrap the Gitea deploy and migration trusted-main fetch in three 120-second bounded attempts with incremental delay and explicit fail-closed exhaustion. Preserve the reviewed-main ancestry check, exact-SHA gate artifact, forward-only deploy policy, and shared mutation queue. Never treat an in-progress or timed-out fetch as a deployment result. +Prevention: Gitea mutation workflows must perform no Git object operations. A successful staging gate packages its already-verified exact-SHA `deploy/` controller plus manifest validator into `controller.tar`, binds its SHA-256 into the strict image manifest, and uploads both as one immutable artifact. Deploy/migration must verify artifact run/SHA, controller digest, archive paths/types/duplicates/size, current `main == staging` refs, and a complete Gitea compare commit-DAG path before mutation; any missing or inconsistent evidence fails closed. Manual rollback still uses the current reviewed controller, never the old target's controller. Preserve exact-SHA images, forward-only defaults, shared mutation lock, and bounded API/artifact requests. ## Fragment Sweep Command Set diff --git a/frontend/scripts/staging-image-manifest.mjs b/frontend/scripts/staging-image-manifest.mjs index 767c5644..f0db7075 100644 --- a/frontend/scripts/staging-image-manifest.mjs +++ b/frontend/scripts/staging-image-manifest.mjs @@ -4,7 +4,8 @@ import { pathToFileURL } from "node:url"; const shaPattern = /^[0-9a-f]{40}$/; const digestPattern = /^sha256:[0-9a-f]{64}$/; -const expectedKeys = ["git_sha", "api_digest", "web_digest"]; +const requiredKeys = ["git_sha", "api_digest", "web_digest"]; +const optionalKeys = ["controller_sha256"]; const defaultRegistry = "ghcr.io/jesse-ux"; const acrRepository = "crpi-d1feco6itet73spp.cn-hongkong.personal.cr.aliyuncs.com/copse/jyotisha"; const registryPattern = /^(?:[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?)(?::[1-9][0-9]{0,4})?(?:\/[a-z0-9]+(?:[._-][a-z0-9]+)*)*$/; @@ -18,7 +19,7 @@ export function parseStagingImageManifest(text, expectedSha, registry = defaultR } const lines = text.endsWith("\n") ? text.slice(0, -1).split("\n") : text.split("\n"); - if (lines.length !== expectedKeys.length) { + if (lines.length !== requiredKeys.length && lines.length !== requiredKeys.length + 1) { throw new Error("invalid staging image manifest"); } @@ -28,11 +29,14 @@ export function parseStagingImageManifest(text, expectedSha, registry = defaultR if (separator <= 0) throw new Error("invalid staging image manifest"); const key = line.slice(0, separator); const value = line.slice(separator + 1); - if (!expectedKeys.includes(key) || values.has(key)) { + if (![...requiredKeys, ...optionalKeys].includes(key) || values.has(key)) { throw new Error("invalid staging image manifest"); } values.set(key, value); } + if (!requiredKeys.every((key) => values.has(key))) { + throw new Error("invalid staging image manifest"); + } if (values.get("git_sha") !== expectedSha) { throw new Error("staging image manifest revision mismatch"); @@ -42,12 +46,17 @@ export function parseStagingImageManifest(text, expectedSha, registry = defaultR throw new Error("invalid staging image digest"); } } + const controllerSha256 = values.get("controller_sha256"); + if (controllerSha256 !== undefined && !/^[0-9a-f]{64}$/.test(controllerSha256)) { + throw new Error("invalid staging controller digest"); + } const sharedRepository = registry === acrRepository; return { gitSha: expectedSha, apiDigest: values.get("api_digest"), webDigest: values.get("web_digest"), + ...(controllerSha256 === undefined ? {} : { controllerSha256 }), apiImage: `${sharedRepository ? registry : `${registry}/jyotisha-api`}@${values.get("api_digest")}`, webImage: `${sharedRepository ? registry : `${registry}/jyotisha-web`}@${values.get("web_digest")}`, }; @@ -73,6 +82,9 @@ if (invokedPath === import.meta.url) { `git_sha=${manifest.gitSha}`, `api_image=${manifest.apiImage}`, `web_image=${manifest.webImage}`, + ...(manifest.controllerSha256 === undefined + ? [] + : [`controller_sha256=${manifest.controllerSha256}`]), ].join("\n") + "\n", ); } catch { diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index c34a9500..50dee9a9 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -212,9 +212,12 @@ test("Gitea quality gate validates before publishing an immutable ACR manifest", assert.match(workflow, /xs=d if isinstance\(d,list\) else \[d\]/); assert.match(workflow, /get\("os"\)=="linux"/); assert.match(workflow, /get\("architecture"\)=="amd64"/); + assert.match(workflow, /git archive --format=tar --output artifacts\/staging-images\/controller\.tar/); + assert.match(workflow, /controller_sha256="\$\(sha256sum artifacts\/staging-images\/controller\.tar/); + assert.match(workflow, /controller_sha256=%s/); assert.match(workflow, /node frontend\/scripts\/staging-image-manifest\.mjs/); assert.match(workflow, /require\("\.\/\.gitea\/actions\/upload-artifact\/dist\/index\.js"\)/); - assert.match(workflow, /--env INPUT_PATH=artifacts\/staging-images\/manifest\.env/); + assert.match(workflow, /--env INPUT_PATH=artifacts\/staging-images\//); assert.match(workflow, /process\.env\["INPUT_IF-NO-FILES-FOUND"\]="error"/); assert.match(workflow, /process\.env\["INPUT_RETENTION-DAYS"\]="30"/); assert.match(workflow, /process\.env\["INPUT_COMPRESSION-LEVEL"\]="6"/); @@ -251,14 +254,14 @@ test("Gitea staging mutation workflows use the available runner and pinned Node assert.match(workflow, /--workdir "\$workdir"/); assert.match(workflow, /node:22-bookworm-slim "\$\{0##\*\/\}" "\$@"/); assert.match(workflow, />> "\$GITHUB_PATH"/); - assert.match(workflow, /node frontend\/scripts\/staging-image-manifest\.mjs/); + assert.match(workflow, /artifacts\/(?:controller|staging-image)\/extracted\/frontend\/scripts\/staging-image-manifest\.mjs/); assert.doesNotMatch(workflow, /packages\+=\((?:nodejs|npm)\)|apt-get install[^\n]*(?:nodejs|npm)/); assert.doesNotMatch(workflow, /(?:--volume|-v)[^\n]*(?:\$HOME\/\.docker|DOCKER_CONFIG)/); assertOrder(workflow, [ - "Checkout trusted main controller", "Prepare pinned Node tooling", - "Download gate-produced", - "node frontend/scripts/staging-image-manifest.mjs", + "Download ", + "controller_sha256", + "extracted/frontend/scripts/staging-image-manifest.mjs", ]); } }); @@ -476,7 +479,7 @@ test("main remains the trusted GitHub deployment controller", () => { } }); -test("Gitea deploy and migration consume the exact successful gate artifact", () => { +test("Gitea deploy and migration consume exact gate-attested controller bundles", () => { for (const workflow of [read(giteaDeployWorkflow), read(giteaMigrationWorkflow)]) { assert.match(workflow, /actions\/runs\?head_sha=\$[A-Z_]+&branch=staging&event=push&status=success/); assert.match(workflow, /\.path \| split\("@"\)\[0\] \| endswith\("backend-quality-gate\.yml"\)/); @@ -484,44 +487,34 @@ test("Gitea deploy and migration consume the exact successful gate artifact", () assert.match(workflow, /\.event == "push"/); assert.match(workflow, /\.conclusion == "success"/); assert.match(workflow, /sort_by\(\.id\) \| reverse \| first/); - assert.match(workflow, /actions\/runs\/\$GATE_RUN_ID\/artifacts\?limit=100/); - assert.match(workflow, /actions\/artifacts\/\$artifact_id\/zip/); - assert.match(workflow, /node frontend\/scripts\/staging-image-manifest\.mjs/); - assert.match(workflow, /name: Checkout trusted main controller/); - assert.match(workflow, /for attempt in 1 2 3/); - assert.match(workflow, /timeout 120 git fetch --no-tags origin main "\$DEPLOY_SHA"/); - assert.match(workflow, /trusted main fetch failed after \$attempt bounded attempts/); - assert.match(workflow, /\[\[ "\$fetch_succeeded" == true \]\]/); - assert.match(workflow, /git checkout --detach --force origin\/main/); - assert.match(workflow, /git merge-base --is-ancestor "\$DEPLOY_SHA" HEAD/); + assert.match(workflow, /actions\/runs\/\$(?:run_id|GATE_RUN_ID)\/artifacts\?limit=100/); + assert.match(workflow, /actions\/artifacts\/\$(?:id|artifact_id)\/zip/); + assert.match(workflow, /controller_sha256/); + assert.match(workflow, /sha256sum --check --status/); + assert.match(workflow, /allowed = \{"manifest\.env", "controller\.tar"\}/); + assert.match(workflow, /unsafe staging artifact path/); + assert.match(workflow, /unsafe staging artifact type/); + assert.match(workflow, /3 \* 1024 \* 1024/); + assert.match(workflow, /unsafe staging controller bundle/); + assert.match(workflow, /extracted\/frontend\/scripts\/staging-image-manifest\.mjs/); + assert.doesNotMatch(workflow, /\bgit (?:fetch|ls-remote|cat-file|merge-base|checkout|init)\b/); assert.doesNotMatch(workflow, /docker manifest inspect/); assert.doesNotMatch(workflow, /\$IMAGE_REPOSITORY:(?:api|web)-\$DEPLOY_SHA/); } }); -test("Gitea staging mutations resolve the manifest from actual gate-run artifacts", () => { +test("Gitea staging mutations resolve bundles from actual gate-run artifacts", () => { for (const workflow of [read(giteaDeployWorkflow), read(giteaMigrationWorkflow)]) { - assert.match( - workflow, - /artifact_prefix="staging-image-manifest-\$DEPLOY_SHA-"/, - ); + assert.match(workflow, /staging-image-manifest-\$(?:sha|DEPLOY_SHA)-/); assert.match(workflow, /\.expired == false/); assert.match(workflow, /\.name \| startswith\(\$prefix\)/); assert.match(workflow, /\.name \| ltrimstr\(\$prefix\)/); assert.match(workflow, /tonumber\?/); assert.match(workflow, /sort_by\(\.attempt, \.id\) \| reverse \| first/); - assert.match(workflow, /artifact_name="\$\(jq -er '\.name'/); - assert.match(workflow, /artifact_id="\$\(jq -er '\.id'/); - assert.match(workflow, /artifact_attempt="\$\{artifact_name#"\$artifact_prefix"\}"/); - assert.match(workflow, /\[\[ "\$artifact_name" == "\$artifact_prefix"\* \]\]/); - assert.match(workflow, /\[\[ "\$artifact_attempt" =~ \^\[1-9\]\[0-9\]\*\$ \]\]/); - assertOrder(workflow, [ - 'artifact_prefix="staging-image-manifest-$DEPLOY_SHA-"', - 'actions/runs/$GATE_RUN_ID/artifacts?limit=100', - 'sort_by(.attempt, .id) | reverse | first', - 'artifact_name="$(jq -er', - 'actions/artifacts/$artifact_id/zip', - ]); + assert.match(workflow, /jq -er '\.name'/); + assert.match(workflow, /jq -er '\.id'/); + assert.match(workflow, /actions\/artifacts\/\$(?:id|artifact_id)\/zip/); + assert.match(workflow, /--connect-timeout 15 --max-time (?:60|120) --retry 3 --retry-all-errors/); assert.doesNotMatch(workflow, /run_attempt/i); assert.doesNotMatch(workflow, /staging-image-manifest-\$DEPLOY_SHA-1/); } @@ -538,8 +531,14 @@ test("Gitea deployment follows only a successful staging push gate and keeps rol assert.match(workflow, /concurrency:\n\s+group: staging-mutation\n\s+cancel-in-progress: false\n\s+queue: max/); assert.match(workflow, /rollback authorization is manual-only/); assert.match(workflow, /stale staging revision refused/); + assert.match(workflow, /ALLOW_ROLLBACK: \$\{\{ steps\.revision\.outputs\.allow_rollback \}\}/); + assert.match(workflow, /reviewed main and staging controller heads differ/); + assert.match(workflow, /rollback revision is not in reviewed main history/); + assert.match(workflow, /controller_gate_run_id/); + assert.match(workflow, /def reaches\(\$sha; \$seen\)/); + assert.match(workflow, /\.total_commits == \(\$commits \| length\)/); assert.match(workflow, /staging advanced during deployment; refusing stale mutation/); - assert.match(workflow, /git merge-base --is-ancestor "\$previous_sha" "\$DEPLOY_SHA"/); + assert.match(workflow, /automatic staging rollback or divergent deploy refused/); assert.match(workflow, /API_IMAGE: \$\{\{ steps\.images\.outputs\.api_image \}\}/); assert.match(workflow, /WEB_IMAGE: \$\{\{ steps\.images\.outputs\.web_image \}\}/); }); @@ -551,6 +550,8 @@ test("Gitea migration remains manual and consumes only the gate-pinned web image assert.doesNotMatch(workflow, /workflow_run:|\n\s+push:/); assert.match(workflow, /concurrency:\n\s+group: staging-mutation\n\s+cancel-in-progress: false\n\s+queue: max/); assert.match(workflow, /migration requires current staging head/); + assert.match(workflow, /staging migration revision must equal reviewed main head/); + assert.doesNotMatch(workflow, /--deepen=/); assert.match(workflow, /staging advanced during migration; refusing stale mutation/); assert.match(workflow, /WEB_IMAGE: \$\{\{ steps\.image\.outputs\.web_image \}\}/); assert.doesNotMatch(workflow, /API_IMAGE:/); diff --git a/frontend/tests/staging-image-manifest.test.ts b/frontend/tests/staging-image-manifest.test.ts index 4494002e..32e9ed92 100644 --- a/frontend/tests/staging-image-manifest.test.ts +++ b/frontend/tests/staging-image-manifest.test.ts @@ -5,8 +5,9 @@ import { parseStagingImageManifest } from "../scripts/staging-image-manifest.mjs const gitSha = "0123456789abcdef0123456789abcdef01234567"; const apiDigest = `sha256:${"a".repeat(64)}`; const webDigest = `sha256:${"b".repeat(64)}`; +const controllerSha256 = "c".repeat(64); -function validManifest(): string { +function legacyManifest(): string { return [ `git_sha=${gitSha}`, `api_digest=${apiDigest}`, @@ -15,11 +16,29 @@ function validManifest(): string { ].join("\n"); } +function validManifest(): string { + return legacyManifest().replace( + `web_digest=${webDigest}\n`, + `web_digest=${webDigest}\ncontroller_sha256=${controllerSha256}\n`, + ); +} + +test("legacy image-only manifest remains valid for the GitHub artifact contract", () => { + assert.deepEqual(parseStagingImageManifest(legacyManifest(), gitSha), { + gitSha, + apiDigest, + webDigest, + apiImage: `ghcr.io/jesse-ux/jyotisha-api@${apiDigest}`, + webImage: `ghcr.io/jesse-ux/jyotisha-web@${webDigest}`, + }); +}); + test("manifest produces immutable GHCR digest references", () => { assert.deepEqual(parseStagingImageManifest(validManifest(), gitSha), { gitSha, apiDigest, webDigest, + controllerSha256, apiImage: `ghcr.io/jesse-ux/jyotisha-api@${apiDigest}`, webImage: `ghcr.io/jesse-ux/jyotisha-web@${webDigest}`, }); @@ -32,6 +51,7 @@ test("manifest produces immutable shared ACR repository references", () => { gitSha, apiDigest, webDigest, + controllerSha256, apiImage: `${repository}@${apiDigest}`, webImage: `${repository}@${webDigest}`, }); @@ -42,6 +62,8 @@ test("manifest rejects revision drift, mutable tags, duplicates, extras, and mal validManifest().replace(gitSha, "f".repeat(40)), validManifest().replace(apiDigest, `${gitSha}`), validManifest().replace(apiDigest, `sha256:${"A".repeat(64)}`), + validManifest().replace(controllerSha256, "C".repeat(64)), + validManifest().replace(controllerSha256, "c".repeat(63)), validManifest().replace( `web_digest=${webDigest}`, `api_digest=${apiDigest}`,