Jesse_Chen
399 lines
17 KiB
YAML
399 lines
17 KiB
YAML
name: Independent Staging Quality Gate
|
|
|
|
on:
|
|
pull_request:
|
|
paths:
|
|
- '.gitea/workflows/backend-quality-gate.yml'
|
|
- '.gitea/workflows/deploy-staging.yml'
|
|
- '.gitea/workflows/migrate-staging-database.yml'
|
|
- '.gitea/workflows/migrate-production-database.yml'
|
|
- '.gitea/workflows/create-production-recovery.yml'
|
|
- 'deploy/**'
|
|
- 'frontend/**'
|
|
- 'jyotish_vedic/**'
|
|
- 'scripts/**'
|
|
- 'tests/**'
|
|
- 'mcp_server.py'
|
|
- 'pyproject.toml'
|
|
- 'requirements*.txt'
|
|
push:
|
|
branches: [staging]
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: staging-quality-${{ gitea.ref }}
|
|
cancel-in-progress: true
|
|
|
|
permissions:
|
|
contents: read
|
|
actions: write
|
|
|
|
jobs:
|
|
validate:
|
|
runs-on: manman-linux
|
|
timeout-minutes: 45
|
|
env:
|
|
GITEA_SHA: ${{ gitea.sha }}
|
|
GITEA_EVENT_NAME: ${{ gitea.event_name }}
|
|
NODE_TOOL_SOURCE_IMAGE: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/library/node:22-bookworm-slim@sha256:ef343465b6a14bbdf2ab52f6e100ec0659a792464fcf72c462370d88b3df909c
|
|
NODE_TOOL_IMAGE: node:22-bookworm-slim
|
|
steps:
|
|
- name: Checkout exact Gitea revision
|
|
run: |
|
|
set -euo pipefail
|
|
[[ "$GITEA_SHA" =~ ^[0-9a-f]{40}$ ]]
|
|
git init .
|
|
git remote remove origin 2>/dev/null || true
|
|
git remote add origin https://git.copse.top/root/Jyotisha.git
|
|
fetch_succeeded=false
|
|
for attempt in 1 2 3; do
|
|
if timeout 300 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 \
|
|
fetch --depth=1 --no-tags origin "$GITEA_SHA"; then
|
|
fetch_succeeded=true
|
|
break
|
|
fi
|
|
if [ "$attempt" -eq 3 ]; then
|
|
echo "exact staging gate checkout failed after $attempt bounded attempts" >&2
|
|
exit 1
|
|
fi
|
|
sleep $((attempt * 10))
|
|
done
|
|
[[ "$fetch_succeeded" == true ]]
|
|
git checkout --detach --force "$GITEA_SHA"
|
|
git clean -ffdx
|
|
test "$(git rev-parse HEAD)" = "$GITEA_SHA"
|
|
test -z "$(git status --porcelain --untracked-files=all)"
|
|
|
|
- name: Install and verify Linux runner toolchain
|
|
run: |
|
|
set -euo pipefail
|
|
packages=()
|
|
if ! docker compose version --short 2>/dev/null | grep -Eq '^v?2\.'; then
|
|
packages+=(docker-compose-v2)
|
|
fi
|
|
venv_probe="$(mktemp -d "${RUNNER_TEMP:-/tmp}/jyotisha-venv-probe.XXXXXX")"
|
|
if ! python3 -m venv "$venv_probe/venv" >/dev/null 2>&1; then
|
|
packages+=(python3-venv)
|
|
fi
|
|
rm -rf "$venv_probe"
|
|
if ! python3 -c 'import pathlib, sysconfig; assert pathlib.Path(sysconfig.get_path("include"), "Python.h").is_file()' >/dev/null 2>&1; then
|
|
packages+=(python3-dev)
|
|
fi
|
|
if ! command -v g++ >/dev/null 2>&1; then
|
|
packages+=(g++)
|
|
fi
|
|
if ! command -v rsync >/dev/null 2>&1; then
|
|
packages+=(rsync)
|
|
fi
|
|
if [ "${#packages[@]}" -gt 0 ]; then
|
|
apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 \
|
|
-o Acquire::https::Timeout=30 update
|
|
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
|
"${packages[@]}"
|
|
fi
|
|
python3 --version
|
|
curl --version
|
|
git --version
|
|
openssl version
|
|
docker version
|
|
docker compose version
|
|
docker compose version --short | grep -Eq '^v?2\.'
|
|
docker compose --help | grep -q -- '--project-name'
|
|
|
|
- name: Prepare pinned Node tooling
|
|
run: |
|
|
set -euo pipefail
|
|
if ! docker image inspect "$NODE_TOOL_SOURCE_IMAGE" >/dev/null 2>&1; then
|
|
for attempt in 1 2 3; do
|
|
if timeout 180 docker pull "$NODE_TOOL_SOURCE_IMAGE"; then
|
|
break
|
|
fi
|
|
if [ "$attempt" -eq 3 ]; then
|
|
echo "Failed to preload $NODE_TOOL_IMAGE after $attempt attempts" >&2
|
|
exit 1
|
|
fi
|
|
sleep $((attempt * 15))
|
|
done
|
|
fi
|
|
docker tag "$NODE_TOOL_SOURCE_IMAGE" "$NODE_TOOL_IMAGE"
|
|
docker image inspect "$NODE_TOOL_IMAGE" >/dev/null
|
|
tool_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/jyotisha-node-tools.XXXXXX")"
|
|
container_id="$(docker create "$NODE_TOOL_IMAGE")"
|
|
trap 'docker rm -f "$container_id" >/dev/null 2>&1 || true' EXIT
|
|
docker cp "$container_id:/usr/local/bin/node" "$tool_dir/node"
|
|
docker cp "$container_id:/usr/local/lib/node_modules/npm" "$tool_dir/npm-package"
|
|
docker rm "$container_id" >/dev/null
|
|
trap - EXIT
|
|
ln -s "$tool_dir/npm-package/bin/npm-cli.js" "$tool_dir/npm"
|
|
chmod 0755 "$tool_dir/node" "$tool_dir/npm-package/bin/npm-cli.js"
|
|
test -n "${GITHUB_PATH:-}"
|
|
printf '%s\n' "$tool_dir" >> "$GITHUB_PATH"
|
|
export PATH="$tool_dir:$PATH"
|
|
node --version | grep -Eq '^v22\.'
|
|
npm --version
|
|
|
|
- name: Preload PostgreSQL integration image
|
|
env:
|
|
POSTGRES_TEST_SOURCE_IMAGE: public.ecr.aws/docker/library/postgres:17-alpine@sha256:742f40ea20b9ff2ff31db5458d127452988a2164df9e17441e191f3b72252193
|
|
POSTGRES_TEST_IMAGE: postgres:17-alpine
|
|
run: |
|
|
set -euo pipefail
|
|
if ! docker image inspect "$POSTGRES_TEST_IMAGE" >/dev/null 2>&1; then
|
|
for attempt in 1 2 3; do
|
|
if timeout 180 docker pull "$POSTGRES_TEST_SOURCE_IMAGE"; then
|
|
docker tag "$POSTGRES_TEST_SOURCE_IMAGE" "$POSTGRES_TEST_IMAGE"
|
|
break
|
|
fi
|
|
if [ "$attempt" -eq 3 ]; then
|
|
echo "Failed to preload $POSTGRES_TEST_IMAGE after $attempt attempts" >&2
|
|
exit 1
|
|
fi
|
|
sleep $((attempt * 15))
|
|
done
|
|
fi
|
|
docker image inspect "$POSTGRES_TEST_IMAGE" >/dev/null
|
|
|
|
- name: Install dependencies
|
|
env:
|
|
PIP_INDEX_URL: https://mirrors.aliyun.com/pypi/simple/
|
|
NPM_CONFIG_REGISTRY: https://registry.npmmirror.com
|
|
run: |
|
|
set -euo pipefail
|
|
python3 -m venv .venv
|
|
export PATH="$PWD/.venv/bin:$PATH"
|
|
python -m pip install --upgrade pip
|
|
python -m pip install \
|
|
"mcp==1.28.1" \
|
|
"pydantic==2.13.4" \
|
|
"numpy==2.5.1" \
|
|
"pandas==2.3.3" \
|
|
"timezonefinder==8.2.5" \
|
|
-r requirements.txt -r requirements-dev.txt
|
|
workdir="$(pwd -P)"
|
|
set +e
|
|
docker run --rm \
|
|
--cpus=1.5 \
|
|
--memory=2g \
|
|
--memory-swap=2g \
|
|
--pids-limit=256 \
|
|
--user "$(id -u):$(id -g)" \
|
|
--volume "$workdir:$workdir" \
|
|
--workdir "$workdir" \
|
|
--env HOME=/tmp \
|
|
--env "NPM_CONFIG_REGISTRY=$NPM_CONFIG_REGISTRY" \
|
|
"$NODE_TOOL_SOURCE_IMAGE" \
|
|
timeout --signal=TERM --kill-after=30s 900s \
|
|
npm ci --prefix frontend \
|
|
--no-audit \
|
|
--no-fund \
|
|
--progress=false \
|
|
--maxsockets=4 \
|
|
--fetch-timeout=60000 \
|
|
--fetch-retries=2 \
|
|
--fetch-retry-mintimeout=1000 \
|
|
--fetch-retry-maxtimeout=10000
|
|
npm_ci_status=$?
|
|
set -e
|
|
if [ "$npm_ci_status" -eq 124 ]; then
|
|
echo "frontend npm ci exceeded bounded 900-second timeout; check npm mirror/network or dependency postinstall hang" >&2
|
|
exit 124
|
|
fi
|
|
if [ "$npm_ci_status" -ne 0 ]; then
|
|
echo "frontend npm ci failed with status $npm_ci_status inside bounded Node container" >&2
|
|
exit "$npm_ci_status"
|
|
fi
|
|
|
|
- name: Validate backend, package, frontend, and database contracts
|
|
run: |
|
|
set -euo pipefail
|
|
export PATH="$PWD/.venv/bin:$PATH"
|
|
ruff check scripts/run_quality_gate.py tests/test_varga_bphs.py \
|
|
tests/test_ashtakavarga_invariants.py tests/test_cli_smoke.py \
|
|
tests/test_yoga_rules_integrity.py
|
|
python -m py_compile scripts/*.py jyotish_vedic/*.py mcp_server.py
|
|
python scripts/run_quality_gate.py \
|
|
--profile quick --skip-yoga-logic --skip-frontend-runtime
|
|
python scripts/commercial_privacy_artifact_scan.py --json
|
|
python -m build
|
|
npm test --prefix frontend
|
|
npm run lint --prefix frontend
|
|
if [ "$GITEA_EVENT_NAME" != "push" ]; then
|
|
if ! timeout 600 npm run build --prefix frontend -- --webpack; then
|
|
echo "frontend production build exceeded bounded 600-second timeout" >&2
|
|
exit 124
|
|
fi
|
|
else
|
|
echo "staging push production build is verified once by the publish image build"
|
|
fi
|
|
|
|
publish:
|
|
if: gitea.event_name == 'push' && gitea.ref == 'refs/heads/staging'
|
|
needs: validate
|
|
runs-on: manman-linux
|
|
timeout-minutes: 60
|
|
env:
|
|
GITEA_SHA: ${{ gitea.sha }}
|
|
GITEA_RUN_ATTEMPT: ${{ gitea.run_attempt }}
|
|
GITEA_API_URL: ${{ gitea.api_url }}
|
|
GITEA_REPOSITORY: ${{ gitea.repository }}
|
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
|
REGISTRY_HOST: crpi-d1feco6itet73spp.cn-hongkong.personal.cr.aliyuncs.com
|
|
IMAGE_REPOSITORY: crpi-d1feco6itet73spp.cn-hongkong.personal.cr.aliyuncs.com/copse/jyotisha
|
|
steps:
|
|
- name: Checkout exact Gitea revision
|
|
run: |
|
|
set -euo pipefail
|
|
[[ "$GITEA_SHA" =~ ^[0-9a-f]{40}$ ]]
|
|
git init .
|
|
git remote remove origin 2>/dev/null || true
|
|
git remote add origin https://git.copse.top/root/Jyotisha.git
|
|
fetch_succeeded=false
|
|
for attempt in 1 2 3; do
|
|
if timeout 300 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 \
|
|
fetch --depth=1 --no-tags origin "$GITEA_SHA"; then
|
|
fetch_succeeded=true
|
|
break
|
|
fi
|
|
if [ "$attempt" -eq 3 ]; then
|
|
echo "exact staging gate checkout failed after $attempt bounded attempts" >&2
|
|
exit 1
|
|
fi
|
|
sleep $((attempt * 10))
|
|
done
|
|
[[ "$fetch_succeeded" == true ]]
|
|
git checkout --detach --force "$GITEA_SHA"
|
|
git clean -ffdx
|
|
test "$(git rev-parse HEAD)" = "$GITEA_SHA"
|
|
test -z "$(git status --porcelain --untracked-files=all)"
|
|
|
|
- name: Prepare pinned Node tooling
|
|
env:
|
|
NODE_TOOL_SOURCE_IMAGE: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/library/node:22-bookworm-slim@sha256:ef343465b6a14bbdf2ab52f6e100ec0659a792464fcf72c462370d88b3df909c
|
|
NODE_TOOL_IMAGE: node:22-bookworm-slim
|
|
run: |
|
|
set -euo pipefail
|
|
if ! docker image inspect "$NODE_TOOL_SOURCE_IMAGE" >/dev/null 2>&1; then
|
|
for attempt in 1 2 3; do
|
|
if timeout 180 docker pull "$NODE_TOOL_SOURCE_IMAGE"; then
|
|
break
|
|
fi
|
|
if [ "$attempt" -eq 3 ]; then
|
|
echo "Failed to preload $NODE_TOOL_IMAGE after $attempt attempts" >&2
|
|
exit 1
|
|
fi
|
|
sleep $((attempt * 15))
|
|
done
|
|
fi
|
|
docker tag "$NODE_TOOL_SOURCE_IMAGE" "$NODE_TOOL_IMAGE"
|
|
docker image inspect "$NODE_TOOL_IMAGE" >/dev/null
|
|
tool_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/jyotisha-node-tools.XXXXXX")"
|
|
cat > "$tool_dir/node" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
workdir="$(pwd -P)"
|
|
exec docker run --rm \
|
|
--user "$(id -u):$(id -g)" \
|
|
--volume "$workdir:$workdir" \
|
|
--workdir "$workdir" \
|
|
--env HOME=/tmp \
|
|
node:22-bookworm-slim "${0##*/}" "$@"
|
|
EOF
|
|
chmod 0755 "$tool_dir/node"
|
|
ln -s node "$tool_dir/npm"
|
|
test -n "${GITHUB_PATH:-}"
|
|
printf '%s\n' "$tool_dir" >> "$GITHUB_PATH"
|
|
export PATH="$tool_dir:$PATH"
|
|
node --version
|
|
npm --version
|
|
|
|
- name: Build and publish exact-SHA ACR images
|
|
env:
|
|
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
|
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
|
run: |
|
|
set -euo pipefail
|
|
printf '%s' "$REGISTRY_PASSWORD" | docker login "$REGISTRY_HOST" --username "$REGISTRY_USERNAME" --password-stdin
|
|
docker build -f deploy/railway-api.Dockerfile -t "$IMAGE_REPOSITORY:api-$GITEA_SHA" .
|
|
docker build -f deploy/railway-web.Dockerfile -t "$IMAGE_REPOSITORY:web-$GITEA_SHA" .
|
|
docker push "$IMAGE_REPOSITORY:api-$GITEA_SHA"
|
|
docker push "$IMAGE_REPOSITORY:web-$GITEA_SHA"
|
|
|
|
- name: Record immutable linux-amd64 image manifest
|
|
run: |
|
|
set -euo pipefail
|
|
[[ "$GITEA_SHA" =~ ^[0-9a-f]{40}$ ]]
|
|
[[ "$GITEA_RUN_ATTEMPT" =~ ^[0-9]+$ ]]
|
|
select_digest='import json,sys; d=json.load(sys.stdin); xs=d if isinstance(d,list) else [d]; xs=[x for x in xs if isinstance(x,dict) and isinstance(x.get("Descriptor",x),dict)]; x=next((x for x in xs if x.get("Descriptor",x).get("platform",{}).get("os")=="linux" and x.get("Descriptor",x).get("platform",{}).get("architecture")=="amd64"),None); print(x.get("Descriptor",x).get("digest","") if x else "")'
|
|
api_digest="$(docker manifest inspect "$IMAGE_REPOSITORY:api-$GITEA_SHA" --verbose | python3 -c "$select_digest")"
|
|
web_digest="$(docker manifest inspect "$IMAGE_REPOSITORY:web-$GITEA_SHA" --verbose | python3 -c "$select_digest")"
|
|
[[ "$api_digest" =~ ^sha256:[0-9a-f]{64}$ ]]
|
|
[[ "$web_digest" =~ ^sha256:[0-9a-f]{64}$ ]]
|
|
install -d -m 700 artifacts/staging-images
|
|
umask 077
|
|
git archive --format=tar --output artifacts/staging-images/controller.tar \
|
|
"$GITEA_SHA" deploy frontend/scripts/staging-image-manifest.mjs
|
|
controller_sha256="$(sha256sum artifacts/staging-images/controller.tar | awk '{print $1}')"
|
|
[[ "$controller_sha256" =~ ^[0-9a-f]{64}$ ]]
|
|
printf 'git_sha=%s\napi_digest=%s\nweb_digest=%s\ncontroller_sha256=%s\n' \
|
|
"$GITEA_SHA" "$api_digest" "$web_digest" "$controller_sha256" \
|
|
> artifacts/staging-images/manifest.env
|
|
node frontend/scripts/staging-image-manifest.mjs \
|
|
artifacts/staging-images/manifest.env "$GITEA_SHA" "$IMAGE_REPOSITORY" >/dev/null
|
|
|
|
- name: Upload immutable staging image manifest
|
|
run: |
|
|
set -euo pipefail
|
|
test -n "${ACTIONS_RUNTIME_TOKEN:-}"
|
|
test -n "${ACTIONS_RESULTS_URL:-}"
|
|
test -n "${GITHUB_RUN_ID:-}"
|
|
test -n "${GITHUB_REPOSITORY:-}"
|
|
workdir="$(pwd -P)"
|
|
docker run --rm \
|
|
--user "$(id -u):$(id -g)" \
|
|
--volume "$workdir:$workdir" \
|
|
--workdir "$workdir" \
|
|
--env HOME=/tmp \
|
|
--env "INPUT_NAME=staging-image-manifest-$GITEA_SHA-$GITEA_RUN_ATTEMPT" \
|
|
--env INPUT_PATH=artifacts/staging-images/ \
|
|
--env INPUT_OVERWRITE=false \
|
|
--env ACTIONS_RUNTIME_TOKEN \
|
|
--env ACTIONS_RESULTS_URL \
|
|
--env GITHUB_RUN_ID \
|
|
--env GITHUB_REPOSITORY \
|
|
--env "GITHUB_SHA=$GITEA_SHA" \
|
|
--env "GITHUB_WORKSPACE=$workdir" \
|
|
node:22-bookworm-slim \
|
|
node -e 'process.env["INPUT_IF-NO-FILES-FOUND"]="error"; process.env["INPUT_RETENTION-DAYS"]="30"; process.env["INPUT_COMPRESSION-LEVEL"]="6"; require("./.gitea/actions/upload-artifact/dist/index.js")'
|
|
|
|
- name: Dispatch exact-SHA staging deployment
|
|
env:
|
|
DEPLOY_SHA: ${{ gitea.sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
[[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]]
|
|
gate_run_id="${GITHUB_RUN_ID:-}"
|
|
[[ "$gate_run_id" =~ ^[0-9]+$ ]]
|
|
current_staging_sha="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
|
|
--header "Authorization: token $GITEA_TOKEN" \
|
|
"$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/staging" |
|
|
jq -er 'select(type == "array" and length == 1) | .[0] |
|
|
select(.ref == "refs/heads/staging") | .object.sha |
|
|
select(test("^[0-9a-f]{40}$"))')"
|
|
[[ "$current_staging_sha" == "$DEPLOY_SHA" ]] || { echo "staging advanced before deployment dispatch; refusing stale release" >&2; exit 1; }
|
|
payload="$(jq -cn --arg ref "refs/heads/staging" --arg deploy_sha "$DEPLOY_SHA" --arg gate_run_id "$gate_run_id" \
|
|
'{ref:$ref,inputs:{deploy_sha:$deploy_sha,gate_run_id:$gate_run_id,allow_rollback:"false"}}')"
|
|
response_file="$(mktemp "${RUNNER_TEMP:-/tmp}/jyotisha-deploy-dispatch.XXXXXX")"
|
|
trap 'rm -f -- "$response_file"' EXIT
|
|
curl --fail --silent --show-error --request POST \
|
|
--header "Authorization: token $GITEA_TOKEN" \
|
|
--header "Content-Type: application/json" \
|
|
--data "$payload" \
|
|
"$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/workflows/deploy-staging.yml/dispatches?return_run_details=true" \
|
|
--output "$response_file"
|
|
deploy_run_id="$(jq -er '.workflow_run_id | select(type == "number" and . > 0)' "$response_file")"
|
|
echo "Dispatched Deploy staging run $deploy_run_id for $DEPLOY_SHA"
|
|
|
|
- name: Logout ACR registry
|
|
if: always()
|
|
run: docker logout "$REGISTRY_HOST" >/dev/null 2>&1 || true
|