From 5687182980795c212ba55e2f0ae4beec3ac09253 Mon Sep 17 00:00:00 2001 From: Jesse_Chen Date: Tue, 18 Aug 2026 10:12:55 +0800 Subject: [PATCH] ci(runner): move every Gitea job to xiaoxin and let the gate reclaim its own disk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 质量门跑在以 :host 模式注册的跳板机上，与无关生产服务共用文件系统，累积的按 SHA 打标镜像把根分区占满，PostgreSQL fixture 起不来，staging 连续 5 个提交发不 出去。把 12 个 job 统一迁到 xiaoxin，并在开跑前回收磁盘、余量不足时直接报磁盘 原因而不是让 23 个数据库测试代为失败。 当年据以搬迁的「xiaoxin 缺少 Docker Compose v2」是误诊：真实原因是用户级插件 目录里一个零字节 docker-compose 遮蔽了系统插件。 Co-authored-by: Cursor --- .gitea/workflows/backend-quality-gate.yml | 14 ++++- .../workflows/create-production-recovery.yml | 2 +- .gitea/workflows/deploy-production.yml | 2 +- .gitea/workflows/deploy-staging.yml | 2 +- .../workflows/migrate-production-database.yml | 2 +- .gitea/workflows/migrate-staging-database.yml | 2 +- .gitea/workflows/release-quality-gate.yml | 2 +- deploy/reclaim-runner-disk.sh | 56 +++++++++++++++++ docs/BUG_HISTORY.md | 16 +++++ docs/research/pre_work_error_ledger.md | 6 ++ .../tests/staging-backend-workflows.test.ts | 62 +++++++++++++++++-- 11 files changed, 152 insertions(+), 14 deletions(-) create mode 100755 deploy/reclaim-runner-disk.sh diff --git a/.gitea/workflows/backend-quality-gate.yml b/.gitea/workflows/backend-quality-gate.yml index 017e5e03..963d3409 100644 --- a/.gitea/workflows/backend-quality-gate.yml +++ b/.gitea/workflows/backend-quality-gate.yml @@ -30,7 +30,7 @@ permissions: jobs: validate: - runs-on: manman-linux + runs-on: xiaoxin timeout-minutes: 45 env: GITEA_SHA: ${{ gitea.sha }} @@ -100,6 +100,11 @@ jobs: docker compose version --short | grep -Eq '^v?2\.' docker compose --help | grep -q -- '--project-name' + - name: Reclaim runner disk + env: + KEEP_IMAGE_SHA: ${{ gitea.sha }} + run: bash deploy/reclaim-runner-disk.sh + - name: Prepare pinned Node tooling run: | set -euo pipefail @@ -229,7 +234,7 @@ jobs: publish: if: gitea.event_name == 'push' && gitea.ref == 'refs/heads/staging' needs: validate - runs-on: manman-linux + runs-on: xiaoxin timeout-minutes: 60 env: GITEA_SHA: ${{ gitea.sha }} @@ -266,6 +271,11 @@ jobs: test "$(git rev-parse HEAD)" = "$GITEA_SHA" test -z "$(git status --porcelain --untracked-files=all)" + - name: Reclaim runner disk + env: + KEEP_IMAGE_SHA: ${{ gitea.sha }} + run: bash deploy/reclaim-runner-disk.sh + - name: Prepare pinned Node tooling env: NODE_TOOL_SOURCE_IMAGE: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/library/node:22-bookworm-slim@sha256:ef343465b6a14bbdf2ab52f6e100ec0659a792464fcf72c462370d88b3df909c diff --git a/.gitea/workflows/create-production-recovery.yml b/.gitea/workflows/create-production-recovery.yml index 3169ac8c..c143a720 100644 --- a/.gitea/workflows/create-production-recovery.yml +++ b/.gitea/workflows/create-production-recovery.yml @@ -19,7 +19,7 @@ concurrency: jobs: recover: - runs-on: manman-linux + runs-on: xiaoxin timeout-minutes: 30 env: GITEA_SHA: ${{ gitea.sha }} diff --git a/.gitea/workflows/deploy-production.yml b/.gitea/workflows/deploy-production.yml index 1a97ed24..2558168d 100644 --- a/.gitea/workflows/deploy-production.yml +++ b/.gitea/workflows/deploy-production.yml @@ -32,7 +32,7 @@ concurrency: jobs: deploy: - runs-on: manman-linux + runs-on: xiaoxin timeout-minutes: 30 env: GITEA_SHA: ${{ gitea.sha }} diff --git a/.gitea/workflows/deploy-staging.yml b/.gitea/workflows/deploy-staging.yml index cd5db1da..75fb1fb1 100644 --- a/.gitea/workflows/deploy-staging.yml +++ b/.gitea/workflows/deploy-staging.yml @@ -28,7 +28,7 @@ concurrency: jobs: deploy: - runs-on: manman-linux + runs-on: xiaoxin timeout-minutes: 30 env: GITEA_SHA: ${{ gitea.sha }} diff --git a/.gitea/workflows/migrate-production-database.yml b/.gitea/workflows/migrate-production-database.yml index 38cef122..ea35ff81 100644 --- a/.gitea/workflows/migrate-production-database.yml +++ b/.gitea/workflows/migrate-production-database.yml @@ -32,7 +32,7 @@ concurrency: jobs: migrate: - runs-on: manman-linux + runs-on: xiaoxin timeout-minutes: 20 env: GITEA_SHA: ${{ gitea.sha }} diff --git a/.gitea/workflows/migrate-staging-database.yml b/.gitea/workflows/migrate-staging-database.yml index 799f11be..69d62c34 100644 --- a/.gitea/workflows/migrate-staging-database.yml +++ b/.gitea/workflows/migrate-staging-database.yml @@ -19,7 +19,7 @@ concurrency: jobs: migrate: - runs-on: manman-linux + runs-on: xiaoxin timeout-minutes: 20 env: GITEA_SHA: ${{ gitea.sha }} diff --git a/.gitea/workflows/release-quality-gate.yml b/.gitea/workflows/release-quality-gate.yml index 37506b1c..7b00cd31 100644 --- a/.gitea/workflows/release-quality-gate.yml +++ b/.gitea/workflows/release-quality-gate.yml @@ -5,7 +5,7 @@ on: jobs: release-quality-gate: - runs-on: manman-linux + runs-on: xiaoxin timeout-minutes: 45 env: GITEA_SHA: ${{ gitea.sha }} diff --git a/deploy/reclaim-runner-disk.sh b/deploy/reclaim-runner-disk.sh new file mode 100755 index 00000000..38c31802 --- /dev/null +++ b/deploy/reclaim-runner-disk.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +MINIMUM_FREE_GIB="${MINIMUM_FREE_GIB:-10}" +IMAGE_REPOSITORY="${IMAGE_REPOSITORY:-*/copse/jyotisha}" +KEEP_IMAGE_SHA="${KEEP_IMAGE_SHA:-}" + +if ! [[ "$MINIMUM_FREE_GIB" =~ ^[0-9]+$ ]]; then + echo "MINIMUM_FREE_GIB must be a whole number of GiB" >&2 + exit 1 +fi + +DOCKER_ROOT="$(docker info --format '{{.DockerRootDir}}')" +if [ -z "$DOCKER_ROOT" ] || [ ! -d "$DOCKER_ROOT" ]; then + echo "unable to resolve the Docker root directory to measure" >&2 + exit 1 +fi + +free_gib() { + df -Pk "$DOCKER_ROOT" | awk 'NR == 2 { printf "%d", $4 / 1048576 }' +} + +BEFORE_GIB="$(free_gib)" +echo "docker root $DOCKER_ROOT has ${BEFORE_GIB} GiB free before reclaim" + +# Every reclaim below is restricted to resources no container currently holds, +# so a quality-gate job running concurrently on this runner keeps its fixtures. +docker container prune --force --filter until=6h +docker volume ls --quiet --filter 'name=^jyotisha-postgres-' --filter dangling=true | + xargs -r docker volume rm +docker network prune --force --filter until=6h +docker image prune --force + +# Exact-SHA release images are immutable in the registry once published, so the +# runner's local copies are rebuildable cache rather than release state. +KEEP_TAG_SHA="${KEEP_IMAGE_SHA:-no-image-to-keep}" +STALE_IMAGES="$(docker image ls --format '{{.Repository}}:{{.Tag}}' \ + --filter "reference=$IMAGE_REPOSITORY:api-*" \ + --filter "reference=$IMAGE_REPOSITORY:web-*" | + grep -v -F -e "api-$KEEP_TAG_SHA" -e "web-$KEEP_TAG_SHA" || true)" +if [ -n "$STALE_IMAGES" ]; then + printf '%s\n' "$STALE_IMAGES" | xargs -r docker image rm || true +fi + +docker builder prune --force --all + +AFTER_GIB="$(free_gib)" +echo "docker root $DOCKER_ROOT has ${AFTER_GIB} GiB free after reclaim" +docker system df + +if [ "$AFTER_GIB" -lt "$MINIMUM_FREE_GIB" ]; then + echo "runner kept only ${AFTER_GIB} GiB free; PostgreSQL fixtures and image builds need at least ${MINIMUM_FREE_GIB} GiB" >&2 + exit 1 +fi + +echo "runner disk reclaimed" diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index 92578a39..f5cce3af 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -3925,3 +3925,19 @@ - 防复发：任何号称个人化的界面文案，其数据来源必须能追到当次计算或当次生成；用哈希在固定文案池里取模不是个人化，只是伪装成个人化。降级路径不得复制一份“看起来正常”的内容——生成失败时必须让用户看出没有生成成功，本次改成明确的一句话降级文案而不是通用建议。 - 相关记录：BUG-265 无前序同类记录。与 BUG-263 同批，均为首页可见问题。 - 修复版本：本地未提交候选 + +## BUG-266 | staging 连续 5 个提交发不出去：质量门跑在跳板机上，把那台机器的磁盘占满，PostgreSQL fixture 全线起不来 + +- 状态：resolved（本地修复，待提交与发布） +- 首次发现：2026-08-18 +- 最近更新：2026-08-18 +- 影响面：`.gitea/workflows/` 全部 12 个 job 的 runner 归属、staging 质量门的数据库集成测试、`publish` 与 `Deploy staging` 的放行，以及跳板机上与本项目无关的 Nacos/MySQL/Redis/xxl-job 服务。 +- 用户现象：推到 `staging` 的提交一个都没上线。`https://staging.jyotisha.chat/api/health` 长时间停在 `561010f2`，而 `origin/staging` 已经前进了 4 个提交；Gitea 上 `validate` 连续 5 次 failure、`publish` 连续 5 次 skipped。 +- 触发条件：向 `staging` 推任何提交。与提交内容无关。 +- 根因：两层。其一，`manman-linux` runner 以 `manman-linux:host` 注册在跳板机上，job 直接跑在该机的宿主文件系统里，而质量门每次 push 都在本地 build 两个按 SHA 打标的镜像（`api-`、`web-`）且全流程没有任何镜像与构建缓存回收，累积到 1453 个镜像、根分区 99G 用满 94G、Avail 归零，于是 `docker compose up -d --wait postgres` 阶段 `initdb` 直接 `No space left on device`，23 个数据库测试级联失败。这台机器同时还跑着与本项目无关的生产服务，等于用别人的磁盘做构建。其二，当初把门禁从 `xiaoxin` 搬到跳板机的理由（ERR-103「xiaoxin 缺少 Docker Compose v2」）是误诊：`xiaoxin` 的 Docker Engine 一直正常，真正的原因是 `/root/.docker/cli-plugins/docker-compose` 有一个 2026-08-05 建立的**零字节文件**，用户级插件目录优先级高于系统目录，把系统里正常的 compose 插件遮蔽成 `exec format error`。误诊导致门禁被搬到一台不该承担构建的机器上，磁盘耗尽只是时间问题。 +- 修复：删掉 `xiaoxin` 上那个零字节插件占位文件并补装 `docker-compose-v2`（2.40.3），确认 `docker compose version --short` 与 `--project-name` 两项门禁能力检查通过；把 `.gitea/workflows/` 里 8 个仍指向 `manman-linux` 的 job 全部改为 `xiaoxin`（该机 20 核、61G 内存、877G 空闲）；新增 `deploy/reclaim-runner-disk.sh`，在 `validate` 与 `publish` 的 checkout 之后、拉取 Node 工具之前回收磁盘，并在空间仍不足 10 GiB 时带着原因 fail-closed，而不是让 23 个数据库测试去暴露磁盘问题。回收只针对没有任何容器持有的资源（`container prune --filter until=6h`、`name=^jyotisha-postgres-` 且 `dangling=true` 的卷、`network prune --filter until=6h`、悬空镜像、buildx 缓存，以及除当次 SHA 外的历史 `api-`/`web-` 标签），因此不会掀掉同一台 runner 上并发 job 的 fixture。 +- 验证：`staging-backend-workflows.test.ts` 38 项通过，其中新增两项——遍历 `.gitea/workflows/*.yml` 断言每个 `runs-on` 都是 `xiaoxin`（并禁止 `manman-linux` 重新出现），以及断言两个 job 都在 Node 工具之前调用回收脚本、脚本只回收未被持有的资源且低于阈值时 fail-closed；回收脚本纳入既有 shell 语法校验。`tsc --noEmit` 与该测试文件 ESLint 清洁。在 `xiaoxin` 上实测：compose 2.40.3 通过门禁的两项能力检查，且到 Gitea、ACR 镜像仓库、staging 主机 22 端口、npm/pypi/ECR/SWR 镜像源的出网全部可达。未做的验证：**回收脚本没有在真实 runner 上跑过一次**（当前 `xiaoxin` 有 877G 空闲，回收逻辑与阈值分支要等首次门禁运行才被真正执行）；跳板机上那 94G 与仍在运行的 `act_runner` 尚未清理下线。 +- 防复发：构建与测试不得跑在承载其他服务的机器上，也不得以 `:host` 模式共用其文件系统。质量门在开跑前自己保证磁盘余量，并把「磁盘不够」报成一句明确的失败，而不是让下游 fixture 去替它失败。runner 能力缺失必须定位到具体原因再决定搬迁：`docker compose` 不可用时要先查 `docker info` 的 client plugins 与用户级 `~/.docker/cli-plugins` 遮蔽，不能直接判定整台机器不支持 Compose——ERR-103 正是漏了这一步，代价是把门禁搬到错误的机器上并最终堵死发布。 +- 相关记录：ERR-103（`docs/research/pre_work_error_ledger.md`，同一 Compose 现象的误诊，本次给出真实根因）、ERR-105（同一台跳板机磁盘耗尽的基础设施记录）、BUG-264（本次被卡住无法发布的修复） +- 复发自：无 +- 修复版本：本地未提交候选 diff --git a/docs/research/pre_work_error_ledger.md b/docs/research/pre_work_error_ledger.md index 96aab958..0e8673cd 100644 --- a/docs/research/pre_work_error_ledger.md +++ b/docs/research/pre_work_error_ledger.md @@ -273,3 +273,9 @@ Prevention: 将该门禁运行在已验证 Docker Compose v2 的 `manman-linux` The gateway did not request an official full snapshot by default, and the consultation workflow built its machine evidence packet before invoking that gateway, so configured credentials and readiness still produced no current raw response for the report. The same workflow also failed to attach the repository's existing Ashtakavarga and KP calculators to `chart.modules`, leaving both Technique Audit rows blocked despite valid local chart inputs. Prevention: make the gateway request official raw evidence without relying on an optional environment switch; invoke it before freezing the evidence packet; require `official_verified` plus a non-empty raw response before promoting official evidence; attach existing local calculator outputs through the shared consultation layer and keep KP values `observation_only` until an external worked-example parity gate closes. Regression tests must lock both raw-response propagation and module presence. + +## ERR-105 | `manman-linux:host` runner 把跳板机磁盘占满，staging 质量门连续 5 次失败且发布中断 | resolved 2026-08-18 + +`manman-linux` runner 以 `:host` 模式注册在一台同时运行 Nacos/MySQL/Redis/xxl-job 的跳板机上，质量门每次 push 都在该宿主机本地 build 两个按 SHA 打标的镜像且从不回收，累积 1453 个镜像后根分区 99G 用满 94G、Avail 归零。`docker compose up -d --wait postgres` 阶段 `initdb` 报 `No space left on device`，23 个数据库集成测试级联失败，`publish` 连续 5 次 skipped，`staging` 上 4 个提交无法发布。ERR-103 当初据以搬迁的「`xiaoxin` 缺少 Docker Compose v2」是误诊：真实原因是 `xiaoxin` 的 `/root/.docker/cli-plugins/docker-compose` 有一个零字节文件遮蔽了系统插件（用户级插件目录优先），Docker Engine 本身一直正常。 + +Prevention: 构建与测试 runner 不得与其他服务共用宿主文件系统；门禁必须在开跑前自行回收磁盘并在余量不足时 fail-closed 报出磁盘原因，而不是让下游 fixture 代为暴露。判定某台 runner「不支持 Docker Compose」之前，必须先看 `docker info` 的 client plugins 报错与用户级 `~/.docker/cli-plugins` 是否存在遮蔽文件；把能力缺失归因到整台机器会导致错误的迁移决策，本次代价是发布中断。 diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index 1c5b2679..acd73fe3 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { spawnSync } from "node:child_process"; @@ -70,6 +70,10 @@ const syncScript = new URL( "../../deploy/sync-staging-tree.sh", import.meta.url, ); +const reclaimRunnerDiskScript = new URL( + "../../deploy/reclaim-runner-disk.sh", + import.meta.url, +); const productionDeployScript = new URL( "../../deploy/run-production-deploy.sh", import.meta.url, @@ -225,7 +229,7 @@ test("Gitea quality gate validates before publishing an immutable ACR manifest", assert.match(workflow, /pull_request:\n\s+paths:/); assert.match(workflow, /push:\n\s+branches: \[staging\]/); assert.match(workflow, /workflow_dispatch:/); - assert.equal((workflow.match(/runs-on: manman-linux/g) ?? []).length, 2); + assert.equal((workflow.match(/runs-on: xiaoxin/g) ?? []).length, 2); assert.match(workflow, /validate:[\s\S]*?timeout-minutes: 45[\s\S]*?publish:[\s\S]*?timeout-minutes: 60/); assert.match(workflow, /GITEA_SHA: \$\{\{ gitea\.sha \}\}/); assert.match( @@ -369,10 +373,55 @@ test("Gitea quality gate validates before publishing an immutable ACR manifest", assert.doesNotMatch(workflow, /(?:^|:)latest$/m); }); +test("every Gitea job runs on xiaoxin, never on the retired jump-host runner", () => { + const directory = fileURLToPath(new URL("../../.gitea/workflows/", import.meta.url)); + const declarations = readdirSync(directory) + .filter((entry) => entry.endsWith(".yml")) + .flatMap((entry) => { + const contents = readFileSync(join(directory, entry), "utf8"); + return [...contents.matchAll(/^\s*runs-on: (\S+)$/gm)].map((match) => `${entry}:${match[1]}`); + }); + + // The jump host registered itself as manman-linux:host, so its jobs shared a + // filesystem with unrelated production services and filled it. + assert.ok(declarations.length >= 12, `expected every Gitea job to declare a runner, saw ${declarations.length}`); + assert.deepEqual(declarations.filter((declaration) => !declaration.endsWith(":xiaoxin")), []); +}); + +test("both gate jobs reclaim runner disk before they need it, and only unheld resources", () => { + const workflow = read(giteaQualityWorkflow); + const script = read(reclaimRunnerDiskScript); + + assert.equal((workflow.match(/bash deploy\/reclaim-runner-disk\.sh/g) ?? []).length, 2); + assert.equal((workflow.match(/KEEP_IMAGE_SHA: \$\{\{ gitea\.sha \}\}/g) ?? []).length, 2); + for (const job of workflow.split(/^ (?=\w+:\n)/m).slice(1)) { + if (!job.includes("bash deploy/reclaim-runner-disk.sh")) continue; + assertOrder(job, [ + "Checkout exact Gitea revision", + "bash deploy/reclaim-runner-disk.sh", + "Prepare pinned Node tooling", + ]); + } + + assert.match(script, /df -Pk "\$DOCKER_ROOT"/); + assert.match(script, /docker container prune --force --filter until=6h/); + assert.match(script, /--filter 'name=\^jyotisha-postgres-' --filter dangling=true/); + assert.match(script, /docker network prune --force --filter until=6h/); + assert.match(script, /docker image prune --force\n/); + assert.match(script, /docker builder prune --force --all/); + assert.match(script, /grep -v -F -e "api-\$KEEP_TAG_SHA" -e "web-\$KEEP_TAG_SHA"/); + assert.match(script, /PostgreSQL fixtures and image builds need at least \$\{MINIMUM_FREE_GIB\} GiB/); + // A gate that silently proceeds on a full disk fails 23 database tests instead + // of naming the disk, and pruning what a container still holds would delete a + // concurrent job's fixture out from under it. + assert.match(script, /if \[ "\$AFTER_GIB" -lt "\$MINIMUM_FREE_GIB" \]; then\n\s+echo[^\n]+\n\s+exit 1/); + assert.doesNotMatch(script, /docker system prune|--volumes|prune[^\n]*--all --force|docker (?:rm|volume rm|kill)[^\n]*--force/); +}); + test("Gitea staging mutation workflows use the available runner and pinned Node tooling", () => { for (const workflow of [read(giteaDeployWorkflow), read(giteaMigrationWorkflow)]) { - assert.equal((workflow.match(/runs-on: manman-linux/g) ?? []).length, 1); - assert.doesNotMatch(workflow, /runs-on: xiaoxin/); + assert.equal((workflow.match(/runs-on: xiaoxin/g) ?? []).length, 1); + assert.doesNotMatch(workflow, /runs-on: manman-linux/); assert.equal((workflow.match(/NODE_TOOL_SOURCE_IMAGE: swr\.cn-north-4\.myhuaweicloud\.com\/ddn-k8s\/docker\.io\/library\/node:22-bookworm-slim@sha256:ef343465b6a14bbdf2ab52f6e100ec0659a792464fcf72c462370d88b3df909c/g) ?? []).length, 1); assert.equal((workflow.match(/NODE_TOOL_IMAGE: node:22-bookworm-slim/g) ?? []).length, 1); assert.match(workflow, /timeout 180 docker pull "\$NODE_TOOL_SOURCE_IMAGE"/); @@ -947,7 +996,7 @@ test("manual release gate runs where Docker Compose v2 is available", () => { const workflow = read(giteaReleaseQualityWorkflow); assert.match(workflow, /^on:\n\s+workflow_dispatch:/m); - assert.match(workflow, /runs-on: manman-linux/); + assert.match(workflow, /runs-on: xiaoxin/); assert.match(workflow, /NODE_TOOL_SOURCE_IMAGE: .*node:22-bookworm-slim@sha256:[0-9a-f]{64}/); assert.match(workflow, /NODE_TOOL_IMAGE: node:22-bookworm-slim/); assert.match(workflow, /node --version \| grep -Eq '\^v22\\\.'/); @@ -994,6 +1043,7 @@ test("deployment scripts pass shell syntax validation", () => { deployScript, migrationScript, syncScript, + reclaimRunnerDiskScript, productionDeployScript, productionMigrationScript, productionSyncScript, @@ -1013,7 +1063,7 @@ test("Gitea production deploy consumes only gate-attested digests under manual c assert.match(workflow, /^on:\n\s+workflow_dispatch:/m); assert.doesNotMatch(workflow, /workflow_run:|\n\s+push:/); assert.match(workflow, /deploy_sha:[\s\S]*allow_rollback:[\s\S]*verification_mode:/); - assert.match(workflow, /runs-on: manman-linux/); + assert.match(workflow, /runs-on: xiaoxin/); assert.match(workflow, /group: production-mutation/); assert.match(workflow, /branch=staging&event=push&status=success/); assert.match(workflow, /endswith\("backend-quality-gate\.yml"\)/);