Jesse_Chen
444 lines
16 KiB
TypeScript
444 lines
16 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import {
|
|
chmodSync,
|
|
mkdtempSync,
|
|
readFileSync,
|
|
rmSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { spawnSync } from "node:child_process";
|
|
import test from "node:test";
|
|
import { fileURLToPath } from "node:url";
|
|
import { GET as healthGet } from "../src/app/api/health/route.ts";
|
|
|
|
function serviceBlock(compose: string, service: string) {
|
|
const match = compose.match(new RegExp(`^ ${service}:\\n([\\s\\S]*?)(?=^ [a-z][a-z0-9_-]*:|^volumes:)`, "m"));
|
|
assert.ok(match, `expected ${service} service in compose file`);
|
|
return match[1];
|
|
}
|
|
|
|
function webHealthcheckBlock(web: string) {
|
|
const match = web.match(/^ healthcheck:\n((?: .*\n?)*)/m);
|
|
assert.ok(match, "expected a web healthcheck in compose file");
|
|
return match[1];
|
|
}
|
|
|
|
function workflowStepBlock(workflow: string, stepName: string) {
|
|
const match = workflow.match(new RegExp(`^ - name: ${stepName}\\n([\\s\\S]*?)(?=^ - name:|(?![\\s\\S]))`, "m"));
|
|
assert.ok(match, `expected ${stepName} workflow step`);
|
|
return match[1];
|
|
}
|
|
|
|
function escapeRegExp(value: string) {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
}
|
|
|
|
const testedShaExpression = "${{ github.sha }}";
|
|
|
|
function assertWorkflowUsesTestedSha(workflow: string) {
|
|
const checkout = workflowStepBlock(workflow, "Checkout tested revision");
|
|
const sync = workflowStepBlock(workflow, "Sync and rebuild");
|
|
const verification = workflowStepBlock(workflow, "Verify production");
|
|
const shaExpression = escapeRegExp(testedShaExpression);
|
|
|
|
assert.match(checkout, new RegExp(`ref: ${shaExpression}`), "Checkout tested revision must check out the tested SHA");
|
|
assert.match(sync, new RegExp(`DEPLOY_GIT_SHA: ${shaExpression}`), "Sync and rebuild must use the tested SHA");
|
|
assert.match(sync, /GITHUB_SHA='\$DEPLOY_GIT_SHA'/, "Sync and rebuild must inject its SHA into the web runtime");
|
|
assert.match(verification, new RegExp(`DEPLOY_GIT_SHA: ${shaExpression}`), "Verify production must use the tested SHA");
|
|
assert.match(verification, /curl --fail --silent --show-error https:\/\/jyotisha\.chat\/api\/health/);
|
|
assert.match(verification, /get\("deployment", \{\}\)\.get\("gitCommit"/);
|
|
assert.match(verification, /Production revision did not converge/);
|
|
}
|
|
|
|
test("health endpoint exposes deployment identity for production verification", () => {
|
|
const source = [
|
|
readFileSync(new URL("../src/app/api/health/route.ts", import.meta.url), "utf8"),
|
|
readFileSync(new URL("../src/lib/conversational-rectification/creation-policy.ts", import.meta.url), "utf8"),
|
|
].join("\n");
|
|
|
|
assert.match(source, /deployment:/);
|
|
assert.match(source, /GITHUB_SHA/);
|
|
assert.match(source, /VERCEL_GIT_COMMIT_SHA/);
|
|
assert.match(source, /gitCommit/);
|
|
});
|
|
|
|
test("manual production deployment passes the selected revision into the web runtime", () => {
|
|
const compose = readFileSync(
|
|
new URL("../../deploy/docker-compose.server.yml", import.meta.url),
|
|
"utf8",
|
|
);
|
|
const workflow = readFileSync(
|
|
new URL("../../.github/workflows/deploy-production.yml", import.meta.url),
|
|
"utf8",
|
|
);
|
|
|
|
assert.match(compose, /GITHUB_SHA: \$\{GITHUB_SHA\}/);
|
|
assert.match(workflow, /workflow_dispatch:/);
|
|
assert.doesNotMatch(workflow, /workflow_run:/);
|
|
assert.match(workflow, /DEPLOY_GIT_SHA: \$\{\{ github\.sha \}\}/);
|
|
assert.match(workflow, /GITHUB_SHA='\$DEPLOY_GIT_SHA'/);
|
|
assert.match(workflow, /get\("deployment", \{\}\)\.get\("gitCommit"/);
|
|
assert.match(workflow, /DEPLOY_GIT_SHA/);
|
|
assert.match(workflow, /Production revision did not converge/);
|
|
assert.match(workflow, /git ls-remote origin refs\/heads\/main/);
|
|
assert.match(workflow, /steps\.revision\.outputs\.deploy == 'true'/);
|
|
});
|
|
|
|
test("server compose accepts staging paths while preserving production defaults", () => {
|
|
const compose = readFileSync(
|
|
new URL("../../deploy/docker-compose.server.yml", import.meta.url),
|
|
"utf8",
|
|
);
|
|
|
|
assert.match(
|
|
compose,
|
|
/env_file:\s*\n\s*- \$\{APP_ENV_FILE:-\.\.\/\.env\.production\}/,
|
|
);
|
|
assert.match(
|
|
compose,
|
|
/\$\{CADDYFILE_PATH:-\.\/Caddyfile\}:\/etc\/caddy\/Caddyfile:ro/,
|
|
);
|
|
assert.match(
|
|
compose,
|
|
/SITE_ADDRESS: \$\{SITE_ADDRESS:-https:\/\/jyotisha\.chat\}/,
|
|
);
|
|
});
|
|
|
|
test("server compose defaults to local images without removing either build", () => {
|
|
const composeFile = fileURLToPath(
|
|
new URL("../../deploy/docker-compose.server.yml", import.meta.url),
|
|
);
|
|
const compose = readFileSync(composeFile, "utf8");
|
|
|
|
assert.match(compose, /^\s+image: \$\{API_IMAGE:-jyotisha-api:local\}$/m);
|
|
assert.match(compose, /^\s+image: \$\{WEB_IMAGE:-jyotisha-web:local\}$/m);
|
|
|
|
const root = mkdtempSync(join(tmpdir(), "jyotisha-server-compose-"));
|
|
const appEnvFile = join(root, ".env.production");
|
|
writeFileSync(appEnvFile, "RUNTIME_FIXTURE=1\n");
|
|
chmodSync(appEnvFile, 0o600);
|
|
|
|
const env: NodeJS.ProcessEnv = {
|
|
...process.env,
|
|
APP_ENV_FILE: appEnvFile,
|
|
CADDYFILE_PATH: fileURLToPath(
|
|
new URL("../../deploy/Caddyfile", import.meta.url),
|
|
),
|
|
GITHUB_SHA: "0000000000000000000000000000000000000000",
|
|
NEXT_PUBLIC_SUPABASE_URL: "https://placeholder.supabase.co",
|
|
NEXT_PUBLIC_SUPABASE_ANON_KEY: "placeholder",
|
|
};
|
|
delete env.API_IMAGE;
|
|
delete env.WEB_IMAGE;
|
|
|
|
try {
|
|
const result = spawnSync(
|
|
"docker",
|
|
["compose", "-f", composeFile, "config", "--format", "json"],
|
|
{ encoding: "utf8", env },
|
|
);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
const rendered = JSON.parse(result.stdout) as {
|
|
services: Record<string, { build?: unknown; image?: string }>;
|
|
};
|
|
assert.equal(rendered.services.api.image, "jyotisha-api:local");
|
|
assert.equal(rendered.services.web.image, "jyotisha-web:local");
|
|
assert.ok(rendered.services.api.build, "api build definition was removed");
|
|
assert.ok(rendered.services.web.build, "web build definition was removed");
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("staging Caddy configuration serves only the configured staging address", () => {
|
|
const caddy = readFileSync(
|
|
new URL("../../deploy/Caddyfile.staging", import.meta.url),
|
|
"utf8",
|
|
);
|
|
|
|
assert.match(caddy, /\{\$SITE_ADDRESS:https:\/\/staging\.jyotisha\.chat\}/);
|
|
assert.match(caddy, /reverse_proxy web:3000/);
|
|
assert.doesNotMatch(caddy, /www\.jyotisha\.chat/);
|
|
});
|
|
|
|
test("staging deploy consumes only the isolated staging environment and tested revision", () => {
|
|
const qualityGate = readFileSync(
|
|
new URL("../../.github/workflows/backend-quality-gate.yml", import.meta.url),
|
|
"utf8",
|
|
);
|
|
const workflow = readFileSync(
|
|
new URL("../../.github/workflows/deploy-staging.yml", import.meta.url),
|
|
"utf8",
|
|
);
|
|
const syncController = readFileSync(
|
|
new URL("../../deploy/sync-staging-tree.sh", import.meta.url),
|
|
"utf8",
|
|
);
|
|
|
|
assert.match(qualityGate, /push:\s*\n\s*branches: \[staging\]/);
|
|
assert.match(workflow, /workflows: \["Staging Backend Quality Gate"\]/);
|
|
assert.match(
|
|
workflow,
|
|
/github\.event\.workflow_run\.head_branch == 'staging'/,
|
|
);
|
|
assert.match(workflow, /actions: read/);
|
|
assert.match(workflow, /packages: read/);
|
|
assert.match(workflow, /environment:\s*\n\s*name: staging/);
|
|
assert.match(workflow, /deploy_sha:/);
|
|
assert.match(workflow, /\^\[0-9a-f\]\{40\}\$/);
|
|
assert.match(
|
|
workflow,
|
|
/actions\/workflows\/backend-quality-gate\.yml\/runs\?head_sha=/,
|
|
);
|
|
assert.match(workflow, /STAGING_SSH_PRIVATE_KEY/);
|
|
assert.match(workflow, /vars\.STAGING_HOST/);
|
|
assert.match(workflow, /vars\.STAGING_KNOWN_HOSTS/);
|
|
assert.match(workflow, /test "\$DEPLOY_HOST" = "118\.26\.111\.127"/);
|
|
assert.match(workflow, /test "\$DEPLOY_USER" = "deploy"/);
|
|
assert.match(workflow, /test "\$DEPLOY_PATH" = "\/opt\/jyotisha-staging"/);
|
|
assert.match(
|
|
workflow,
|
|
/--include='\/deploy\/' --include='\/deploy\/\*\*\*' --exclude='\*'/,
|
|
);
|
|
assert.match(workflow, /run-staging-deploy\.sh/);
|
|
assert.match(workflow, /steps\.images\.outputs\.api_image/);
|
|
assert.match(workflow, /steps\.images\.outputs\.web_image/);
|
|
assert.doesNotMatch(workflow, /PRODUCTION_SSH_PRIVATE_KEY/);
|
|
assert.doesNotMatch(workflow, /103\.117\.123\.53/);
|
|
assert.match(syncController, /--exclude='\/\.env\*'/);
|
|
assert.match(syncController, /--exclude='\/\.docker\/'/);
|
|
assert.match(syncController, /--exclude='\/backups\/'/);
|
|
});
|
|
|
|
test("staging env validator rejects selector drift, duplicates, and unsafe permissions", () => {
|
|
const validator = fileURLToPath(
|
|
new URL("../../deploy/validate-staging-env.sh", import.meta.url),
|
|
);
|
|
const root = mkdtempSync(join(tmpdir(), "jyotisha-staging-env-"));
|
|
const envFile = join(root, ".env.staging");
|
|
const composeFile = join(root, "compose.yml");
|
|
const validSelectors = [
|
|
"APP_ENV_FILE=../.env.staging",
|
|
"CADDYFILE_PATH=./Caddyfile.staging",
|
|
"SITE_ADDRESS=https://staging.jyotisha.chat",
|
|
];
|
|
const run = () =>
|
|
spawnSync("bash", [validator, envFile], { encoding: "utf8" });
|
|
const writeEnv = (lines: string[], mode = 0o600) => {
|
|
writeFileSync(envFile, `${lines.join("\n")}\n`);
|
|
chmodSync(envFile, mode);
|
|
};
|
|
|
|
try {
|
|
writeFileSync(
|
|
composeFile,
|
|
[
|
|
"services:",
|
|
" probe:",
|
|
" image: alpine",
|
|
" environment:",
|
|
" SELECTED: ${APP_ENV_FILE}",
|
|
"",
|
|
].join("\n"),
|
|
);
|
|
writeEnv(validSelectors);
|
|
assert.equal(run().status, 0);
|
|
const shellOverride = spawnSync(
|
|
"docker",
|
|
[
|
|
"compose",
|
|
"--env-file",
|
|
envFile,
|
|
"-f",
|
|
composeFile,
|
|
"config",
|
|
"--format",
|
|
"json",
|
|
],
|
|
{
|
|
encoding: "utf8",
|
|
env: { ...process.env, APP_ENV_FILE: "../.env.production" },
|
|
},
|
|
);
|
|
assert.equal(shellOverride.status, 0, shellOverride.stderr);
|
|
assert.equal(
|
|
JSON.parse(shellOverride.stdout).services.probe.environment.SELECTED,
|
|
"../.env.production",
|
|
);
|
|
|
|
writeEnv(["APP_ENV_FILE=../.env.production", ...validSelectors.slice(1)]);
|
|
assert.notEqual(run().status, 0);
|
|
|
|
writeEnv([...validSelectors, "SITE_ADDRESS=https://example.invalid"]);
|
|
assert.notEqual(run().status, 0);
|
|
|
|
writeEnv([...validSelectors, "APP_ENV_FILE = ../.env.production"]);
|
|
assert.notEqual(run().status, 0);
|
|
const rendered = spawnSync(
|
|
"docker",
|
|
[
|
|
"compose",
|
|
"--env-file",
|
|
envFile,
|
|
"-f",
|
|
composeFile,
|
|
"config",
|
|
"--format",
|
|
"json",
|
|
],
|
|
{ encoding: "utf8" },
|
|
);
|
|
assert.equal(rendered.status, 0, rendered.stderr);
|
|
assert.equal(
|
|
JSON.parse(rendered.stdout).services.probe.environment.SELECTED,
|
|
"../.env.production",
|
|
);
|
|
|
|
writeEnv([...validSelectors, "export CADDYFILE_PATH=./Caddyfile"]);
|
|
assert.notEqual(run().status, 0);
|
|
|
|
writeEnv([...validSelectors, "SITE_ADDRESS"]);
|
|
assert.notEqual(run().status, 0);
|
|
|
|
writeEnv(validSelectors, 0o644);
|
|
assert.notEqual(run().status, 0);
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("production traffic waits for a healthy web container and retries short replacement gaps", () => {
|
|
const compose = readFileSync(new URL("../../deploy/docker-compose.server.yml", import.meta.url), "utf8");
|
|
const caddyfile = readFileSync(new URL("../../deploy/Caddyfile", import.meta.url), "utf8");
|
|
const web = serviceBlock(compose, "web");
|
|
const caddy = serviceBlock(compose, "caddy");
|
|
const healthcheck = webHealthcheckBlock(web);
|
|
|
|
assert.match(web, /GITHUB_SHA: \$\{GITHUB_SHA\}/);
|
|
assert.match(web, /healthcheck:\n\s+test: \["CMD", "node", "-e", "fetch\('http:\/\/127\.0\.0\.1:3000\/api\/health'\)\.then\(r=>\{if\(!r\.ok\)process\.exit\(1\)\}\)"\]/);
|
|
assert.match(healthcheck, /^ interval: 30s$/m);
|
|
assert.match(healthcheck, /^ timeout: 5s$/m);
|
|
assert.match(healthcheck, /^ retries: 5$/m);
|
|
assert.match(healthcheck, /^ start_period: 30s$/m);
|
|
assert.match(healthcheck, /^ start_interval: 1s$/m);
|
|
assert.match(caddy, /web:\n\s+condition: service_healthy/);
|
|
assert.match(caddyfile, /reverse_proxy web:3000 \{\n\s+lb_try_duration 10s\n\s+lb_try_interval 250ms\n\s+\}/);
|
|
});
|
|
|
|
test("production workflow consistently uses its tested revision from checkout through verification", () => {
|
|
const workflow = readFileSync(new URL("../../.github/workflows/deploy-production.yml", import.meta.url), "utf8");
|
|
|
|
assertWorkflowUsesTestedSha(workflow);
|
|
});
|
|
|
|
test("production workflow rejects a SHA mismatch in verification", () => {
|
|
const workflow = readFileSync(new URL("../../.github/workflows/deploy-production.yml", import.meta.url), "utf8");
|
|
const verification = workflowStepBlock(workflow, "Verify production");
|
|
const mismatchedVerification = verification.replace(
|
|
testedShaExpression,
|
|
"${{ github.event.workflow_run.head_sha || github.sha }}",
|
|
);
|
|
|
|
assert.throws(
|
|
() => assertWorkflowUsesTestedSha(workflow.replace(verification, mismatchedVerification)),
|
|
/Verify production must use the tested SHA/,
|
|
);
|
|
});
|
|
|
|
test("v3 readiness requires healthy dependencies and smoke proof for the exact full deployment SHA", async () => {
|
|
const keys = [
|
|
"GITHUB_SHA", "NEXT_PUBLIC_SUPABASE_URL", "NEXT_PUBLIC_SUPABASE_ANON_KEY",
|
|
"SUPABASE_SERVICE_ROLE_KEY", "OPENAI_API_KEY",
|
|
"RECTIFICATION_V3_CREATE_ENABLED", "RECTIFICATION_V3_MIGRATIONS_READY",
|
|
"RECTIFICATION_V3_SYNTHETIC_SMOKE_SHA", "RECTIFICATION_V3_SYNTHETIC_SMOKE_USER_IDS",
|
|
] as const;
|
|
const prior = Object.fromEntries(keys.map((key) => [key, process.env[key]]));
|
|
const originalFetch = globalThis.fetch;
|
|
const currentSha = "0123456789abcdef0123456789abcdef01234567";
|
|
const oldSha = "89abcdef0123456789abcdef0123456789abcdef";
|
|
Object.assign(process.env, {
|
|
GITHUB_SHA: currentSha,
|
|
NEXT_PUBLIC_SUPABASE_URL: "https://example.invalid",
|
|
NEXT_PUBLIC_SUPABASE_ANON_KEY: "synthetic-public-key",
|
|
SUPABASE_SERVICE_ROLE_KEY: "synthetic-service-key",
|
|
OPENAI_API_KEY: "synthetic-model-key",
|
|
RECTIFICATION_V3_CREATE_ENABLED: "true",
|
|
RECTIFICATION_V3_MIGRATIONS_READY: "true",
|
|
RECTIFICATION_V3_SYNTHETIC_SMOKE_USER_IDS: "00000000-0000-4000-8000-000000009001",
|
|
});
|
|
globalThis.fetch = async () => Response.json({ status: "ok" });
|
|
|
|
async function readiness() {
|
|
const response = await healthGet();
|
|
const body = await response.json() as {
|
|
status: string;
|
|
rollout: { conversationalRectificationV3: {
|
|
syntheticSmoke: string;
|
|
creationAudience: string;
|
|
readyForNewCases: boolean;
|
|
} };
|
|
};
|
|
return body;
|
|
}
|
|
|
|
try {
|
|
delete process.env.RECTIFICATION_V3_SYNTHETIC_SMOKE_SHA;
|
|
assert.deepEqual((await readiness()).rollout.conversationalRectificationV3, {
|
|
protocol: "conversational-evidence-v3",
|
|
newCaseCreation: "enabled",
|
|
creationAudience: "smoke_only",
|
|
migrations: "ready",
|
|
syntheticSmoke: "pending",
|
|
readyForNewCases: false,
|
|
});
|
|
|
|
process.env.GITHUB_SHA = "deadbee";
|
|
process.env.RECTIFICATION_V3_SYNTHETIC_SMOKE_SHA = "deadbee";
|
|
assert.deepEqual((await readiness()).rollout.conversationalRectificationV3, {
|
|
protocol: "conversational-evidence-v3",
|
|
newCaseCreation: "paused",
|
|
creationAudience: "paused",
|
|
migrations: "ready",
|
|
syntheticSmoke: "pending",
|
|
readyForNewCases: false,
|
|
});
|
|
|
|
process.env.GITHUB_SHA = currentSha;
|
|
process.env.RECTIFICATION_V3_SYNTHETIC_SMOKE_SHA = oldSha;
|
|
assert.equal((await readiness()).rollout.conversationalRectificationV3.creationAudience, "smoke_only");
|
|
|
|
process.env.RECTIFICATION_V3_SYNTHETIC_SMOKE_SHA = currentSha;
|
|
const ready = await readiness();
|
|
assert.equal(ready.status, "ok");
|
|
assert.deepEqual(ready.rollout.conversationalRectificationV3, {
|
|
protocol: "conversational-evidence-v3",
|
|
newCaseCreation: "enabled",
|
|
creationAudience: "public",
|
|
migrations: "ready",
|
|
syntheticSmoke: "matched",
|
|
readyForNewCases: true,
|
|
});
|
|
|
|
delete process.env.SUPABASE_SERVICE_ROLE_KEY;
|
|
const blocked = await readiness();
|
|
assert.equal(blocked.status, "blocked");
|
|
assert.equal(blocked.rollout.conversationalRectificationV3.syntheticSmoke, "matched");
|
|
assert.equal(blocked.rollout.conversationalRectificationV3.readyForNewCases, false);
|
|
} finally {
|
|
globalThis.fetch = originalFetch;
|
|
for (const key of keys) {
|
|
const value = prior[key];
|
|
if (value === undefined) delete process.env[key];
|
|
else process.env[key] = value;
|
|
}
|
|
}
|
|
});
|
|
|
|
test("production API probes health rapidly while a replacement container starts", () => {
|
|
const compose = readFileSync(new URL("../../deploy/docker-compose.server.yml", import.meta.url), "utf8");
|
|
|
|
assert.match(compose, /healthcheck:[\s\S]*start_period:\s*30s[\s\S]*start_interval:\s*1s/);
|
|
});
|