root/Jyotisha
2
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity
Files
9b07e12b0c708c4a9d7c65ee18ac5e145502de9a
Jyotisha/frontend/tests/health-deployment.test.ts
T
Codex 9b07e12b0c
Staging Backend Quality Gate / validate (push) Has been cancelled
Details
Staging Backend Quality Gate / publish (push) Has been cancelled
Details
ops: prepare self-hosted production migration
2026-08-08 21:12:55 -04:00

440 lines
18 KiB
TypeScript
Raw Blame History

import assert from "node:assert/strict";
import {
chmodSync,
mkdtempSync,
readFileSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { spawnSync } from "node:child_process";
import test from "node:test";
import { fileURLToPath } from "node:url";
function serviceBlock(compose: string, service: string) {
const match = compose.match(new RegExp(`^ ${service}:\\n([\\s\\S]*?)(?=^ [a-z][a-z0-9_-]*:|^volumes:)`, "m"));
assert.ok(match, `expected ${service} service in compose file`);
return match[1];
}
function webHealthcheckBlock(web: string) {
const match = web.match(/^ healthcheck:\n((?: .*\n?)*)/m);
assert.ok(match, "expected a web healthcheck in compose file");
return match[1];
}
test("health endpoint exposes deployment identity for production verification", () => {
const source = readFileSync(
new URL("../src/app/api/health/route.ts", import.meta.url),
"utf8",
);
assert.match(source, /deployment:/);
assert.match(source, /GITHUB_SHA/);
assert.match(source, /VERCEL_GIT_COMMIT_SHA/);
assert.match(source, /gitCommit/);
assert.match(source, /loadLanguageModelCatalog/);
assert.match(source, /const defaults = catalog\.models\.filter/);
assert.match(source, /defaults\.length === 1/);
assert.match(source, /default_model_unavailable/);
assert.match(source, /message === "database_model_catalog_unavailable"/);
assert.match(source, /: \{ status: "degraded", message \}/);
assert.match(source, /status: response\.ok \? "ok" : "blocked"/);
assert.match(source, /status === "blocked" \? 503 : 200/);
assert.doesNotMatch(source, /anyEnvCheck\(\["LLM_MODELS_JSON"|OPENAI_API_KEY|DEEPSEEK_API_KEY|LLM_API_KEY/);
});
test("GitHub mirror cannot deploy production", () => {
const workflow = readFileSync(
new URL("../../.github/workflows/deploy-production.yml", import.meta.url),
"utf8",
);
assert.match(workflow, /^on:\n\s+workflow_dispatch:/m);
assert.match(workflow, /Refuse deployment from the mirror/);
assert.match(workflow, /exit 1/);
assert.doesNotMatch(workflow, /ssh|rsync|docker compose/);
});
test("server compose accepts staging paths while preserving production defaults", () => {
const compose = readFileSync(
new URL("../../deploy/docker-compose.server.yml", import.meta.url),
"utf8",
);
assert.match(
compose,
/env_file:\s*\n\s*- \$\{APP_ENV_FILE:-\.\.\/\.env\.production\}/,
);
assert.match(
compose,
/\$\{CADDYFILE_PATH:-\.\/Caddyfile\}:\/etc\/caddy\/Caddyfile:ro/,
);
assert.match(
compose,
/SITE_ADDRESS: \$\{SITE_ADDRESS:-https:\/\/jyotisha\.chat\}/,
);
assert.doesNotMatch(compose, /ADMIN_SITE_ADDRESS|AUTH_ADMIN_ORIGIN|BETTER_AUTH_ADMIN_SECRET/);
});
test("server compose defaults to local images without removing either build", () => {
const composeFile = fileURLToPath(
new URL("../../deploy/docker-compose.server.yml", import.meta.url),
);
const compose = readFileSync(composeFile, "utf8");
assert.match(compose, /^\s+image: \$\{API_IMAGE:-jyotisha-api:local\}$/m);
assert.match(compose, /^\s+image: \$\{WEB_IMAGE:-jyotisha-web:local\}$/m);
const root = mkdtempSync(join(tmpdir(), "jyotisha-server-compose-"));
const appEnvFile = join(root, ".env.production");
writeFileSync(appEnvFile, "RUNTIME_FIXTURE=1\n");
chmodSync(appEnvFile, 0o600);
const env: NodeJS.ProcessEnv = {
...process.env,
APP_ENV_FILE: appEnvFile,
CADDYFILE_PATH: fileURLToPath(
new URL("../../deploy/Caddyfile", import.meta.url),
),
GITHUB_SHA: "0000000000000000000000000000000000000000",
NEXT_PUBLIC_SUPABASE_URL: "https://placeholder.supabase.co",
NEXT_PUBLIC_SUPABASE_ANON_KEY: "placeholder",
};
delete env.API_IMAGE;
delete env.WEB_IMAGE;
try {
const result = spawnSync(
"docker",
["compose", "-f", composeFile, "config", "--format", "json"],
{ encoding: "utf8", env },
);
assert.equal(result.status, 0, result.stderr);
const rendered = JSON.parse(result.stdout) as {
services: Record<string, { build?: unknown; image?: string }>;
};
assert.equal(rendered.services.api.image, "jyotisha-api:local");
assert.equal(rendered.services.web.image, "jyotisha-web:local");
assert.ok(rendered.services.api.build, "api build definition was removed");
assert.ok(rendered.services.web.build, "web build definition was removed");
} finally {
rmSync(root, { recursive: true, force: true });
}
});
test("staging Caddy serves the same app on two exact hosts", () => {
const caddy = readFileSync(
new URL("../../deploy/Caddyfile.staging", import.meta.url),
"utf8",
);
assert.match(caddy, /\{\$SITE_ADDRESS:https:\/\/staging\.jyotisha\.chat\}/);
assert.match(caddy, /@adminPaths path \/admin \/admin\/\* \/api\/admin\/\*/);
assert.match(caddy, /respond @adminPaths "Not found" 404/);
assert.match(caddy, /^https:\/\/admin\.staging\.jyotisha\.chat \{$/m);
assert.equal((caddy.match(/reverse_proxy web:3000/g) ?? []).length, 2);
assert.match(caddy, /@root path \/\n\s+redir @root \/admin 308/);
assert.doesNotMatch(caddy, /\*\.staging\.jyotisha\.chat|:443 \{/);
assert.doesNotMatch(caddy, /www\.jyotisha\.chat/);
});
test("self-hosted production Caddy isolates user and admin hosts", () => {
const caddy = readFileSync(
new URL("../../deploy/Caddyfile.production.selfhosted", import.meta.url),
"utf8",
);
assert.match(caddy, /\{\$SITE_ADDRESS:https:\/\/jyotisha\.chat\}/);
assert.match(caddy, /@adminPaths path \/admin \/admin\/\* \/api\/admin\/\*/);
assert.match(caddy, /respond @adminPaths "Not found" 404/);
assert.match(caddy, /^https:\/\/admin\.jyotisha\.chat \{$/m);
assert.match(caddy, /^https:\/\/www\.jyotisha\.chat \{$/m);
assert.match(caddy, /redir https:\/\/jyotisha\.chat\{uri\} 308/);
assert.equal((caddy.match(/reverse_proxy web:3000/g) ?? []).length, 2);
});
test("staging deploy consumes only the isolated staging environment and tested revision", () => {
const qualityGate = readFileSync(
new URL("../../.github/workflows/backend-quality-gate.yml", import.meta.url),
"utf8",
);
const workflow = readFileSync(
new URL("../../.github/workflows/deploy-staging.yml", import.meta.url),
"utf8",
);
const syncController = readFileSync(
new URL("../../deploy/sync-staging-tree.sh", import.meta.url),
"utf8",
);
assert.match(qualityGate, /push:\s*\n\s*branches: \[staging\]/);
assert.match(workflow, /workflows: \["Staging Backend Quality Gate"\]/);
assert.match(
workflow,
/github\.event\.workflow_run\.head_branch == 'staging'/,
);
assert.match(workflow, /actions: read/);
assert.match(workflow, /packages: read/);
assert.match(workflow, /environment:\s*\n\s*name: staging/);
assert.match(workflow, /deploy_sha:/);
assert.match(workflow, /\^\[0-9a-f\]\{40\}\$/);
assert.match(
workflow,
/actions\/workflows\/backend-quality-gate\.yml\/runs\?head_sha=/,
);
assert.match(workflow, /STAGING_SSH_PRIVATE_KEY/);
assert.match(workflow, /vars\.STAGING_HOST/);
assert.match(workflow, /vars\.STAGING_KNOWN_HOSTS/);
assert.match(workflow, /test "\$DEPLOY_HOST" = "118\.26\.111\.127"/);
assert.match(workflow, /test "\$DEPLOY_USER" = "deploy"/);
assert.match(workflow, /test "\$DEPLOY_PATH" = "\/opt\/jyotisha-staging"/);
assert.match(
workflow,
/--include='\/deploy\/' --include='\/deploy\/\*\*\*' --exclude='\*'/,
);
assert.match(workflow, /run-staging-deploy\.sh/);
assert.match(workflow, /steps\.images\.outputs\.api_image/);
assert.match(workflow, /steps\.images\.outputs\.web_image/);
assert.doesNotMatch(workflow, /PRODUCTION_SSH_PRIVATE_KEY/);
assert.doesNotMatch(workflow, /103\.117\.123\.53/);
assert.match(syncController, /--exclude='\/\.env\*'/);
assert.match(syncController, /--exclude='\/\.docker\/'/);
assert.match(syncController, /--exclude='\/backups\/'/);
});
test("staging model provider env preparation removes legacy settings and keeps one stable key", () => {
const prepare = fileURLToPath(
new URL("../../deploy/prepare-staging-model-provider-env.sh", import.meta.url),
);
const root = mkdtempSync(join(tmpdir(), "jyotisha-staging-model-env-"));
const envFile = join(root, ".env.staging");
try {
writeFileSync(envFile, "APP_ENV_FILE=../.env.staging\nOPENAI_API_KEY=legacy\nLLM_MODEL=legacy\n");
chmodSync(envFile, 0o600);
const first = spawnSync("bash", [prepare, envFile], { encoding: "utf8" });
assert.equal(first.status, 0, first.stderr);
const prepared = readFileSync(envFile, "utf8");
assert.match(prepared, /^APP_ENV_FILE=\.\.\/\.env\.staging$/m);
assert.doesNotMatch(prepared, /^(OPENAI_API_KEY|LLM_MODEL)=/m);
const key = prepared.match(/^MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=([A-Za-z0-9+/]{43}=)$/m)?.[1];
assert.ok(key);
assert.equal(statSync(envFile).mode & 0o777, 0o600);
const second = spawnSync("bash", [prepare, envFile], { encoding: "utf8" });
assert.equal(second.status, 0, second.stderr);
const secondKey = readFileSync(envFile, "utf8").match(/^MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=([A-Za-z0-9+/]{43}=)$/m)?.[1];
assert.equal(secondKey, key);
} finally {
rmSync(root, { recursive: true, force: true });
}
});
test("staging env validator rejects selector drift, duplicates, and unsafe permissions", () => {
const validator = fileURLToPath(
new URL("../../deploy/validate-staging-env.sh", import.meta.url),
);
const root = mkdtempSync(join(tmpdir(), "jyotisha-staging-env-"));
const envFile = join(root, ".env.staging");
const composeFile = join(root, "compose.yml");
const validSelectors = [
"APP_ENV_FILE=../.env.staging",
"CADDYFILE_PATH=./Caddyfile.staging",
"SITE_ADDRESS=https://staging.jyotisha.chat",
"AUTH_PROVIDER=self-hosted",
"SELF_HOSTED_IDENTITY_ENABLED=true",
"AUTH_USER_ORIGIN=https://staging.jyotisha.chat",
"ADMIN_USER_ORIGIN=https://admin.staging.jyotisha.chat",
"IDENTITY_DATABASE_URL=postgresql://identity_runtime:identity-runtime-test-password@postgres:5432/jyotisha",
"APP_DATABASE_URL=postgresql://app_runtime:app-runtime-test-password@postgres:5432/jyotisha",
"SERVICE_DATABASE_URL=postgresql://service_runtime:service-runtime-test-password@postgres:5432/jyotisha",
"ADMIN_DATABASE_URL=postgresql://admin_runtime:admin-runtime-test-password@postgres:5432/jyotisha",
"BETTER_AUTH_USER_SECRET=user-secret-that-is-at-least-32-bytes-long",
"RESEND_API_KEY=re_test_key_that_must_not_be_printed",
"RESEND_FROM_EMAIL=Jyotisha Staging <login@staging.jyotisha.chat>",
"ADMIN_EMAILS=admin@example.com",
"EPAY_CONFIG_ENCRYPTION_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
"MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=",
"EPAY_CHAT_ENABLED=false",
"JYOTISH_DYNAMIC_RECTIFICATION_TOKEN=dynamic-token-that-is-at-least-32-bytes",
"PERSONAL_REPORT_ENABLED=true",
"PERSONAL_REPORT_DAILY_LIMIT=5",
];
const run = () =>
spawnSync("bash", [validator, envFile], { encoding: "utf8" });
const writeEnv = (lines: string[], mode = 0o600) => {
writeFileSync(envFile, `${lines.join("\n")}\n`);
chmodSync(envFile, mode);
};
try {
writeFileSync(
composeFile,
[
"services:",
" probe:",
" image: alpine",
" environment:",
" SELECTED: ${APP_ENV_FILE}",
"",
].join("\n"),
);
writeEnv(validSelectors);
assert.equal(run().status, 0);
writeEnv(validSelectors.map((line) => line.startsWith("EPAY_CONFIG_ENCRYPTION_KEY=") ? "EPAY_CONFIG_ENCRYPTION_KEY=invalid" : line));
assert.notEqual(run().status, 0);
writeEnv(validSelectors.map((line) => line.startsWith("MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=") ? "MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=invalid" : line));
assert.notEqual(run().status, 0);
for (const legacyModelSetting of [
"OPENAI_API_KEY=legacy",
"ANTHROPIC_API_KEY=legacy",
"DEEPSEEK_API_KEY=legacy",
"LLM_API_KEY=legacy",
"LLM_MODELS_JSON=[]",
"LLM_BASE_URL=https://models.example.com",
"LLM_MODEL=legacy-model",
"LLM_DEFAULT_MODEL_ID=legacy-model",
"LLM_PROVIDER_ID=legacy-provider",
"MASTRA_MODEL=legacy-model",
"MODEL_PROVIDER_TRUSTED_EDGE_API_KEY=legacy",
]) {
writeEnv([...validSelectors, legacyModelSetting]);
const legacyResult = run();
assert.notEqual(legacyResult.status, 0, `${legacyModelSetting} must be rejected`);
assert.match(`${legacyResult.stdout}${legacyResult.stderr}`, /legacy model environment settings are forbidden/);
}
writeEnv(validSelectors.map((line) => line.startsWith("EPAY_CHAT_ENABLED=") ? "EPAY_CHAT_ENABLED=true" : line));
assert.notEqual(run().status, 0);
writeEnv(validSelectors.map((line) => line.startsWith("PERSONAL_REPORT_ENABLED=") ? "PERSONAL_REPORT_ENABLED=false" : line));
assert.equal(run().status, 0);
writeEnv(validSelectors.map((line) => line.startsWith("PERSONAL_REPORT_ENABLED=") ? "PERSONAL_REPORT_ENABLED=TRUE" : line));
assert.notEqual(run().status, 0);
writeEnv(validSelectors.map((line) => line.startsWith("PERSONAL_REPORT_DAILY_LIMIT=") ? "PERSONAL_REPORT_DAILY_LIMIT=0" : line));
assert.notEqual(run().status, 0);
writeEnv(validSelectors.map((line) => line.startsWith("PERSONAL_REPORT_DAILY_LIMIT=") ? "PERSONAL_REPORT_DAILY_LIMIT=5x" : line));
assert.notEqual(run().status, 0);
writeEnv(validSelectors);
const shellOverride = spawnSync(
"docker",
[
"compose",
"--env-file",
envFile,
"-f",
composeFile,
"config",
"--format",
"json",
],
{
encoding: "utf8",
env: { ...process.env, APP_ENV_FILE: "../.env.production" },
},
);
assert.equal(shellOverride.status, 0, shellOverride.stderr);
assert.equal(
JSON.parse(shellOverride.stdout).services.probe.environment.SELECTED,
"../.env.production",
);
writeEnv(["APP_ENV_FILE=../.env.production", ...validSelectors.slice(1)]);
assert.notEqual(run().status, 0);
writeEnv([...validSelectors, "SITE_ADDRESS=https://example.invalid"]);
assert.notEqual(run().status, 0);
writeEnv([...validSelectors, "APP_ENV_FILE = ../.env.production"]);
assert.notEqual(run().status, 0);
const rendered = spawnSync(
"docker",
[
"compose",
"--env-file",
envFile,
"-f",
composeFile,
"config",
"--format",
"json",
],
{ encoding: "utf8" },
);
assert.equal(rendered.status, 0, rendered.stderr);
assert.equal(
JSON.parse(rendered.stdout).services.probe.environment.SELECTED,
"../.env.production",
);
writeEnv([...validSelectors, "export CADDYFILE_PATH=./Caddyfile"]);
assert.notEqual(run().status, 0);
writeEnv([...validSelectors, "SITE_ADDRESS"]);
assert.notEqual(run().status, 0);
writeEnv(
validSelectors.map((line) =>
line.startsWith("AUTH_PROVIDER=")
? "AUTH_PROVIDER=supabase"
: line,
),
);
assert.notEqual(run().status, 0);
writeEnv([
...validSelectors,
"BETTER_AUTH_USER_SECRET=duplicate-secret-that-must-not-be-printed",
]);
const duplicateSecret = run();
assert.notEqual(duplicateSecret.status, 0);
assert.doesNotMatch(
`${duplicateSecret.stdout}${duplicateSecret.stderr}`,
/duplicate-secret-that-must-not-be-printed|re_test_key_that_must_not_be_printed/,
);
writeEnv(validSelectors, 0o644);
assert.notEqual(run().status, 0);
} finally {
rmSync(root, { recursive: true, force: true });
}
});
test("production traffic waits for a healthy web container and retries short replacement gaps", () => {
const compose = readFileSync(new URL("../../deploy/docker-compose.server.yml", import.meta.url), "utf8");
const caddyfile = readFileSync(new URL("../../deploy/Caddyfile", import.meta.url), "utf8");
const web = serviceBlock(compose, "web");
const caddy = serviceBlock(compose, "caddy");
const healthcheck = webHealthcheckBlock(web);
assert.match(web, /GITHUB_SHA: \$\{GITHUB_SHA\}/);
assert.match(web, /healthcheck:\n\s+test: \["CMD", "node", "-e", "fetch\('http:\/\/127\.0\.0\.1:3000\/api\/health'\)\.then\(r=>\{if\(!r\.ok\)process\.exit\(1\)\}\)"\]/);
assert.match(healthcheck, /^ interval: 30s$/m);
assert.match(healthcheck, /^ timeout: 5s$/m);
assert.match(healthcheck, /^ retries: 5$/m);
assert.match(healthcheck, /^ start_period: 30s$/m);
assert.match(healthcheck, /^ start_interval: 1s$/m);
assert.match(caddy, /web:\n\s+condition: service_healthy/);
assert.match(caddyfile, /reverse_proxy web:3000 \{\n\s+lb_try_duration 10s\n\s+lb_try_interval 250ms\n\s+\}/);
});
test("Gitea production verification binds the exact requested SHA", () => {
const workflow = readFileSync(new URL("../../.gitea/workflows/deploy-production.yml", import.meta.url), "utf8");
const runner = readFileSync(new URL("../../deploy/run-production-deploy.sh", import.meta.url), "utf8");
assert.match(workflow, /REQUESTED_SHA: \$\{\{ inputs\.deploy_sha \}\}/);
assert.match(workflow, /main and staging must identify the same reviewed release/);
assert.match(workflow, /public staging has not accepted the requested SHA/);
assert.match(workflow, /DEPLOY_SHA='\$DEPLOY_SHA'/);
assert.match(runner, /publicBody\.deployment\?\.gitCommit === process\.env\.EXPECTED_SHA/);
assert.match(runner, /VERIFICATION_MODE/);
assert.match(runner, /select 1 as ready/);
});
test("production API probes health rapidly while a replacement container starts", () => {
const compose = readFileSync(new URL("../../deploy/docker-compose.server.yml", import.meta.url), "utf8");
assert.match(compose, /healthcheck:[\s\S]*start_period:\s*30s[\s\S]*start_interval:\s*1s/);
});
Reference in New Issue View Git Blame Copy Permalink