import assert from "node:assert/strict"; import { chmodSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { spawnSync } from "node:child_process"; import test from "node:test"; import { fileURLToPath } from "node:url"; function serviceBlock(compose: string, service: string) { const match = compose.match(new RegExp(`^ ${service}:\\n([\\s\\S]*?)(?=^ [a-z][a-z0-9_-]*:|^volumes:)`, "m")); assert.ok(match, `expected ${service} service in compose file`); return match[1]; } function webHealthcheckBlock(web: string) { const match = web.match(/^ healthcheck:\n((?: .*\n?)*)/m); assert.ok(match, "expected a web healthcheck in compose file"); return match[1]; } test("health endpoint exposes deployment identity for production verification", () => { const source = readFileSync( new URL("../src/app/api/health/route.ts", import.meta.url), "utf8", ); assert.match(source, /deployment:/); assert.match(source, /GITHUB_SHA/); assert.match(source, /VERCEL_GIT_COMMIT_SHA/); assert.match(source, /gitCommit/); assert.match(source, /loadLanguageModelCatalog/); assert.match(source, /const defaults = catalog\.models\.filter/); assert.match(source, /defaults\.length === 1/); assert.match(source, /default_model_unavailable/); assert.match(source, /message === "database_model_catalog_unavailable"/); assert.match(source, /: \{ status: "degraded", message \}/); assert.match(source, /status: response\.ok \? "ok" : "blocked"/); assert.match(source, /status === "blocked" \? 503 : 200/); assert.doesNotMatch(source, /anyEnvCheck\(\["LLM_MODELS_JSON"|OPENAI_API_KEY|DEEPSEEK_API_KEY|LLM_API_KEY/); }); test("GitHub mirror cannot deploy production", () => { const workflow = readFileSync( new URL("../../.github/workflows/deploy-production.yml", import.meta.url), "utf8", ); assert.match(workflow, /^on:\n\s+workflow_dispatch:/m); assert.match(workflow, /Refuse deployment from the mirror/); assert.match(workflow, /exit 1/); assert.doesNotMatch(workflow, /ssh|rsync|docker compose/); }); test("server compose accepts staging paths while preserving production defaults", () => { const compose = readFileSync( new URL("../../deploy/docker-compose.server.yml", import.meta.url), "utf8", ); assert.match( compose, /env_file:\s*\n\s*- \$\{APP_ENV_FILE:-\.\.\/\.env\.production\}/, ); assert.match( compose, /\$\{CADDYFILE_PATH:-\.\/Caddyfile\}:\/etc\/caddy\/Caddyfile:ro/, ); assert.match( compose, /SITE_ADDRESS: \$\{SITE_ADDRESS:-https:\/\/jyotisha\.chat\}/, ); assert.doesNotMatch(compose, /ADMIN_SITE_ADDRESS|AUTH_ADMIN_ORIGIN|BETTER_AUTH_ADMIN_SECRET/); }); test("server compose defaults to local images without removing either build", () => { const composeFile = fileURLToPath( new URL("../../deploy/docker-compose.server.yml", import.meta.url), ); const compose = readFileSync(composeFile, "utf8"); assert.match(compose, /^\s+image: \$\{API_IMAGE:-jyotisha-api:local\}$/m); assert.match(compose, /^\s+image: \$\{WEB_IMAGE:-jyotisha-web:local\}$/m); const root = mkdtempSync(join(tmpdir(), "jyotisha-server-compose-")); const appEnvFile = join(root, ".env.production"); writeFileSync(appEnvFile, "RUNTIME_FIXTURE=1\n"); chmodSync(appEnvFile, 0o600); const env: NodeJS.ProcessEnv = { ...process.env, APP_ENV_FILE: appEnvFile, CADDYFILE_PATH: fileURLToPath( new URL("../../deploy/Caddyfile", import.meta.url), ), GITHUB_SHA: "0000000000000000000000000000000000000000", NEXT_PUBLIC_SUPABASE_URL: "https://placeholder.supabase.co", NEXT_PUBLIC_SUPABASE_ANON_KEY: "placeholder", }; delete env.API_IMAGE; delete env.WEB_IMAGE; try { const result = spawnSync( "docker", ["compose", "-f", composeFile, "config", "--format", "json"], { encoding: "utf8", env }, ); assert.equal(result.status, 0, result.stderr); const rendered = JSON.parse(result.stdout) as { services: Record; }; assert.equal(rendered.services.api.image, "jyotisha-api:local"); assert.equal(rendered.services.web.image, "jyotisha-web:local"); assert.ok(rendered.services.api.build, "api build definition was removed"); assert.ok(rendered.services.web.build, "web build definition was removed"); } finally { rmSync(root, { recursive: true, force: true }); } }); test("staging Caddy serves the same app on two exact hosts", () => { const caddy = readFileSync( new URL("../../deploy/Caddyfile.staging", import.meta.url), "utf8", ); assert.match(caddy, /\{\$SITE_ADDRESS:https:\/\/staging\.jyotisha\.chat\}/); assert.match(caddy, /@adminPaths path \/admin \/admin\/\* \/api\/admin\/\*/); assert.match(caddy, /respond @adminPaths "Not found" 404/); assert.match(caddy, /^https:\/\/admin\.staging\.jyotisha\.chat \{$/m); assert.equal((caddy.match(/reverse_proxy web:3000/g) ?? []).length, 2); assert.match(caddy, /@root path \/\n\s+redir @root \/admin 308/); assert.doesNotMatch(caddy, /\*\.staging\.jyotisha\.chat|:443 \{/); assert.doesNotMatch(caddy, /www\.jyotisha\.chat/); }); test("self-hosted production Caddy isolates user and admin hosts", () => { const caddy = readFileSync( new URL("../../deploy/Caddyfile.production.selfhosted", import.meta.url), "utf8", ); assert.match(caddy, /\{\$SITE_ADDRESS:https:\/\/jyotisha\.chat\}/); assert.match(caddy, /@adminPaths path \/admin \/admin\/\* \/api\/admin\/\*/); assert.match(caddy, /respond @adminPaths "Not found" 404/); assert.match(caddy, /^https:\/\/admin\.jyotisha\.chat \{$/m); assert.match(caddy, /^https:\/\/www\.jyotisha\.chat \{$/m); assert.match(caddy, /redir https:\/\/jyotisha\.chat\{uri\} 308/); assert.equal((caddy.match(/reverse_proxy web:3000/g) ?? []).length, 2); }); test("staging deploy consumes only the isolated staging environment and tested revision", () => { const qualityGate = readFileSync( new URL("../../.github/workflows/backend-quality-gate.yml", import.meta.url), "utf8", ); const workflow = readFileSync( new URL("../../.github/workflows/deploy-staging.yml", import.meta.url), "utf8", ); const syncController = readFileSync( new URL("../../deploy/sync-staging-tree.sh", import.meta.url), "utf8", ); assert.match(qualityGate, /push:\s*\n\s*branches: \[staging\]/); assert.match(workflow, /workflows: \["Staging Backend Quality Gate"\]/); assert.match( workflow, /github\.event\.workflow_run\.head_branch == 'staging'/, ); assert.match(workflow, /actions: read/); assert.match(workflow, /packages: read/); assert.match(workflow, /environment:\s*\n\s*name: staging/); assert.match(workflow, /deploy_sha:/); assert.match(workflow, /\^\[0-9a-f\]\{40\}\$/); assert.match( workflow, /actions\/workflows\/backend-quality-gate\.yml\/runs\?head_sha=/, ); assert.match(workflow, /STAGING_SSH_PRIVATE_KEY/); assert.match(workflow, /vars\.STAGING_HOST/); assert.match(workflow, /vars\.STAGING_KNOWN_HOSTS/); assert.match(workflow, /test "\$DEPLOY_HOST" = "118\.26\.111\.127"/); assert.match(workflow, /test "\$DEPLOY_USER" = "deploy"/); assert.match(workflow, /test "\$DEPLOY_PATH" = "\/opt\/jyotisha-staging"/); assert.match( workflow, /--include='\/deploy\/' --include='\/deploy\/\*\*\*' --exclude='\*'/, ); assert.match(workflow, /run-staging-deploy\.sh/); assert.match(workflow, /steps\.images\.outputs\.api_image/); assert.match(workflow, /steps\.images\.outputs\.web_image/); assert.doesNotMatch(workflow, /PRODUCTION_SSH_PRIVATE_KEY/); assert.doesNotMatch(workflow, /103\.117\.123\.53/); assert.match(syncController, /--exclude='\/\.env\*'/); assert.match(syncController, /--exclude='\/\.docker\/'/); assert.match(syncController, /--exclude='\/backups\/'/); }); test("staging model provider env preparation removes legacy settings and keeps one stable key", () => { const prepare = fileURLToPath( new URL("../../deploy/prepare-staging-model-provider-env.sh", import.meta.url), ); const root = mkdtempSync(join(tmpdir(), "jyotisha-staging-model-env-")); const envFile = join(root, ".env.staging"); try { writeFileSync(envFile, "APP_ENV_FILE=../.env.staging\nOPENAI_API_KEY=legacy\nLLM_MODEL=legacy\n"); chmodSync(envFile, 0o600); const first = spawnSync("bash", [prepare, envFile], { encoding: "utf8" }); assert.equal(first.status, 0, first.stderr); const prepared = readFileSync(envFile, "utf8"); assert.match(prepared, /^APP_ENV_FILE=\.\.\/\.env\.staging$/m); assert.doesNotMatch(prepared, /^(OPENAI_API_KEY|LLM_MODEL)=/m); const key = prepared.match(/^MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=([A-Za-z0-9+/]{43}=)$/m)?.[1]; assert.ok(key); assert.equal(statSync(envFile).mode & 0o777, 0o600); const second = spawnSync("bash", [prepare, envFile], { encoding: "utf8" }); assert.equal(second.status, 0, second.stderr); const secondKey = readFileSync(envFile, "utf8").match(/^MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=([A-Za-z0-9+/]{43}=)$/m)?.[1]; assert.equal(secondKey, key); } finally { rmSync(root, { recursive: true, force: true }); } }); test("staging env validator rejects selector drift, duplicates, and unsafe permissions", () => { const validator = fileURLToPath( new URL("../../deploy/validate-staging-env.sh", import.meta.url), ); const root = mkdtempSync(join(tmpdir(), "jyotisha-staging-env-")); const envFile = join(root, ".env.staging"); const composeFile = join(root, "compose.yml"); const validSelectors = [ "APP_ENV_FILE=../.env.staging", "CADDYFILE_PATH=./Caddyfile.staging", "SITE_ADDRESS=https://staging.jyotisha.chat", "AUTH_PROVIDER=self-hosted", "SELF_HOSTED_IDENTITY_ENABLED=true", "AUTH_USER_ORIGIN=https://staging.jyotisha.chat", "ADMIN_USER_ORIGIN=https://admin.staging.jyotisha.chat", "IDENTITY_DATABASE_URL=postgresql://identity_runtime:identity-runtime-test-password@postgres:5432/jyotisha", "APP_DATABASE_URL=postgresql://app_runtime:app-runtime-test-password@postgres:5432/jyotisha", "SERVICE_DATABASE_URL=postgresql://service_runtime:service-runtime-test-password@postgres:5432/jyotisha", "ADMIN_DATABASE_URL=postgresql://admin_runtime:admin-runtime-test-password@postgres:5432/jyotisha", "BETTER_AUTH_USER_SECRET=user-secret-that-is-at-least-32-bytes-long", "RESEND_API_KEY=re_test_key_that_must_not_be_printed", "RESEND_FROM_EMAIL=Jyotisha Staging ", "ADMIN_EMAILS=admin@example.com", "EPAY_CONFIG_ENCRYPTION_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", "MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=", "EPAY_CHAT_ENABLED=false", "JYOTISH_DYNAMIC_RECTIFICATION_TOKEN=dynamic-token-that-is-at-least-32-bytes", "PERSONAL_REPORT_ENABLED=true", "PERSONAL_REPORT_DAILY_LIMIT=5", ]; const run = () => spawnSync("bash", [validator, envFile], { encoding: "utf8" }); const writeEnv = (lines: string[], mode = 0o600) => { writeFileSync(envFile, `${lines.join("\n")}\n`); chmodSync(envFile, mode); }; try { writeFileSync( composeFile, [ "services:", " probe:", " image: alpine", " environment:", " SELECTED: ${APP_ENV_FILE}", "", ].join("\n"), ); writeEnv(validSelectors); assert.equal(run().status, 0); writeEnv(validSelectors.map((line) => line.startsWith("EPAY_CONFIG_ENCRYPTION_KEY=") ? "EPAY_CONFIG_ENCRYPTION_KEY=invalid" : line)); assert.notEqual(run().status, 0); writeEnv(validSelectors.map((line) => line.startsWith("MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=") ? "MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=invalid" : line)); assert.notEqual(run().status, 0); for (const legacyModelSetting of [ "OPENAI_API_KEY=legacy", "ANTHROPIC_API_KEY=legacy", "DEEPSEEK_API_KEY=legacy", "LLM_API_KEY=legacy", "LLM_MODELS_JSON=[]", "LLM_BASE_URL=https://models.example.com", "LLM_MODEL=legacy-model", "LLM_DEFAULT_MODEL_ID=legacy-model", "LLM_PROVIDER_ID=legacy-provider", "MASTRA_MODEL=legacy-model", "MODEL_PROVIDER_TRUSTED_EDGE_API_KEY=legacy", ]) { writeEnv([...validSelectors, legacyModelSetting]); const legacyResult = run(); assert.notEqual(legacyResult.status, 0, `${legacyModelSetting} must be rejected`); assert.match(`${legacyResult.stdout}${legacyResult.stderr}`, /legacy model environment settings are forbidden/); } writeEnv(validSelectors.map((line) => line.startsWith("EPAY_CHAT_ENABLED=") ? "EPAY_CHAT_ENABLED=true" : line)); assert.notEqual(run().status, 0); writeEnv(validSelectors.map((line) => line.startsWith("PERSONAL_REPORT_ENABLED=") ? "PERSONAL_REPORT_ENABLED=false" : line)); assert.equal(run().status, 0); writeEnv(validSelectors.map((line) => line.startsWith("PERSONAL_REPORT_ENABLED=") ? "PERSONAL_REPORT_ENABLED=TRUE" : line)); assert.notEqual(run().status, 0); writeEnv(validSelectors.map((line) => line.startsWith("PERSONAL_REPORT_DAILY_LIMIT=") ? "PERSONAL_REPORT_DAILY_LIMIT=0" : line)); assert.notEqual(run().status, 0); writeEnv(validSelectors.map((line) => line.startsWith("PERSONAL_REPORT_DAILY_LIMIT=") ? "PERSONAL_REPORT_DAILY_LIMIT=5x" : line)); assert.notEqual(run().status, 0); writeEnv(validSelectors); const shellOverride = spawnSync( "docker", [ "compose", "--env-file", envFile, "-f", composeFile, "config", "--format", "json", ], { encoding: "utf8", env: { ...process.env, APP_ENV_FILE: "../.env.production" }, }, ); assert.equal(shellOverride.status, 0, shellOverride.stderr); assert.equal( JSON.parse(shellOverride.stdout).services.probe.environment.SELECTED, "../.env.production", ); writeEnv(["APP_ENV_FILE=../.env.production", ...validSelectors.slice(1)]); assert.notEqual(run().status, 0); writeEnv([...validSelectors, "SITE_ADDRESS=https://example.invalid"]); assert.notEqual(run().status, 0); writeEnv([...validSelectors, "APP_ENV_FILE = ../.env.production"]); assert.notEqual(run().status, 0); const rendered = spawnSync( "docker", [ "compose", "--env-file", envFile, "-f", composeFile, "config", "--format", "json", ], { encoding: "utf8" }, ); assert.equal(rendered.status, 0, rendered.stderr); assert.equal( JSON.parse(rendered.stdout).services.probe.environment.SELECTED, "../.env.production", ); writeEnv([...validSelectors, "export CADDYFILE_PATH=./Caddyfile"]); assert.notEqual(run().status, 0); writeEnv([...validSelectors, "SITE_ADDRESS"]); assert.notEqual(run().status, 0); writeEnv( validSelectors.map((line) => line.startsWith("AUTH_PROVIDER=") ? "AUTH_PROVIDER=supabase" : line, ), ); assert.notEqual(run().status, 0); writeEnv([ ...validSelectors, "BETTER_AUTH_USER_SECRET=duplicate-secret-that-must-not-be-printed", ]); const duplicateSecret = run(); assert.notEqual(duplicateSecret.status, 0); assert.doesNotMatch( `${duplicateSecret.stdout}${duplicateSecret.stderr}`, /duplicate-secret-that-must-not-be-printed|re_test_key_that_must_not_be_printed/, ); writeEnv(validSelectors, 0o644); assert.notEqual(run().status, 0); } finally { rmSync(root, { recursive: true, force: true }); } }); test("production traffic waits for a healthy web container and retries short replacement gaps", () => { const compose = readFileSync(new URL("../../deploy/docker-compose.server.yml", import.meta.url), "utf8"); const caddyfile = readFileSync(new URL("../../deploy/Caddyfile", import.meta.url), "utf8"); const web = serviceBlock(compose, "web"); const caddy = serviceBlock(compose, "caddy"); const healthcheck = webHealthcheckBlock(web); assert.match(web, /GITHUB_SHA: \$\{GITHUB_SHA\}/); assert.match(web, /healthcheck:\n\s+test: \["CMD", "node", "-e", "fetch\('http:\/\/127\.0\.0\.1:3000\/api\/health'\)\.then\(r=>\{if\(!r\.ok\)process\.exit\(1\)\}\)"\]/); assert.match(healthcheck, /^ interval: 30s$/m); assert.match(healthcheck, /^ timeout: 5s$/m); assert.match(healthcheck, /^ retries: 5$/m); assert.match(healthcheck, /^ start_period: 30s$/m); assert.match(healthcheck, /^ start_interval: 1s$/m); assert.match(caddy, /web:\n\s+condition: service_healthy/); assert.match(caddyfile, /reverse_proxy web:3000 \{\n\s+lb_try_duration 10s\n\s+lb_try_interval 250ms\n\s+\}/); }); test("Gitea production verification binds the exact requested SHA", () => { const workflow = readFileSync(new URL("../../.gitea/workflows/deploy-production.yml", import.meta.url), "utf8"); const runner = readFileSync(new URL("../../deploy/run-production-deploy.sh", import.meta.url), "utf8"); assert.match(workflow, /REQUESTED_SHA: \$\{\{ inputs\.deploy_sha \}\}/); assert.match(workflow, /main and staging must identify the same reviewed release/); assert.match(workflow, /public staging has not accepted the requested SHA/); assert.match(workflow, /DEPLOY_SHA='\$DEPLOY_SHA'/); assert.match(runner, /publicBody\.deployment\?\.gitCommit === process\.env\.EXPECTED_SHA/); assert.match(runner, /VERIFICATION_MODE/); assert.match(runner, /select 1 as ready/); }); test("production API probes health rapidly while a replacement container starts", () => { const compose = readFileSync(new URL("../../deploy/docker-compose.server.yml", import.meta.url), "utf8"); assert.match(compose, /healthcheck:[\s\S]*start_period:\s*30s[\s\S]*start_interval:\s*1s/); });