Jyotisha/frontend/tests/health-deployment.test.ts

import assert from "node:assert/strict";
import {
  chmodSync,
  mkdtempSync,
  readFileSync,
  rmSync,
  writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { spawnSync } from "node:child_process";
import test from "node:test";
import { fileURLToPath } from "node:url";

function serviceBlock(compose: string, service: string) {
  const match = compose.match(new RegExp(`^  ${service}:\\n([\\s\\S]*?)(?=^  [a-z][a-z0-9_-]*:|^volumes:)`, "m"));
  assert.ok(match, `expected ${service} service in compose file`);
  return match[1];
}

function webHealthcheckBlock(web: string) {
  const match = web.match(/^    healthcheck:\n((?:      .*\n?)*)/m);
  assert.ok(match, "expected a web healthcheck in compose file");
  return match[1];
}

function workflowStepBlock(workflow: string, stepName: string) {
  const match = workflow.match(new RegExp(`^      - name: ${stepName}\\n([\\s\\S]*?)(?=^      - name:|(?![\\s\\S]))`, "m"));
  assert.ok(match, `expected ${stepName} workflow step`);
  return match[1];
}

function escapeRegExp(value: string) {
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}

const testedShaExpression = "${{ github.sha }}";

function assertWorkflowUsesTestedSha(workflow: string) {
  const checkout = workflowStepBlock(workflow, "Checkout tested revision");
  const sync = workflowStepBlock(workflow, "Sync and rebuild");
  const verification = workflowStepBlock(workflow, "Verify production");
  const shaExpression = escapeRegExp(testedShaExpression);

  assert.match(checkout, new RegExp(`ref: ${shaExpression}`), "Checkout tested revision must check out the tested SHA");
  assert.match(sync, new RegExp(`DEPLOY_GIT_SHA: ${shaExpression}`), "Sync and rebuild must use the tested SHA");
  assert.match(sync, /GITHUB_SHA='\$DEPLOY_GIT_SHA'/, "Sync and rebuild must inject its SHA into the web runtime");
  assert.match(verification, new RegExp(`DEPLOY_GIT_SHA: ${shaExpression}`), "Verify production must use the tested SHA");
  assert.match(verification, /curl --fail --silent --show-error https:\/\/jyotisha\.chat\/api\/health/);
  assert.match(verification, /get\("deployment", \{\}\)\.get\("gitCommit"/);
  assert.match(verification, /Production revision did not converge/);
}

test("health endpoint exposes deployment identity for production verification", () => {
  const source = readFileSync(
    new URL("../src/app/api/health/route.ts", import.meta.url),
    "utf8",
  );

  assert.match(source, /deployment:/);
  assert.match(source, /GITHUB_SHA/);
  assert.match(source, /VERCEL_GIT_COMMIT_SHA/);
  assert.match(source, /gitCommit/);
  assert.match(source, /loadLanguageModelCatalog/);
  assert.match(source, /const defaults = catalog\.models\.filter/);
  assert.match(source, /defaults\.length === 1/);
  assert.match(source, /default_model_unavailable/);
  assert.doesNotMatch(source, /anyEnvCheck\(\["LLM_MODELS_JSON"|OPENAI_API_KEY|DEEPSEEK_API_KEY|LLM_API_KEY/);
});

test("manual production deployment passes the selected revision into the web runtime", () => {
  const compose = readFileSync(
    new URL("../../deploy/docker-compose.server.yml", import.meta.url),
    "utf8",
  );
  const workflow = readFileSync(
    new URL("../../.github/workflows/deploy-production.yml", import.meta.url),
    "utf8",
  );

  assert.match(compose, /GITHUB_SHA: \$\{GITHUB_SHA\}/);
  assert.match(workflow, /workflow_dispatch:/);
  assert.doesNotMatch(workflow, /workflow_run:/);
  assert.match(workflow, /DEPLOY_GIT_SHA: \$\{\{ github\.sha \}\}/);
  assert.match(workflow, /GITHUB_SHA='\$DEPLOY_GIT_SHA'/);
  assert.match(workflow, /get\("deployment", \{\}\)\.get\("gitCommit"/);
  assert.match(workflow, /DEPLOY_GIT_SHA/);
  assert.match(workflow, /Production revision did not converge/);
  assert.match(workflow, /git ls-remote origin refs\/heads\/main/);
  assert.match(workflow, /steps\.revision\.outputs\.deploy == 'true'/);
});

test("server compose accepts staging paths while preserving production defaults", () => {
  const compose = readFileSync(
    new URL("../../deploy/docker-compose.server.yml", import.meta.url),
    "utf8",
  );

  assert.match(
    compose,
    /env_file:\s*\n\s*- \$\{APP_ENV_FILE:-\.\.\/\.env\.production\}/,
  );
  assert.match(
    compose,
    /\$\{CADDYFILE_PATH:-\.\/Caddyfile\}:\/etc\/caddy\/Caddyfile:ro/,
  );
  assert.match(
    compose,
    /SITE_ADDRESS: \$\{SITE_ADDRESS:-https:\/\/jyotisha\.chat\}/,
  );
  assert.doesNotMatch(compose, /ADMIN_SITE_ADDRESS|AUTH_ADMIN_ORIGIN|BETTER_AUTH_ADMIN_SECRET/);
});

test("server compose defaults to local images without removing either build", () => {
  const composeFile = fileURLToPath(
    new URL("../../deploy/docker-compose.server.yml", import.meta.url),
  );
  const compose = readFileSync(composeFile, "utf8");

  assert.match(compose, /^\s+image: \$\{API_IMAGE:-jyotisha-api:local\}$/m);
  assert.match(compose, /^\s+image: \$\{WEB_IMAGE:-jyotisha-web:local\}$/m);

  const root = mkdtempSync(join(tmpdir(), "jyotisha-server-compose-"));
  const appEnvFile = join(root, ".env.production");
  writeFileSync(appEnvFile, "RUNTIME_FIXTURE=1\n");
  chmodSync(appEnvFile, 0o600);

  const env: NodeJS.ProcessEnv = {
    ...process.env,
    APP_ENV_FILE: appEnvFile,
    CADDYFILE_PATH: fileURLToPath(
      new URL("../../deploy/Caddyfile", import.meta.url),
    ),
    GITHUB_SHA: "0000000000000000000000000000000000000000",
    NEXT_PUBLIC_SUPABASE_URL: "https://placeholder.supabase.co",
    NEXT_PUBLIC_SUPABASE_ANON_KEY: "placeholder",
  };
  delete env.API_IMAGE;
  delete env.WEB_IMAGE;

  try {
    const result = spawnSync(
      "docker",
      ["compose", "-f", composeFile, "config", "--format", "json"],
      { encoding: "utf8", env },
    );
    assert.equal(result.status, 0, result.stderr);
    const rendered = JSON.parse(result.stdout) as {
      services: Record<string, { build?: unknown; image?: string }>;
    };
    assert.equal(rendered.services.api.image, "jyotisha-api:local");
    assert.equal(rendered.services.web.image, "jyotisha-web:local");
    assert.ok(rendered.services.api.build, "api build definition was removed");
    assert.ok(rendered.services.web.build, "web build definition was removed");
  } finally {
    rmSync(root, { recursive: true, force: true });
  }
});

test("staging Caddy serves the same app on two exact hosts", () => {
  const caddy = readFileSync(
    new URL("../../deploy/Caddyfile.staging", import.meta.url),
    "utf8",
  );

  assert.match(caddy, /\{\$SITE_ADDRESS:https:\/\/staging\.jyotisha\.chat\}/);
  assert.match(caddy, /@adminPaths path \/admin \/admin\/\* \/api\/admin\/\*/);
  assert.match(caddy, /respond @adminPaths "Not found" 404/);
  assert.match(caddy, /^https:\/\/admin\.staging\.jyotisha\.chat \{$/m);
  assert.equal((caddy.match(/reverse_proxy web:3000/g) ?? []).length, 2);
  assert.match(caddy, /@root path \/\n\s+redir @root \/admin 308/);
  assert.doesNotMatch(caddy, /\*\.staging\.jyotisha\.chat|:443 \{/);
  assert.doesNotMatch(caddy, /www\.jyotisha\.chat/);
});

test("staging deploy consumes only the isolated staging environment and tested revision", () => {
  const qualityGate = readFileSync(
    new URL("../../.github/workflows/backend-quality-gate.yml", import.meta.url),
    "utf8",
  );
  const workflow = readFileSync(
    new URL("../../.github/workflows/deploy-staging.yml", import.meta.url),
    "utf8",
  );
  const syncController = readFileSync(
    new URL("../../deploy/sync-staging-tree.sh", import.meta.url),
    "utf8",
  );

  assert.match(qualityGate, /push:\s*\n\s*branches: \[staging\]/);
  assert.match(workflow, /workflows: \["Staging Backend Quality Gate"\]/);
  assert.match(
    workflow,
    /github\.event\.workflow_run\.head_branch == 'staging'/,
  );
  assert.match(workflow, /actions: read/);
  assert.match(workflow, /packages: read/);
  assert.match(workflow, /environment:\s*\n\s*name: staging/);
  assert.match(workflow, /deploy_sha:/);
  assert.match(workflow, /\^\[0-9a-f\]\{40\}\$/);
  assert.match(
    workflow,
    /actions\/workflows\/backend-quality-gate\.yml\/runs\?head_sha=/,
  );
  assert.match(workflow, /STAGING_SSH_PRIVATE_KEY/);
  assert.match(workflow, /vars\.STAGING_HOST/);
  assert.match(workflow, /vars\.STAGING_KNOWN_HOSTS/);
  assert.match(workflow, /test "\$DEPLOY_HOST" = "118\.26\.111\.127"/);
  assert.match(workflow, /test "\$DEPLOY_USER" = "deploy"/);
  assert.match(workflow, /test "\$DEPLOY_PATH" = "\/opt\/jyotisha-staging"/);
  assert.match(
    workflow,
    /--include='\/deploy\/' --include='\/deploy\/\*\*\*' --exclude='\*'/,
  );
  assert.match(workflow, /run-staging-deploy\.sh/);
  assert.match(workflow, /steps\.images\.outputs\.api_image/);
  assert.match(workflow, /steps\.images\.outputs\.web_image/);
  assert.doesNotMatch(workflow, /PRODUCTION_SSH_PRIVATE_KEY/);
  assert.doesNotMatch(workflow, /103\.117\.123\.53/);
  assert.match(syncController, /--exclude='\/\.env\*'/);
  assert.match(syncController, /--exclude='\/\.docker\/'/);
  assert.match(syncController, /--exclude='\/backups\/'/);
});

test("staging env validator rejects selector drift, duplicates, and unsafe permissions", () => {
  const validator = fileURLToPath(
    new URL("../../deploy/validate-staging-env.sh", import.meta.url),
  );
  const root = mkdtempSync(join(tmpdir(), "jyotisha-staging-env-"));
  const envFile = join(root, ".env.staging");
  const composeFile = join(root, "compose.yml");
  const validSelectors = [
    "APP_ENV_FILE=../.env.staging",
    "CADDYFILE_PATH=./Caddyfile.staging",
    "SITE_ADDRESS=https://staging.jyotisha.chat",
    "AUTH_PROVIDER=self-hosted",
    "SELF_HOSTED_IDENTITY_ENABLED=true",
    "AUTH_USER_ORIGIN=https://staging.jyotisha.chat",
    "ADMIN_USER_ORIGIN=https://admin.staging.jyotisha.chat",
    "IDENTITY_DATABASE_URL=postgresql://identity_runtime:identity-runtime-test-password@postgres:5432/jyotisha",
    "APP_DATABASE_URL=postgresql://app_runtime:app-runtime-test-password@postgres:5432/jyotisha",
    "SERVICE_DATABASE_URL=postgresql://service_runtime:service-runtime-test-password@postgres:5432/jyotisha",
    "ADMIN_DATABASE_URL=postgresql://admin_runtime:admin-runtime-test-password@postgres:5432/jyotisha",
    "BETTER_AUTH_USER_SECRET=user-secret-that-is-at-least-32-bytes-long",
    "RESEND_API_KEY=re_test_key_that_must_not_be_printed",
    "RESEND_FROM_EMAIL=Jyotisha Staging <login@staging.jyotisha.chat>",
    "ADMIN_EMAILS=admin@example.com",
    "EPAY_CONFIG_ENCRYPTION_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
    "MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=",
    "EPAY_CHAT_ENABLED=false",
    "JYOTISH_DYNAMIC_RECTIFICATION_TOKEN=dynamic-token-that-is-at-least-32-bytes",
    "PERSONAL_REPORT_ENABLED=true",
    "PERSONAL_REPORT_DAILY_LIMIT=5",
  ];
  const run = () =>
    spawnSync("bash", [validator, envFile], { encoding: "utf8" });
  const writeEnv = (lines: string[], mode = 0o600) => {
    writeFileSync(envFile, `${lines.join("\n")}\n`);
    chmodSync(envFile, mode);
  };

  try {
    writeFileSync(
      composeFile,
      [
        "services:",
        "  probe:",
        "    image: alpine",
        "    environment:",
        "      SELECTED: ${APP_ENV_FILE}",
        "",
      ].join("\n"),
    );
    writeEnv(validSelectors);
    assert.equal(run().status, 0);

    writeEnv(validSelectors.map((line) => line.startsWith("EPAY_CONFIG_ENCRYPTION_KEY=") ? "EPAY_CONFIG_ENCRYPTION_KEY=invalid" : line));
    assert.notEqual(run().status, 0);
    writeEnv(validSelectors.map((line) => line.startsWith("MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=") ? "MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=invalid" : line));
    assert.notEqual(run().status, 0);
    for (const legacyModelSetting of [
      "OPENAI_API_KEY=legacy",
      "ANTHROPIC_API_KEY=legacy",
      "DEEPSEEK_API_KEY=legacy",
      "LLM_API_KEY=legacy",
      "LLM_MODELS_JSON=[]",
      "LLM_BASE_URL=https://models.example.com",
      "LLM_MODEL=legacy-model",
      "LLM_DEFAULT_MODEL_ID=legacy-model",
      "LLM_PROVIDER_ID=legacy-provider",
      "MASTRA_MODEL=legacy-model",
      "MODEL_PROVIDER_TRUSTED_EDGE_API_KEY=legacy",
    ]) {
      writeEnv([...validSelectors, legacyModelSetting]);
      const legacyResult = run();
      assert.notEqual(legacyResult.status, 0, `${legacyModelSetting} must be rejected`);
      assert.match(`${legacyResult.stdout}${legacyResult.stderr}`, /legacy model environment settings are forbidden/);
    }
    writeEnv(validSelectors.map((line) => line.startsWith("EPAY_CHAT_ENABLED=") ? "EPAY_CHAT_ENABLED=true" : line));
    assert.notEqual(run().status, 0);
    writeEnv(validSelectors.map((line) => line.startsWith("PERSONAL_REPORT_ENABLED=") ? "PERSONAL_REPORT_ENABLED=false" : line));
    assert.equal(run().status, 0);
    writeEnv(validSelectors.map((line) => line.startsWith("PERSONAL_REPORT_ENABLED=") ? "PERSONAL_REPORT_ENABLED=TRUE" : line));
    assert.notEqual(run().status, 0);
    writeEnv(validSelectors.map((line) => line.startsWith("PERSONAL_REPORT_DAILY_LIMIT=") ? "PERSONAL_REPORT_DAILY_LIMIT=0" : line));
    assert.notEqual(run().status, 0);
    writeEnv(validSelectors.map((line) => line.startsWith("PERSONAL_REPORT_DAILY_LIMIT=") ? "PERSONAL_REPORT_DAILY_LIMIT=5x" : line));
    assert.notEqual(run().status, 0);
    writeEnv(validSelectors);

    const shellOverride = spawnSync(
      "docker",
      [
        "compose",
        "--env-file",
        envFile,
        "-f",
        composeFile,
        "config",
        "--format",
        "json",
      ],
      {
        encoding: "utf8",
        env: { ...process.env, APP_ENV_FILE: "../.env.production" },
      },
    );
    assert.equal(shellOverride.status, 0, shellOverride.stderr);
    assert.equal(
      JSON.parse(shellOverride.stdout).services.probe.environment.SELECTED,
      "../.env.production",
    );

    writeEnv(["APP_ENV_FILE=../.env.production", ...validSelectors.slice(1)]);
    assert.notEqual(run().status, 0);

    writeEnv([...validSelectors, "SITE_ADDRESS=https://example.invalid"]);
    assert.notEqual(run().status, 0);

    writeEnv([...validSelectors, "APP_ENV_FILE = ../.env.production"]);
    assert.notEqual(run().status, 0);
    const rendered = spawnSync(
      "docker",
      [
        "compose",
        "--env-file",
        envFile,
        "-f",
        composeFile,
        "config",
        "--format",
        "json",
      ],
      { encoding: "utf8" },
    );
    assert.equal(rendered.status, 0, rendered.stderr);
    assert.equal(
      JSON.parse(rendered.stdout).services.probe.environment.SELECTED,
      "../.env.production",
    );

    writeEnv([...validSelectors, "export CADDYFILE_PATH=./Caddyfile"]);
    assert.notEqual(run().status, 0);

    writeEnv([...validSelectors, "SITE_ADDRESS"]);
    assert.notEqual(run().status, 0);

    writeEnv(
      validSelectors.map((line) =>
        line.startsWith("AUTH_PROVIDER=")
          ? "AUTH_PROVIDER=supabase"
          : line,
      ),
    );
    assert.notEqual(run().status, 0);

    writeEnv([
      ...validSelectors,
      "BETTER_AUTH_USER_SECRET=duplicate-secret-that-must-not-be-printed",
    ]);
    const duplicateSecret = run();
    assert.notEqual(duplicateSecret.status, 0);
    assert.doesNotMatch(
      `${duplicateSecret.stdout}${duplicateSecret.stderr}`,
      /duplicate-secret-that-must-not-be-printed|re_test_key_that_must_not_be_printed/,
    );

    writeEnv(validSelectors, 0o644);
    assert.notEqual(run().status, 0);
  } finally {
    rmSync(root, { recursive: true, force: true });
  }
});

test("production traffic waits for a healthy web container and retries short replacement gaps", () => {
  const compose = readFileSync(new URL("../../deploy/docker-compose.server.yml", import.meta.url), "utf8");
  const caddyfile = readFileSync(new URL("../../deploy/Caddyfile", import.meta.url), "utf8");
  const web = serviceBlock(compose, "web");
  const caddy = serviceBlock(compose, "caddy");
  const healthcheck = webHealthcheckBlock(web);

  assert.match(web, /GITHUB_SHA: \$\{GITHUB_SHA\}/);
  assert.match(web, /healthcheck:\n\s+test: \["CMD", "node", "-e", "fetch\('http:\/\/127\.0\.0\.1:3000\/api\/health'\)\.then\(r=>\{if\(!r\.ok\)process\.exit\(1\)\}\)"\]/);
  assert.match(healthcheck, /^      interval: 30s$/m);
  assert.match(healthcheck, /^      timeout: 5s$/m);
  assert.match(healthcheck, /^      retries: 5$/m);
  assert.match(healthcheck, /^      start_period: 30s$/m);
  assert.match(healthcheck, /^      start_interval: 1s$/m);
  assert.match(caddy, /web:\n\s+condition: service_healthy/);
  assert.match(caddyfile, /reverse_proxy web:3000 \{\n\s+lb_try_duration 10s\n\s+lb_try_interval 250ms\n\s+\}/);
});

test("production workflow consistently uses its tested revision from checkout through verification", () => {
  const workflow = readFileSync(new URL("../../.github/workflows/deploy-production.yml", import.meta.url), "utf8");

  assertWorkflowUsesTestedSha(workflow);
});

test("production workflow rejects a SHA mismatch in verification", () => {
  const workflow = readFileSync(new URL("../../.github/workflows/deploy-production.yml", import.meta.url), "utf8");
  const verification = workflowStepBlock(workflow, "Verify production");
  const mismatchedVerification = verification.replace(
    testedShaExpression,
    "${{ github.event.workflow_run.head_sha || github.sha }}",
  );

  assert.throws(
    () => assertWorkflowUsesTestedSha(workflow.replace(verification, mismatchedVerification)),
    /Verify production must use the tested SHA/,
  );
});

test("production API probes health rapidly while a replacement container starts", () => {
  const compose = readFileSync(new URL("../../deploy/docker-compose.server.yml", import.meta.url), "utf8");

  assert.match(compose, /healthcheck:[\s\S]*start_period:\s*30s[\s\S]*start_interval:\s*1s/);
});