diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bae554e4..1ca5930f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,8 @@ name: Jyotish Skill CI on: + push: + branches: [staging] workflow_dispatch: jobs: diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 00000000..62658856 --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,144 @@ +name: Deploy staging + +on: + workflow_run: + workflows: ["Jyotish Skill CI"] + types: [completed] + workflow_dispatch: + inputs: + git_sha: + description: Exact 40-character commit SHA from a successful CI run + required: true + +permissions: + contents: read + actions: read + +concurrency: + group: staging + cancel-in-progress: false + +jobs: + deploy: + if: >- + github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'staging') + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: + name: staging + url: ${{ vars.STAGING_URL }} + env: + DEPLOY_HOST: ${{ vars.STAGING_HOST }} + DEPLOY_PORT: ${{ vars.STAGING_PORT }} + DEPLOY_USER: ${{ vars.STAGING_USER }} + DEPLOY_PATH: ${{ vars.STAGING_PATH }} + STAGING_URL: ${{ vars.STAGING_URL }} + STAGING_KNOWN_HOSTS: ${{ vars.STAGING_KNOWN_HOSTS }} + + steps: + - name: Validate tested revision + id: revision + env: + REQUESTED_SHA: ${{ github.event.workflow_run.head_sha || inputs.git_sha }} + GH_TOKEN: ${{ github.token }} + run: | + test "${#REQUESTED_SHA}" -eq 40 + case "$REQUESTED_SHA" in + *[!0-9a-fA-F]*) echo "git_sha must be a full hexadecimal commit SHA" >&2; exit 1 ;; + esac + DEPLOY_GIT_SHA="$(printf '%s' "$REQUESTED_SHA" | tr '[:upper:]' '[:lower:]')" + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + TESTED_RUNS="$(curl --fail --silent --show-error \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/workflows/ci.yml/runs?head_sha=$DEPLOY_GIT_SHA&status=success&per_page=1")" + test "$(printf '%s' "$TESTED_RUNS" | jq -r '.total_count')" -ge 1 || { + echo "No successful Jyotish Skill CI run found for $DEPLOY_GIT_SHA" >&2 + exit 1 + } + fi + echo "sha=$DEPLOY_GIT_SHA" >> "$GITHUB_OUTPUT" + + - name: Checkout tested revision + uses: actions/checkout@v4 + with: + ref: ${{ steps.revision.outputs.sha }} + + - name: Verify checked-out revision + env: + DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} + run: test "$(git rev-parse HEAD)" = "$DEPLOY_GIT_SHA" + + - name: Validate staging target configuration + run: | + test "$DEPLOY_HOST" = "118.26.111.127" + test "$DEPLOY_PORT" = "22" + test "$DEPLOY_USER" = "deploy" + test "$DEPLOY_PATH" = "/opt/jyotisha-staging" + test "$STAGING_URL" = "https://staging.jyotisha.chat" + test -n "$STAGING_KNOWN_HOSTS" + + - name: Configure pinned staging SSH + env: + SSH_PRIVATE_KEY: ${{ secrets.STAGING_SSH_PRIVATE_KEY }} + run: | + test -n "$SSH_PRIVATE_KEY" + install -m 700 -d ~/.ssh + printf '%s\n' "$SSH_PRIVATE_KEY" > ~/.ssh/jyotisha-staging + chmod 600 ~/.ssh/jyotisha-staging + printf '%s\n' "$STAGING_KNOWN_HOSTS" > ~/.ssh/known_hosts + chmod 600 ~/.ssh/known_hosts + + - name: Record previous staging state + env: + DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} + run: | + SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" + PREVIOUS_SHA="$(curl --fail --silent --show-error --max-time 10 "$STAGING_URL/api/health" 2>/dev/null | jq -r '.deployment.gitCommit // empty' || true)" + test -n "$PREVIOUS_SHA" || PREVIOUS_SHA="not-deployed" + PREVIOUS_IMAGES="$(ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ + "if [ -f '$DEPLOY_PATH/.env.staging' ] && [ -f '$DEPLOY_PATH/deploy/docker-compose.server.yml' ]; then cd '$DEPLOY_PATH' && APP_ENV_FILE='../.env.staging' CADDYFILE_PATH='./Caddyfile.staging' SITE_ADDRESS='https://staging.jyotisha.chat' docker compose --env-file .env.staging -f deploy/docker-compose.server.yml images --quiet; else echo not-deployed; fi")" + test -n "$PREVIOUS_IMAGES" || PREVIOUS_IMAGES="not-deployed" + { + echo "### Staging deployment state" + echo "- Previous verified SHA: \`$PREVIOUS_SHA\`" + echo "- Target SHA: \`$DEPLOY_GIT_SHA\`" + echo "- Previous image IDs:" + echo '```text' + printf '%s\n' "$PREVIOUS_IMAGES" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Sync and rebuild staging + env: + DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} + run: | + SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" + RSYNC_SSH="ssh $SSH_OPTIONS" + ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "install -d -m 755 '$DEPLOY_PATH'" + rsync -az --delete \ + --exclude='.git/' \ + --exclude='.env*' \ + --exclude='frontend/node_modules/' \ + --exclude='frontend/.next/' \ + -e "$RSYNC_SSH" \ + ./ "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/" + ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ + "cd '$DEPLOY_PATH' && bash deploy/validate-staging-env.sh .env.staging && APP_ENV_FILE='../.env.staging' CADDYFILE_PATH='./Caddyfile.staging' SITE_ADDRESS='https://staging.jyotisha.chat' docker compose --env-file .env.staging -f deploy/docker-compose.server.yml config --quiet && APP_ENV_FILE='../.env.staging' CADDYFILE_PATH='./Caddyfile.staging' SITE_ADDRESS='https://staging.jyotisha.chat' GITHUB_SHA='$DEPLOY_GIT_SHA' docker compose --env-file .env.staging -f deploy/docker-compose.server.yml up -d --build --remove-orphans" + + - name: Verify staging + env: + DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} + run: | + curl --fail --silent --show-error --retry 12 --retry-delay 5 "$STAGING_URL/login" >/dev/null + test "$(curl --silent --output /dev/null --write-out '%{http_code}' "$STAGING_URL/api/account")" = "401" + test "$(curl --fail --silent --show-error "$STAGING_URL/api/health" | jq -r '.deployment.gitCommit')" = "$DEPLOY_GIT_SHA" + ssh -i ~/.ssh/jyotisha-staging -p "$DEPLOY_PORT" \ + -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ + "$DEPLOY_USER@$DEPLOY_HOST" \ + "cd '$DEPLOY_PATH' && APP_ENV_FILE='../.env.staging' CADDYFILE_PATH='./Caddyfile.staging' SITE_ADDRESS='https://staging.jyotisha.chat' docker compose --env-file .env.staging -f deploy/docker-compose.server.yml exec -T web node -e 'fetch(\"http://api:5200/api/health\").then(async r => { const body = await r.json(); if (!r.ok || body.status !== \"ok\" || body.swisseph_available !== true) process.exit(1); console.log(JSON.stringify(body)); })'" + echo "- Verified deployed SHA: \`$DEPLOY_GIT_SHA\`" >> "$GITHUB_STEP_SUMMARY" diff --git a/deploy/Caddyfile.staging b/deploy/Caddyfile.staging new file mode 100644 index 00000000..66bbb59c --- /dev/null +++ b/deploy/Caddyfile.staging @@ -0,0 +1,4 @@ +{$SITE_ADDRESS:https://staging.jyotisha.chat} { + encode zstd gzip + reverse_proxy web:3000 +} diff --git a/deploy/README.md b/deploy/README.md index 7c37ff94..3d729fc5 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -130,6 +130,54 @@ PRODUCTION_SSH_PRIVATE_KEY = dedicated production deploy private key The workflow pins the VPS Ed25519 host key and serializes deployments with the `production` concurrency group. +## Staging deployment + +Staging is isolated from production: + +| Item | Value | +| --- | --- | +| URL | `https://staging.jyotisha.chat` | +| Host | `118.26.111.127` | +| Path | `/opt/jyotisha-staging` | +| Runtime env | `/opt/jyotisha-staging/.env.staging` (`0600`) | +| Supabase | separate `Jyotisha Staging` project | +| GitHub Environment | `staging` | + +The GitHub Environment contains `STAGING_SSH_PRIVATE_KEY` and the variables `STAGING_HOST`, `STAGING_PORT`, `STAGING_USER`, `STAGING_PATH`, `STAGING_URL`, and `STAGING_KNOWN_HOSTS`. Its deployment branch policy allows the `main` controller branch: GitHub's `workflow_run` event executes from the default branch while the workflow separately requires the successfully tested upstream branch to be `staging`. The staging key, database, Supabase keys, and model-provider keys must not be shared with production. + +A push to branch `staging` runs `Jyotish Skill CI`. A successful push run triggers `.github/workflows/deploy-staging.yml`, which deploys the tested SHA and verifies the login route, logged-out account response, deployment SHA, and private Python health endpoint. + +The staging env file must include these non-secret selectors so Compose cannot fall back to production paths: + +```dotenv +APP_ENV_FILE=../.env.staging +CADDYFILE_PATH=./Caddyfile.staging +SITE_ADDRESS=https://staging.jyotisha.chat +``` + +After source sync and before `up`, the workflow validates `.env.staging` mode/selectors, explicitly pins the three staging selectors against ambient shell overrides, and runs `docker compose --env-file .env.staging -f deploy/docker-compose.server.yml config --quiet`. For later manual inspections, run the same checks only after the tracked deployment files exist on the server. The first deployment should be manual: + +1. Confirm `/opt/jyotisha-staging/.env.staging` exists, has mode `0600`, and contains the three selectors above. +2. Open GitHub Actions -> Jyotish Skill CI -> Run workflow, using workflow from `main`. +3. Wait for success and copy that run's exact 40-character commit SHA. +4. Open GitHub Actions -> Deploy staging -> Run workflow, using workflow from `main`, and enter the SHA in `git_sha`. +5. Confirm `https://staging.jyotisha.chat/api/health` reports that SHA. +6. Only after the manual deployment passes, push a reviewed revision to branch `staging` to validate automatic deployment. + +Application rollback uses the same workflow: manually dispatch `Deploy staging` from `main` with a previous known-good full SHA that has a successful CI run. Database migrations are separate and are not rolled back by an application deployment. Restore a staging database backup before running any destructive migration rehearsal. + +Inspect staging without printing secrets: + +```bash +ssh -i ~/.ssh/jyotisha-staging deploy@118.26.111.127 +cd /opt/jyotisha-staging +docker compose --env-file .env.staging -f deploy/docker-compose.server.yml ps +docker compose --env-file .env.staging -f deploy/docker-compose.server.yml logs --tail=100 api web caddy +curl -fsS https://staging.jyotisha.chat/api/health +``` + +The normal application deployment workflow never runs database migrations. Apply migrations to the separate staging project first, verify them, and only then deploy application code that depends on them. + ## Manual deployment fallback If GitHub Actions is unavailable, deploy the tracked tree without copying local secrets: diff --git a/deploy/docker-compose.server.yml b/deploy/docker-compose.server.yml index caa615f6..42998526 100644 --- a/deploy/docker-compose.server.yml +++ b/deploy/docker-compose.server.yml @@ -4,7 +4,8 @@ services: context: .. dockerfile: deploy/railway-api.Dockerfile restart: unless-stopped - env_file: ../.env.production + env_file: + - ${APP_ENV_FILE:-../.env.production} environment: PORT: 5200 JYOTISH_ALLOWED_HOSTS: localhost,127.0.0.1,::1,api @@ -24,7 +25,8 @@ services: NEXT_PUBLIC_SUPABASE_URL: ${NEXT_PUBLIC_SUPABASE_URL} NEXT_PUBLIC_SUPABASE_ANON_KEY: ${NEXT_PUBLIC_SUPABASE_ANON_KEY} restart: unless-stopped - env_file: ../.env.production + env_file: + - ${APP_ENV_FILE:-../.env.production} environment: GITHUB_SHA: ${GITHUB_SHA} PORT: 3000 @@ -45,7 +47,7 @@ services: - "443:443" - "443:443/udp" volumes: - - ./Caddyfile:/etc/caddy/Caddyfile:ro + - ${CADDYFILE_PATH:-./Caddyfile}:/etc/caddy/Caddyfile:ro - caddy_data:/data - caddy_config:/config depends_on: diff --git a/deploy/validate-staging-env.sh b/deploy/validate-staging-env.sh new file mode 100755 index 00000000..0c82f578 --- /dev/null +++ b/deploy/validate-staging-env.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail + +ENV_FILE="${1:-.env.staging}" + +if [ ! -f "$ENV_FILE" ]; then + echo "staging environment file is missing: $ENV_FILE" >&2 + exit 1 +fi + +if MODE="$(stat -c '%a' "$ENV_FILE" 2>/dev/null)"; then + : +else + MODE="$(stat -f '%Lp' "$ENV_FILE")" +fi + +if [ "$MODE" != "600" ]; then + echo "staging environment file must have mode 0600" >&2 + exit 1 +fi + +require_selector() { + local key="$1" + local expected="$2" + local count + local definition_pattern + + definition_pattern="^[[:space:]]*(export[[:space:]]+)?${key}([[:space:]]*=|[[:space:]]*$)" + count="$(grep -Ec "$definition_pattern" "$ENV_FILE" || true)" + if [ "$count" -ne 1 ] || ! grep -Fqx "${key}=${expected}" "$ENV_FILE"; then + echo "invalid staging selector: $key" >&2 + exit 1 + fi +} + +require_selector APP_ENV_FILE ../.env.staging +require_selector CADDYFILE_PATH ./Caddyfile.staging +require_selector SITE_ADDRESS https://staging.jyotisha.chat + +echo "staging environment selectors: valid" diff --git a/docs/superpowers/plans/2026-07-20-staging-deployment-automation.md b/docs/superpowers/plans/2026-07-20-staging-deployment-automation.md new file mode 100644 index 00000000..6ee361b8 --- /dev/null +++ b/docs/superpowers/plans/2026-07-20-staging-deployment-automation.md @@ -0,0 +1,610 @@ +# Jyotisha Staging Deployment Automation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the existing Compose deployment environment-selectable and automatically deploy tested `staging` revisions to `https://staging.jyotisha.chat` without changing production defaults. + +**Architecture:** Production and staging share one Compose topology, with explicit variables selecting the runtime env file and Caddyfile. A dedicated GitHub workflow consumes only `staging` Environment credentials, syncs the tested revision, builds it on the staging VPS, and verifies public and private health contracts. Manual dispatch accepts a known Git SHA for rollback. + +**Tech Stack:** GitHub Actions, Docker Compose, Caddy, rsync, SSH, Next.js node:test contract tests. + +## Global Constraints + +- Complete `2026-07-20-staging-infrastructure-bootstrap.md` before running the deployment workflow. +- Preserve production defaults: `.env.production`, `deploy/Caddyfile`, `https://jyotisha.chat`, and production workflow behavior. +- Staging runtime file remains `/opt/jyotisha-staging/.env.staging` and is never committed or synced. +- Staging workflow must use GitHub Environment `staging`, not repository production secrets. +- Staging host, port, user, path, URL, and known-hosts entry come from Environment variables. +- Staging deploys only after a successful `Jyotish Skill CI` push run on branch `staging`, or an explicit manual dispatch. +- Database migrations remain a separate operation and are never automatically run by the application deployment workflow. +- Do not expose host ports 3000 or 5200. +- Use TDD for repository changes and commit only task-owned files at each task boundary. + +--- + +## File Structure + +- Modify `deploy/docker-compose.server.yml`: environment-specific env file and Caddyfile selection while retaining production defaults. +- Create `deploy/Caddyfile.staging`: staging-only public reverse proxy with no production `www` redirect. +- Create `deploy/validate-staging-env.sh`: fail closed unless the staging env is mode `0600` and contains exactly the three fixed staging selectors. +- Modify `frontend/tests/health-deployment.test.ts`: Compose, Caddy, CI-trigger, and staging-workflow contracts. +- Modify `.github/workflows/ci.yml`: run the existing CI on pushes to `staging`; do not add a `main` push trigger in this task. +- Create `.github/workflows/deploy-staging.yml`: tested-revision staging deployment and smoke checks. +- Modify `deploy/README.md`: operator-facing staging setup, first deploy, verification, and rollback. + +### Task 1: Parameterize Compose Without Changing Production Defaults + +**Files:** +- Modify: `frontend/tests/health-deployment.test.ts` +- Modify: `deploy/docker-compose.server.yml` +- Create: `deploy/Caddyfile.staging` + +**Interfaces:** +- Consumes: `APP_ENV_FILE` and `CADDYFILE_PATH` from Compose interpolation. +- Produces: production defaults `../.env.production` and `./Caddyfile`; staging selections `../.env.staging` and `./Caddyfile.staging`. + +- [ ] **Step 1: Add failing deployment configuration tests** + +Append these tests to `frontend/tests/health-deployment.test.ts`: + +```ts +test("server compose accepts staging paths while preserving production defaults", () => { + const compose = readFileSync(new URL("../../deploy/docker-compose.server.yml", import.meta.url), "utf8"); + + assert.match(compose, /env_file:\s*\n\s*- \$\{APP_ENV_FILE:-\.\.\/\.env\.production\}/); + assert.match(compose, /\$\{CADDYFILE_PATH:-\.\/Caddyfile\}:\/etc\/caddy\/Caddyfile:ro/); + assert.match(compose, /SITE_ADDRESS: \$\{SITE_ADDRESS:-https:\/\/jyotisha\.chat\}/); +}); + +test("staging Caddy configuration serves only the configured staging address", () => { + const caddy = readFileSync(new URL("../../deploy/Caddyfile.staging", import.meta.url), "utf8"); + + assert.match(caddy, /\{\$SITE_ADDRESS:https:\/\/staging\.jyotisha\.chat\}/); + assert.match(caddy, /reverse_proxy web:3000/); + assert.doesNotMatch(caddy, /www\.jyotisha\.chat/); +}); +``` + +- [ ] **Step 2: Run the focused test and verify it fails** + +Run: + +```bash +cd /Users/jesse/Downloads/Copse/astrology/yinduzhanxing/frontend +npx tsx --test tests/health-deployment.test.ts +``` + +Expected: FAIL because `deploy/Caddyfile.staging` does not exist and Compose does not contain the environment-specific paths. + +- [ ] **Step 3: Parameterize both service env files and the Caddy volume** + +In `deploy/docker-compose.server.yml`, replace each current scalar env file: + +```yaml + env_file: ../.env.production +``` + +with this list form for both `api` and `web`: + +```yaml + env_file: + - ${APP_ENV_FILE:-../.env.production} +``` + +Replace the Caddyfile volume: + +```yaml + - ./Caddyfile:/etc/caddy/Caddyfile:ro +``` + +with: + +```yaml + - ${CADDYFILE_PATH:-./Caddyfile}:/etc/caddy/Caddyfile:ro +``` + +Do not change ports, health checks, `SITE_ADDRESS` default, volumes, or service dependencies. + +- [ ] **Step 4: Create the staging-only Caddyfile** + +Create `deploy/Caddyfile.staging` with exactly: + +```caddyfile +{$SITE_ADDRESS:https://staging.jyotisha.chat} { + encode zstd gzip + reverse_proxy web:3000 +} +``` + +This avoids the production Caddyfile's `www.jyotisha.chat` redirect and certificate request on the staging host. + +- [ ] **Step 5: Re-run the focused tests** + +Run: + +```bash +cd /Users/jesse/Downloads/Copse/astrology/yinduzhanxing/frontend +npx tsx --test tests/health-deployment.test.ts +``` + +Expected: all tests in `health-deployment.test.ts` PASS. + +- [ ] **Step 6: Validate production and staging Compose interpolation** + +From the repository root, create one non-secret runtime env file and one Compose interpolation file: + +```bash +cd /Users/jesse/Downloads/Copse/astrology/yinduzhanxing +RUNTIME_ENV_TMP=$(mktemp) +COMPOSE_ENV_TMP=$(mktemp) +printf '%s\n' \ + 'NEXT_PUBLIC_SUPABASE_URL=https://ci-placeholder.supabase.co' \ + 'NEXT_PUBLIC_SUPABASE_ANON_KEY=ci-placeholder' \ + > "$RUNTIME_ENV_TMP" +printf '%s\n' \ + "APP_ENV_FILE=$RUNTIME_ENV_TMP" \ + 'CADDYFILE_PATH=./Caddyfile.staging' \ + 'SITE_ADDRESS=https://staging.jyotisha.chat' \ + 'NEXT_PUBLIC_SUPABASE_URL=https://ci-placeholder.supabase.co' \ + 'NEXT_PUBLIC_SUPABASE_ANON_KEY=ci-placeholder' \ + > "$COMPOSE_ENV_TMP" +docker compose --env-file "$COMPOSE_ENV_TMP" -f deploy/docker-compose.server.yml config >/dev/null +rm "$RUNTIME_ENV_TMP" "$COMPOSE_ENV_TMP" +``` + +Expected: both `docker compose config` commands exit `0` without revealing a real secret. + +- [ ] **Step 7: Commit Task 1** + +```bash +git add frontend/tests/health-deployment.test.ts deploy/docker-compose.server.yml deploy/Caddyfile.staging +git commit -m "feat: parameterize staging compose configuration" +``` + +### Task 2: Add a Tested-Revision Staging Deployment Workflow + +**Files:** +- Modify: `frontend/tests/health-deployment.test.ts` +- Modify: `.github/workflows/ci.yml` +- Create: `.github/workflows/deploy-staging.yml` + +**Interfaces:** +- Consumes: successful `Jyotish Skill CI` runs for push events on branch `staging`; GitHub Environment variables and `STAGING_SSH_PRIVATE_KEY`. +- Produces: deployment of the exact tested SHA and verified staging health at `vars.STAGING_URL`. + +- [ ] **Step 1: Add a failing staging workflow contract test** + +Append this test to `frontend/tests/health-deployment.test.ts`: + +```ts +test("staging deploy consumes only the isolated staging environment and tested revision", () => { + const ci = readFileSync(new URL("../../.github/workflows/ci.yml", import.meta.url), "utf8"); + const workflow = readFileSync(new URL("../../.github/workflows/deploy-staging.yml", import.meta.url), "utf8"); + + assert.match(ci, /push:\s*\n\s*branches: \[staging\]/); + assert.match(workflow, /workflows: \["Jyotish Skill CI"\]/); + assert.match(workflow, /github\.event\.workflow_run\.head_branch == 'staging'/); + assert.match(workflow, /actions: read/); + assert.match(workflow, /environment:\s*\n\s*name: staging/); + assert.match(workflow, /git_sha:/); + assert.doesNotMatch(workflow, /default: staging/); + assert.match(workflow, /test "\$\{#REQUESTED_SHA\}" -eq 40/); + assert.match(workflow, /actions\/workflows\/ci\.yml\/runs\?head_sha=/); + assert.match(workflow, /STAGING_SSH_PRIVATE_KEY/); + assert.match(workflow, /vars\.STAGING_HOST/); + assert.match(workflow, /vars\.STAGING_KNOWN_HOSTS/); + assert.match(workflow, /--exclude='\.env\*'/); + assert.match(workflow, /docker compose --env-file \.env\.staging/); + assert.match(workflow, /bash deploy\/validate-staging-env\.sh \.env\.staging/); + assert.match(workflow, /docker compose --env-file \.env\.staging -f deploy\/docker-compose\.server\.yml config --quiet/); + assert.match(workflow, /deployment\.gitCommit/); + assert.doesNotMatch(workflow, /PRODUCTION_SSH_PRIVATE_KEY/); + assert.doesNotMatch(workflow, /103\.117\.123\.53/); +}); +``` + +- [ ] **Step 2: Run the test and verify it fails** + +```bash +cd /Users/jesse/Downloads/Copse/astrology/yinduzhanxing/frontend +npx tsx --test tests/health-deployment.test.ts +``` + +Expected: FAIL because `.github/workflows/deploy-staging.yml` does not exist and CI has no staging push trigger. + +- [ ] **Step 3: Add the staging push trigger to the existing CI** + +Change only the `on` block in `.github/workflows/ci.yml` to: + +```yaml +on: + push: + branches: [staging] + workflow_dispatch: +``` + +Do not add `main` in this task. This prevents an unintended change to the current production deployment trigger while enabling a tested staging revision. + +- [ ] **Step 4: Create the staging deployment workflow** + +Create `.github/workflows/deploy-staging.yml` with exactly: + +```yaml +name: Deploy staging + +on: + workflow_run: + workflows: ["Jyotish Skill CI"] + types: [completed] + workflow_dispatch: + inputs: + git_sha: + description: Exact 40-character commit SHA from a successful CI run + required: true + +permissions: + contents: read + actions: read + +concurrency: + group: staging + cancel-in-progress: false + +jobs: + deploy: + if: >- + github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'staging') + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: + name: staging + url: ${{ vars.STAGING_URL }} + env: + DEPLOY_HOST: ${{ vars.STAGING_HOST }} + DEPLOY_PORT: ${{ vars.STAGING_PORT }} + DEPLOY_USER: ${{ vars.STAGING_USER }} + DEPLOY_PATH: ${{ vars.STAGING_PATH }} + STAGING_URL: ${{ vars.STAGING_URL }} + STAGING_KNOWN_HOSTS: ${{ vars.STAGING_KNOWN_HOSTS }} + + steps: + - name: Validate tested revision + id: revision + env: + REQUESTED_SHA: ${{ github.event.workflow_run.head_sha || inputs.git_sha }} + GH_TOKEN: ${{ github.token }} + run: | + test "${#REQUESTED_SHA}" -eq 40 + case "$REQUESTED_SHA" in + *[!0-9a-fA-F]*) echo "git_sha must be a full hexadecimal commit SHA" >&2; exit 1 ;; + esac + DEPLOY_GIT_SHA="$(printf '%s' "$REQUESTED_SHA" | tr '[:upper:]' '[:lower:]')" + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + TESTED_RUNS="$(curl --fail --silent --show-error \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/workflows/ci.yml/runs?head_sha=$DEPLOY_GIT_SHA&status=success&per_page=1")" + test "$(printf '%s' "$TESTED_RUNS" | jq -r '.total_count')" -ge 1 || { + echo "No successful Jyotish Skill CI run found for $DEPLOY_GIT_SHA" >&2 + exit 1 + } + fi + echo "sha=$DEPLOY_GIT_SHA" >> "$GITHUB_OUTPUT" + + - name: Checkout tested revision + uses: actions/checkout@v4 + with: + ref: ${{ steps.revision.outputs.sha }} + + - name: Verify checked-out revision + env: + DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} + run: test "$(git rev-parse HEAD)" = "$DEPLOY_GIT_SHA" + + - name: Validate staging target configuration + run: | + test "$DEPLOY_HOST" = "118.26.111.127" + test "$DEPLOY_PORT" = "22" + test "$DEPLOY_USER" = "deploy" + test "$DEPLOY_PATH" = "/opt/jyotisha-staging" + test "$STAGING_URL" = "https://staging.jyotisha.chat" + test -n "$STAGING_KNOWN_HOSTS" + + - name: Configure pinned staging SSH + env: + SSH_PRIVATE_KEY: ${{ secrets.STAGING_SSH_PRIVATE_KEY }} + run: | + test -n "$SSH_PRIVATE_KEY" + install -m 700 -d ~/.ssh + printf '%s\n' "$SSH_PRIVATE_KEY" > ~/.ssh/jyotisha-staging + chmod 600 ~/.ssh/jyotisha-staging + printf '%s\n' "$STAGING_KNOWN_HOSTS" > ~/.ssh/known_hosts + chmod 600 ~/.ssh/known_hosts + + - name: Record previous staging state + env: + DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} + run: | + # Query the current public deployment SHA and current Compose image IDs. + # Append both, plus DEPLOY_GIT_SHA, to GITHUB_STEP_SUMMARY before rebuilding. + + - name: Sync and rebuild staging + env: + DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} + run: | + SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" + RSYNC_SSH="ssh $SSH_OPTIONS" + ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "install -d -m 755 '$DEPLOY_PATH'" + rsync -az --delete \ + --exclude='.git/' \ + --exclude='.env*' \ + --exclude='frontend/node_modules/' \ + --exclude='frontend/.next/' \ + -e "$RSYNC_SSH" \ + ./ "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/" + ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ + "cd '$DEPLOY_PATH' && bash deploy/validate-staging-env.sh .env.staging && APP_ENV_FILE='../.env.staging' CADDYFILE_PATH='./Caddyfile.staging' SITE_ADDRESS='https://staging.jyotisha.chat' docker compose --env-file .env.staging -f deploy/docker-compose.server.yml config --quiet && APP_ENV_FILE='../.env.staging' CADDYFILE_PATH='./Caddyfile.staging' SITE_ADDRESS='https://staging.jyotisha.chat' GITHUB_SHA='$DEPLOY_GIT_SHA' docker compose --env-file .env.staging -f deploy/docker-compose.server.yml up -d --build --remove-orphans" + + - name: Verify staging + env: + DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} + run: | + curl --fail --silent --show-error --retry 12 --retry-delay 5 "$STAGING_URL/login" >/dev/null + test "$(curl --silent --output /dev/null --write-out '%{http_code}' "$STAGING_URL/api/account")" = "401" + test "$(curl --fail --silent --show-error "$STAGING_URL/api/health" | jq -r '.deployment.gitCommit')" = "$DEPLOY_GIT_SHA" + ssh -i ~/.ssh/jyotisha-staging -p "$DEPLOY_PORT" \ + -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ + "$DEPLOY_USER@$DEPLOY_HOST" \ + "cd '$DEPLOY_PATH' && docker compose --env-file .env.staging -f deploy/docker-compose.server.yml exec -T web node -e 'fetch(\"http://api:5200/api/health\").then(async r => { const body = await r.json(); if (!r.ok || body.status !== \"ok\" || body.swisseph_available !== true) process.exit(1); console.log(JSON.stringify(body)); })'" +``` + +- [ ] **Step 5: Re-run the focused test** + +```bash +cd /Users/jesse/Downloads/Copse/astrology/yinduzhanxing/frontend +npx tsx --test tests/health-deployment.test.ts +``` + +Expected: all focused tests PASS. + +- [ ] **Step 6: Review the workflow for secret and target isolation** + +Run from repository root: + +```bash +rg -n 'PRODUCTION_SSH_PRIVATE_KEY|103\.117\.123\.53' .github/workflows/deploy-staging.yml +rg -n 'STAGING_SSH_PRIVATE_KEY|vars\.STAGING_|\.env\.staging|head_branch == '\''staging'\''' .github/workflows/deploy-staging.yml +``` + +Expected: the first command returns no matches. The second command shows the staging secret, variables, env file, and branch guard. The workflow may safely mention `.env.production` only in an rsync exclusion so that a stray local file can never be copied. + +- [ ] **Step 7: Commit Task 2** + +```bash +git add frontend/tests/health-deployment.test.ts .github/workflows/ci.yml .github/workflows/deploy-staging.yml +git commit -m "ci: deploy tested revisions to staging" +``` + +### Task 3: Document First Deployment, Verification, and Rollback + +**Files:** +- Modify: `deploy/README.md` + +**Interfaces:** +- Consumes: infrastructure and workflow from earlier tasks. +- Produces: an operator runbook that does not require reading workflow internals. + +- [ ] **Step 1: Add the staging section to the deployment README** + +Append this section immediately before the existing `## Manual deployment fallback` heading in `deploy/README.md`: + +````markdown +## Staging deployment + +Staging is isolated from production: + +| Item | Value | +| --- | --- | +| URL | `https://staging.jyotisha.chat` | +| Host | `118.26.111.127` | +| Path | `/opt/jyotisha-staging` | +| Runtime env | `/opt/jyotisha-staging/.env.staging` (`0600`) | +| Supabase | separate `jyotisha-staging` project | +| GitHub Environment | `staging` | + +The GitHub Environment contains `STAGING_SSH_PRIVATE_KEY` and the variables `STAGING_HOST`, `STAGING_PORT`, `STAGING_USER`, `STAGING_PATH`, `STAGING_URL`, and `STAGING_KNOWN_HOSTS`. Its deployment policy allows the `main` controller branch; the workflow separately requires an upstream successful CI push from branch `staging`. The staging key, database, Supabase keys, and model-provider keys must not be shared with production. + +A push to branch `staging` runs `Jyotish Skill CI`. A successful push run triggers `.github/workflows/deploy-staging.yml`, which records the previous SHA/images, validates the env selectors and Compose configuration, deploys the tested SHA, and verifies the login route, logged-out account response, deployment SHA, and private Python health endpoint. + +The first deployment should be manual, after `.env.staging` is verified to contain `APP_ENV_FILE=../.env.staging`, `CADDYFILE_PATH=./Caddyfile.staging`, and `SITE_ADDRESS=https://staging.jyotisha.chat`: + +1. Confirm `/opt/jyotisha-staging/.env.staging` exists and has mode `0600`. +2. Run `Jyotish Skill CI` manually using workflow from `main` and wait for success. +3. Open GitHub Actions -> Deploy staging -> Run workflow, using workflow from `main`. +4. Enter that successful CI run's exact 40-character commit SHA in `git_sha`. +5. Confirm `https://staging.jyotisha.chat/api/health` reports that SHA. +6. Only after the manual deployment passes, push a reviewed revision to branch `staging` to validate automatic deployment. + +Application rollback uses the same workflow: manually dispatch `Deploy staging` with the previous known-good commit SHA. Database migrations are separate and are not rolled back by an application deployment. Restore a staging database backup before running any destructive migration rehearsal. + +Inspect staging without printing secrets: + +```bash +ssh -i ~/.ssh/jyotisha-staging deploy@118.26.111.127 +cd /opt/jyotisha-staging +docker compose --env-file .env.staging -f deploy/docker-compose.server.yml ps +docker compose --env-file .env.staging -f deploy/docker-compose.server.yml logs --tail=100 api web caddy +curl -fsS https://staging.jyotisha.chat/api/health +``` + +The normal application deployment workflow never runs database migrations. Apply migrations to the separate staging project first, verify them, and only then deploy application code that depends on them. +```` + +- [ ] **Step 2: Check the README for production/staging ambiguity** + +Run: + +```bash +rg -n 'Staging deployment|118\.26\.111\.127|\.env\.staging|STAGING_SSH_PRIVATE_KEY|Database migrations' deploy/README.md +``` + +Expected: all five staging concepts appear in the new section, and the existing production section remains unchanged. + +- [ ] **Step 3: Commit Task 3** + +```bash +git add deploy/README.md +git commit -m "docs: add staging deployment runbook" +``` + +### Task 4: Repository Verification and Controlled First Deployment + +**Files:** +- Verify only; no new repository files expected. + +**Interfaces:** +- Consumes: all previous tasks and completed infrastructure plan. +- Produces: a tested repository revision, then a verified first staging deployment. + +- [ ] **Step 1: Run whitespace and focused contract checks** + +From repository root: + +```bash +git diff --check HEAD~3..HEAD +cd frontend +npx tsx --test tests/health-deployment.test.ts +``` + +Expected: no whitespace errors and all deployment contract tests PASS. + +- [ ] **Step 2: Run the complete frontend checks** + +```bash +cd /Users/jesse/Downloads/Copse/astrology/yinduzhanxing/frontend +npm test +npm run lint +NEXT_PUBLIC_SUPABASE_URL=https://ci-placeholder.supabase.co \ +NEXT_PUBLIC_SUPABASE_ANON_KEY=ci-placeholder \ +npm run build +``` + +Expected: tests, lint, and production build all exit `0`. + +- [ ] **Step 3: Re-run project pre-work checks with the repository virtualenv** + +```bash +cd /Users/jesse/Downloads/Copse/astrology/yinduzhanxing +.venv/bin/python scripts/pre_work_check.py --remote-timeout 8 --command-timeout 45 +``` + +Expected: record the exact result. Do not claim a green project gate if the existing fragment-governance failure remains; staging-specific focused tests must still be green. + +- [ ] **Step 4: Push the implementation revision for review without altering production** + +Use an isolated implementation branch and open a pull request. Do not push the dirty local `main` directly. After review and merge, record the merge SHA: + +```bash +git rev-parse HEAD +``` + +Expected: one 40-character commit SHA used for the first manual staging deployment. + +- [ ] **Step 5: Manually deploy the tested SHA** + +In GitHub: + +```text +Actions -> Deploy staging -> Run workflow +Use workflow from -> main +git_sha -> exact 40-character SHA from a successful Jyotish Skill CI run +``` + +Expected: `Configure pinned staging SSH`, `Sync and rebuild staging`, and `Verify staging` all pass. GitHub Environment shows the deployment URL. + +- [ ] **Step 6: Verify the live deployment independently** + +Run locally: + +```bash +curl -fsS https://staging.jyotisha.chat/login >/dev/null +test "$(curl -sS -o /dev/null -w '%{http_code}' https://staging.jyotisha.chat/api/account)" = "401" +curl -fsS https://staging.jyotisha.chat/api/health | jq '{status, deployment, checks}' +ssh -i "$HOME/.ssh/jyotisha-staging" -o IdentitiesOnly=yes deploy@118.26.111.127 \ + 'cd /opt/jyotisha-staging && docker compose --env-file .env.staging -f deploy/docker-compose.server.yml ps' +``` + +Expected: login succeeds, account returns 401, health is `ok` with the deployed SHA, and `api`, `web`, and `caddy` are running/healthy. + +- [ ] **Step 7: Validate Auth, cookies, and database isolation manually** + +Use a new disposable email address in a private browser window: + +```text +1. Open https://staging.jyotisha.chat/login. +2. Complete OTP login using the staging Supabase email. +3. Complete onboarding and save one profile. +4. Create one chat session and send one low-cost test prompt. +5. Open browser developer tools -> Application -> Cookies. +6. Confirm staging session cookies are scoped to staging.jyotisha.chat and are not Domain=.jyotisha.chat cookies. +7. Open https://jyotisha.chat in a separate normal window and confirm its login state did not change. +``` + +In the staging Supabase Dashboard, confirm the disposable UUID appears in staging Auth and its profile/chat/credit rows exist only in the staging project. Search the production Auth dashboard for the disposable email and confirm it is absent. Do not copy production rows into staging. + +Expected: staging OTP, profile, chat, and credit behavior work; no staging identity or row appears in production; production cookies and session remain unchanged. + +- [ ] **Step 8: Rehearse a failed health check and recovery** + +On the staging VPS, back up the staging env file and deliberately remove only the staging service-role value: + +```bash +ssh -i "$HOME/.ssh/jyotisha-staging" -o IdentitiesOnly=yes deploy@118.26.111.127 +cd /opt/jyotisha-staging +cp -p .env.staging /home/deploy/.env.staging.health-rehearsal +sed -i 's/^SUPABASE_SERVICE_ROLE_KEY=.*/SUPABASE_SERVICE_ROLE_KEY=/' .env.staging +``` + +Manually dispatch `Deploy staging` from `main` using the current full SHA that already has a successful `Jyotish Skill CI` run. + +Expected: deployment reaches `Verify staging`, `/api/health` is not `ok`, and GitHub marks the workflow failed rather than successful. + +Restore the untouched staging secret file and redeploy the same tested SHA: + +```bash +mv /home/deploy/.env.staging.health-rehearsal .env.staging +chmod 600 .env.staging +exit +``` + +Expected: the second workflow passes and `/api/health` returns `ok`. If the backup file is missing, stop and recover the staging service-role value from the password manager; never use the production key. + +- [ ] **Step 9: Validate automatic staging deployment** + +After the manual deployment succeeds, update branch `staging` to the same reviewed SHA without force-pushing: + +```bash +cd /Users/jesse/Downloads/Copse/astrology/yinduzhanxing +read -r -p 'Reviewed commit SHA: ' REVIEWED_SHA +git fetch origin +git push origin "$REVIEWED_SHA:refs/heads/staging" +unset REVIEWED_SHA +``` + +Expected sequence in GitHub Actions: + +```text +Jyotish Skill CI: success, event push, branch staging +Deploy staging: automatically started +Deploy staging: success, same tested SHA +``` + +- [ ] **Step 10: Rehearse application rollback with a known-good SHA** + +After a second harmless staging revision has deployed successfully, manually dispatch `Deploy staging` with the first known-good SHA. + +Expected: workflow passes and `/api/health` reports the first SHA. Then redeploy the latest tested staging SHA. Do not combine this rehearsal with a database migration. diff --git a/docs/superpowers/plans/2026-07-20-staging-infrastructure-bootstrap.md b/docs/superpowers/plans/2026-07-20-staging-infrastructure-bootstrap.md new file mode 100644 index 00000000..6739218c --- /dev/null +++ b/docs/superpowers/plans/2026-07-20-staging-infrastructure-bootstrap.md @@ -0,0 +1,651 @@ +# Jyotisha Staging Infrastructure Bootstrap Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Prepare `118.26.111.127` as a secure, isolated staging target with DNS, a separate Supabase project, and a protected GitHub staging environment. + +**Architecture:** The Hong Kong VPS exposes only SSH, HTTP, and HTTPS. Application containers run behind Caddy, while staging data and authentication live in a separate Supabase project. GitHub Actions deploys through a dedicated Ed25519 key and pinned SSH host key. + +**Tech Stack:** Ubuntu 24.04 LTS, OpenSSH, UFW, Docker Engine, Docker Compose, Caddy, Supabase, GitHub Actions Environments. + +## Global Constraints + +- Production server, production DNS, production Supabase, and production GitHub secrets are out of scope. +- Staging domain is exactly `staging.jyotisha.chat`. +- Staging host is exactly `118.26.111.127`. +- Application path is exactly `/opt/jyotisha-staging`. +- Runtime environment file is exactly `/opt/jyotisha-staging/.env.staging` with mode `0600`. +- SSH, Supabase, database, and model-provider credentials must be staging-specific. +- Do not expose container ports `3000` or `5200` on the host. +- Do not disable password SSH until both `ubuntu` admin-key login and `deploy` deploy-key login succeed in separate terminals. +- Do not execute a database reset against any linked remote project. + +--- + +## File and Control-Plane Map + +- Cloud provider console: OS image, rescue console, security group. +- Local Mac: deploy pair `~/.ssh/jyotisha-staging*` and admin pair `~/.ssh/jyotisha-staging-admin*`. +- VPS: `/home/deploy/.ssh/authorized_keys`, `/etc/ssh/sshd_config.d/00-jyotisha-staging.conf`, `/etc/docker/daemon.json`, `/opt/jyotisha-staging/.env.staging`. +- DNS provider: `A staging.jyotisha.chat -> 118.26.111.127`. +- Supabase Dashboard: a new staging project, Auth URL configuration, staging credentials. +- GitHub repository settings: Environment named `staging`, one secret, six variables. + +### Task 1: Verify the Purchased VPS Before Configuration + +**Interfaces:** +- Consumes: Provider account containing `118.26.111.127`. +- Produces: A reachable Ubuntu 24.04 x86_64 server with console recovery available. + +- [ ] **Step 1: Verify provider-console facts** + +In the provider console, confirm all of these values before continuing: + +```text +Public IPv4: 118.26.111.127 +Region: Hong Kong +Architecture: x86_64 / amd64 +Operating system: Ubuntu 24.04 LTS +RAM: 4 GB +CPU: 2 vCPU +System disk: at least 40 GB +Console or rescue login: enabled +Automatic snapshot/backup: enabled if included +``` + +Expected: every value matches. If the architecture is ARM, the disk is below 40 GB, or no console/reinstall path exists, stop before configuring the server. + +- [ ] **Step 2: Restrict the provider security group** + +Create inbound rules: + +```text +TCP 22 source 0.0.0.0/0 SSH during bootstrap +TCP 80 source 0.0.0.0/0 HTTP and ACME redirect +TCP 443 source 0.0.0.0/0 HTTPS +UDP 443 source 0.0.0.0/0 HTTP/3, optional but used by Caddy +``` + +Delete inbound rules for `3000`, `5200`, database ports, provider control panels, and unrestricted custom port ranges. + +Expected: a provider-console screenshot or rule list contains only the four intended inbound rules. + +- [ ] **Step 3: Test the initial provider login** + +From the local Mac: + +```bash +ssh ubuntu@118.26.111.127 +``` + +Expected: a first-use host-key prompt followed by the provider's `ubuntu` password prompt, then an Ubuntu shell. Do not send the password in chat, GitHub, or shell history. + +- [ ] **Step 4: Confirm machine identity from the server** + +Run on the VPS: + +```bash +uname -m +source /etc/os-release +printf '%s %s\n' "$ID" "$VERSION_ID" +free -h +df -h / +ip -brief address +``` + +Expected: + +```text +x86_64 +ubuntu 24.04 +approximately 4 GiB RAM +at least 40 GB root disk +118.26.111.127 present on the public interface or provider NAT mapping +``` + +### Task 2: Create and Verify the Dedicated Deploy Identity + +**Interfaces:** +- Consumes: Initial `ubuntu` access from Task 1. +- Produces: `ubuntu@118.26.111.127` authenticated by the admin key and `deploy@118.26.111.127` authenticated by the deploy key. + +- [ ] **Step 1: Generate separate admin and deploy keys on the local Mac** + +Run locally, not on the VPS: + +```bash +test ! -e "$HOME/.ssh/jyotisha-staging-admin" +test ! -e "$HOME/.ssh/jyotisha-staging" +ssh-keygen -t ed25519 -a 64 -N '' -f "$HOME/.ssh/jyotisha-staging-admin" -C "jyotisha-staging-admin" +ssh-keygen -t ed25519 -a 64 -N '' -f "$HOME/.ssh/jyotisha-staging" -C "github-actions-jyotisha-staging" +chmod 600 "$HOME/.ssh/jyotisha-staging-admin" +chmod 600 "$HOME/.ssh/jyotisha-staging" +chmod 644 "$HOME/.ssh/jyotisha-staging-admin.pub" +chmod 644 "$HOME/.ssh/jyotisha-staging.pub" +ssh-keygen -lf "$HOME/.ssh/jyotisha-staging-admin.pub" +ssh-keygen -lf "$HOME/.ssh/jyotisha-staging.pub" +``` + +Expected: four key files are created and both fingerprints use `ED25519`. The admin private key remains only on the Mac. The deploy private key is later stored only in the GitHub `staging` Environment and must never be used for production. + +- [ ] **Step 2: Install only the admin public key on the ubuntu account** + +Run locally and type the server password only at the terminal prompt: + +```bash +ssh-copy-id -i "$HOME/.ssh/jyotisha-staging-admin.pub" ubuntu@118.26.111.127 +ssh -i "$HOME/.ssh/jyotisha-staging-admin" -o IdentitiesOnly=yes ubuntu@118.26.111.127 'id && sudo -n true' +``` + +Expected: the first command installs the public key; the second logs in as `ubuntu`. `sudo -n true` must exit `0`; if the provider requires a sudo password, keep the interactive admin session open and use `sudo` with the password typed directly at its prompt. + +- [ ] **Step 3: Create the deploy user on the VPS** + +Log in with the admin key and run: + +```bash +ssh -i "$HOME/.ssh/jyotisha-staging-admin" -o IdentitiesOnly=yes ubuntu@118.26.111.127 +sudo adduser --disabled-password --gecos "" deploy +sudo install -d -m 700 -o deploy -g deploy /home/deploy/.ssh +sudo install -d -m 755 -o deploy -g deploy /opt/jyotisha-staging +``` + +Expected: + +```bash +id deploy +sudo stat -c '%U %G %a %n' /home/deploy/.ssh /opt/jyotisha-staging +``` + +The output shows user `deploy`, `.ssh` mode `700`, and `/opt/jyotisha-staging` owned by `deploy`. + +- [ ] **Step 4: Copy only the deploy public key to the VPS** + +From a second local terminal: + +```bash +scp -i "$HOME/.ssh/jyotisha-staging-admin" -o IdentitiesOnly=yes \ + "$HOME/.ssh/jyotisha-staging.pub" ubuntu@118.26.111.127:/tmp/jyotisha-staging.pub +``` + +Then in the authenticated `ubuntu` session: + +```bash +sudo install -m 600 -o deploy -g deploy /tmp/jyotisha-staging.pub /home/deploy/.ssh/authorized_keys +sudo shred -u /tmp/jyotisha-staging.pub +``` + +Expected: + +```bash +sudo stat -c '%U %G %a %n' /home/deploy/.ssh/authorized_keys +``` + +Output: `deploy deploy 600 /home/deploy/.ssh/authorized_keys`. + +- [ ] **Step 5: Verify deploy-key login in a new terminal** + +Keep the `ubuntu` admin session open. From the local Mac: + +```bash +ssh -i "$HOME/.ssh/jyotisha-staging" -o IdentitiesOnly=yes deploy@118.26.111.127 'id && hostname' +``` + +Expected: exit code `0`; output contains `uid=` for `deploy`. Do not continue if this fails. + +- [ ] **Step 6: Pin and compare the server host key** + +On the VPS `ubuntu` session: + +```bash +sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub +``` + +On the local Mac: + +```bash +ssh-keyscan -t ed25519 -p 22 118.26.111.127 2>/dev/null > /tmp/jyotisha-staging-known-hosts +ssh-keygen -lf /tmp/jyotisha-staging-known-hosts +``` + +Expected: both fingerprints are identical. Preserve the exact line in `/tmp/jyotisha-staging-known-hosts` for GitHub Task 6. If they differ, stop and use the provider console to investigate. + +### Task 3: Patch, Harden, and Add Swap + +**Interfaces:** +- Consumes: Verified deploy-key login. +- Produces: Patched Ubuntu, 4 GB swap, key-only SSH, and host firewall rules. + +- [ ] **Step 1: Install base administration packages** + +Run from the authenticated `ubuntu` session with `sudo`: + +```bash +export DEBIAN_FRONTEND=noninteractive +sudo apt-get update +sudo apt-get dist-upgrade -y +sudo apt-get install -y ca-certificates curl git rsync ufw unattended-upgrades +sudo hostnamectl set-hostname jyotisha-staging +sudo timedatectl set-timezone UTC +sudo systemctl enable --now unattended-upgrades +``` + +Expected: all commands exit `0` and `hostnamectl --static` prints `jyotisha-staging`. + +- [ ] **Step 2: Create swap only if the VPS has none** + +Run with `sudo`: + +```bash +if [ "$(swapon --noheadings | wc -l)" -eq 0 ]; then + sudo fallocate -l 4G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + printf '%s\n' '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab >/dev/null +fi +sudo sysctl vm.swappiness=10 +printf '%s\n' 'vm.swappiness=10' | sudo tee /etc/sysctl.d/60-jyotisha-staging.conf >/dev/null +``` + +Expected: + +```bash +swapon --show +free -h +grep -F '/swapfile none swap sw 0 0' /etc/fstab +``` + +Output shows one 4 GB swap file and one matching `fstab` entry. + +- [ ] **Step 3: Configure UFW before enabling it** + +Run with `sudo`: + +```bash +sudo ufw default deny incoming +sudo ufw default allow outgoing +sudo ufw allow 22/tcp comment 'SSH' +sudo ufw allow 80/tcp comment 'HTTP' +sudo ufw allow 443/tcp comment 'HTTPS' +sudo ufw allow 443/udp comment 'HTTP3' +sudo ufw --force enable +sudo systemctl enable --now ufw +sudo ufw status verbose +systemctl is-enabled ufw +``` + +Expected: UFW is active and enabled at boot; only `22/tcp`, `80/tcp`, `443/tcp`, and `443/udp` are allowed. Docker-published ports must still be reviewed separately because Docker can bypass UFW; the application Compose file may publish only 80/443. + +- [ ] **Step 4: Harden SSH with a configuration snippet** + +Run with `sudo`: + +```bash +sudo install -m 600 /dev/null /etc/ssh/sshd_config.d/00-jyotisha-staging.conf +printf '%s\n' \ + 'PubkeyAuthentication yes' \ + 'PasswordAuthentication no' \ + 'KbdInteractiveAuthentication no' \ + 'PermitRootLogin no' \ + 'X11Forwarding no' \ + 'MaxAuthTries 3' \ + | sudo tee /etc/ssh/sshd_config.d/00-jyotisha-staging.conf >/dev/null +sudo sshd -t +sudo systemctl reload ssh +sudo sshd -T | grep -E '^(passwordauthentication|kbdinteractiveauthentication|permitrootlogin|pubkeyauthentication|maxauthtries) ' +``` + +Expected: `sshd -t` emits nothing and exits `0`; the effective configuration shows password and keyboard-interactive authentication disabled, root login disabled, public-key authentication enabled, and `maxauthtries 3`. The `00-` prefix intentionally loads before provider-generated snippets such as `50-cloud-init.conf`, because OpenSSH uses the first obtained value for these settings. + +- [ ] **Step 5: Re-test access before closing the original password session** + +From the local Mac: + +```bash +ssh -i "$HOME/.ssh/jyotisha-staging-admin" -o IdentitiesOnly=yes ubuntu@118.26.111.127 'printf "admin-key-ok\n"' +ssh -i "$HOME/.ssh/jyotisha-staging" -o IdentitiesOnly=yes deploy@118.26.111.127 'printf "deploy-key-ok\n"' +ssh -o PreferredAuthentications=password -o PubkeyAuthentication=no -o NumberOfPasswordPrompts=0 ubuntu@118.26.111.127 true +``` + +Expected: the first two commands print `admin-key-ok` and `deploy-key-ok`. The password-only command is rejected. Only now close the original password-authenticated session. + +### Task 4: Install Docker and Bound Its Disk Usage + +**Interfaces:** +- Consumes: Hardened server from Task 3. +- Produces: Docker Engine and Compose available to `deploy` with bounded local logs. + +- [ ] **Step 1: Install Docker from Docker's official apt repository** + +Use the authenticated `ubuntu` admin-key session and enter `sudo -i`. The `deploy` user intentionally has no general sudo access. Run: + +```bash +apt-get update +apt-get install -y ca-certificates curl +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +chmod a+r /etc/apt/keyrings/docker.asc +printf '%s\n' \ + 'Types: deb' \ + 'URIs: https://download.docker.com/linux/ubuntu' \ + "Suites: $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}")" \ + 'Components: stable' \ + "Architectures: $(dpkg --print-architecture)" \ + 'Signed-By: /etc/apt/keyrings/docker.asc' \ + > /etc/apt/sources.list.d/docker.sources +apt-get update +apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +systemctl enable --now docker +``` + +Expected: packages come from `download.docker.com`; no convenience `curl | sh` installer is used. Reference: . + +- [ ] **Step 2: Configure bounded Docker logs** + +Run in the `sudo -i` admin shell on the fresh server: + +```bash +printf '%s\n' \ + '{' \ + ' "log-driver": "local",' \ + ' "log-opts": {' \ + ' "max-size": "10m",' \ + ' "max-file": "5"' \ + ' }' \ + '}' \ + > /etc/docker/daemon.json +systemctl restart docker +``` + +Expected: `docker info --format '{{.LoggingDriver}}'` prints `local`. + +- [ ] **Step 3: Allow the deploy user to run Docker** + +Run in the `sudo -i` admin shell: + +```bash +usermod -aG docker deploy +chown -R deploy:deploy /opt/jyotisha-staging +``` + +Log out and reconnect as deploy, then run: + +```bash +docker version +docker compose version +docker run --rm hello-world +``` + +Expected: all three commands succeed without `sudo`. + +- [ ] **Step 4: Record the clean-server capacity baseline** + +Run as deploy: + +```bash +free -h +df -h / +docker system df +systemctl is-active docker +``` + +Expected: approximately 4 GB RAM plus 4 GB swap, at least 20 GB free disk, and Docker `active`. + +### Task 5: Create DNS and the Isolated Supabase Staging Project + +**Interfaces:** +- Consumes: Working HTTPS ports and access to DNS/Supabase dashboards. +- Produces: `staging.jyotisha.chat` resolving to the VPS and an empty staging Supabase project at the current schema. + +- [ ] **Step 1: Create the staging DNS record** + +In the authoritative DNS provider for `jyotisha.chat`, create exactly: + +```text +Type: A +Name/Host: staging +Value: 118.26.111.127 +TTL: 600 seconds (or provider default if fixed) +Proxy/CDN: DNS only during bootstrap +``` + +Verify locally: + +```bash +dig +short A staging.jyotisha.chat +``` + +Expected: `118.26.111.127` and no production IP. + +- [ ] **Step 2: Create a separate Supabase project** + +In the Supabase organization, create a new project with: + +```text +Project name: jyotisha-staging +Region: Singapore / ap-southeast-1 +Database password: newly generated and stored in password manager +Production restore/data import: disabled +``` + +Expected: the project Dashboard URL has a new project reference different from `vtvnfqmonbfuxmqkqdlc`. + +- [ ] **Step 3: Configure staging Auth URLs** + +In `Authentication -> URL Configuration`, set: + +```text +Site URL: https://staging.jyotisha.chat +Redirect URL: https://staging.jyotisha.chat/** +``` + +Do not add the production URL to the staging project. + +- [ ] **Step 4: Link the local CLI to staging and preview migrations** + +From the repository on the local Mac: + +```bash +cd /Users/jesse/Downloads/Copse/astrology/yinduzhanxing/frontend +npx supabase login +read -r -p 'Paste the jyotisha-staging project reference: ' STAGING_PROJECT_REF +test "$STAGING_PROJECT_REF" != 'vtvnfqmonbfuxmqkqdlc' +npx supabase link --project-ref "$STAGING_PROJECT_REF" +npx supabase db push --dry-run +``` + +Expected: the production-project guard passes, and the dry run lists the repository migrations without applying them. Supabase documents `--dry-run` for exactly this preview: . + +- [ ] **Step 5: Apply migrations only after reviewing the dry run** + +Run: + +```bash +npx supabase db push +npx supabase migration list +``` + +Expected: `db push` succeeds and local/remote migration versions align. If any migration fails, stop; do not mark migration history manually and do not fall back to production credentials. + +- [ ] **Step 6: Create the server-only staging environment file** + +Collect from the staging project Dashboard: project URL, publishable/anon key, service-role key, and Session Pooler database URL. Create low-limit staging model-provider keys. Then log in as deploy and run the following interactive sequence. Secret prompts do not echo their values: + +```bash +cd /opt/jyotisha-staging +read -r -p 'Staging Supabase URL: ' STAGING_SUPABASE_URL +read -r -s -p 'Staging Supabase publishable/anon key: ' STAGING_SUPABASE_ANON_KEY; printf '\n' +read -r -s -p 'Staging Supabase service-role key: ' STAGING_SUPABASE_SERVICE_KEY; printf '\n' +read -r -s -p 'Staging Session Pooler database URL: ' STAGING_DB_URL; printf '\n' +read -r -p 'Staging admin email: ' STAGING_ADMIN_EMAIL +read -r -p 'Staging default model id: ' STAGING_DEFAULT_MODEL +read -r -s -p 'Staging LLM_MODELS_JSON: ' STAGING_MODEL_CATALOG; printf '\n' +read -r -s -p 'Staging OpenAI key (press Enter if unused): ' STAGING_OPENAI_KEY; printf '\n' +read -r -s -p 'Staging DeepSeek key (press Enter if unused): ' STAGING_DEEPSEEK_KEY; printf '\n' +read -r -s -p 'Staging VedAstro key: ' STAGING_VEDASTRO_KEY; printf '\n' +umask 077 +printf '%s\n' \ + 'APP_ENV_FILE=../.env.staging' \ + 'CADDYFILE_PATH=./Caddyfile.staging' \ + 'SITE_ADDRESS=https://staging.jyotisha.chat' \ + 'JYOTISH_API_BASE=http://api:5200' \ + "NEXT_PUBLIC_SUPABASE_URL=$STAGING_SUPABASE_URL" \ + "NEXT_PUBLIC_SUPABASE_ANON_KEY=$STAGING_SUPABASE_ANON_KEY" \ + "SUPABASE_SERVICE_ROLE_KEY=$STAGING_SUPABASE_SERVICE_KEY" \ + "SUPABASE_DB_URL=$STAGING_DB_URL" \ + "ADMIN_EMAILS=$STAGING_ADMIN_EMAIL" \ + "LLM_DEFAULT_MODEL_ID=$STAGING_DEFAULT_MODEL" \ + "LLM_MODELS_JSON=$STAGING_MODEL_CATALOG" \ + "OPENAI_API_KEY=$STAGING_OPENAI_KEY" \ + "DEEPSEEK_API_KEY=$STAGING_DEEPSEEK_KEY" \ + 'VEDASTRO_GATEWAY_MODE=official_first' \ + 'VEDASTRO_API_ENDPOINT=https://api.vedastro.org/api' \ + 'VEDASTRO_ENABLE_NETWORK=1' \ + 'VEDASTRO_TIMEOUT_SECONDS=20' \ + "VEDASTRO_API_KEY=$STAGING_VEDASTRO_KEY" \ + > .env.staging +chmod 600 .env.staging +unset STAGING_SUPABASE_URL STAGING_SUPABASE_ANON_KEY STAGING_SUPABASE_SERVICE_KEY STAGING_DB_URL +unset STAGING_ADMIN_EMAIL STAGING_DEFAULT_MODEL STAGING_MODEL_CATALOG STAGING_OPENAI_KEY STAGING_DEEPSEEK_KEY STAGING_VEDASTRO_KEY +``` + +Do not use production values in any prompt. If a model provider is unused, its key may be blank only when `LLM_MODELS_JSON` does not reference that environment variable. + +Verify names without printing values: + +```bash +awk -F= 'NF && $1 !~ /^#/ {print $1}' .env.staging | sort +stat -c '%U %G %a %n' .env.staging +``` + +Expected: all required names appear and mode is `600` owned by deploy. + +### Task 6: Create the Protected GitHub Staging Environment + +**Interfaces:** +- Consumes: Deploy private key, verified known-hosts line, VPS and DNS values. +- Produces: GitHub Environment `staging` with one secret and six variables. + +- [ ] **Step 1: Create the Environment in GitHub UI** + +Open: + +```text +Repository -> Settings -> Environments -> New environment +Name: staging +``` + +Set deployment branches to `Selected branches and tags`, then allow only branch pattern `main`. + +Expected: the Environment page displays `staging` and allows the `main` controller branch. GitHub evaluates Environment branch rules against the deployment workflow's own `GITHUB_REF`; a `workflow_run` controller executes from the default branch even when the tested upstream revision came from branch `staging`. The workflow separately enforces `head_branch == 'staging'` and deploys the upstream `head_sha`. + +- [ ] **Step 2: Add the staging SSH private key as an Environment secret** + +Create exactly one Environment secret: + +```text +Name: STAGING_SSH_PRIVATE_KEY +Value: complete contents of ~/.ssh/jyotisha-staging +``` + +Do not put this key in repository-level secrets and do not reuse `PRODUCTION_SSH_PRIVATE_KEY`. + +- [ ] **Step 3: Add non-secret Environment variables** + +Create the first five variables with the literal values shown: + +```text +STAGING_HOST=118.26.111.127 +STAGING_PORT=22 +STAGING_USER=deploy +STAGING_PATH=/opt/jyotisha-staging +STAGING_URL=https://staging.jyotisha.chat +``` + +Create the sixth variable with name `STAGING_KNOWN_HOSTS`. Its value is the complete output of this local command: + +```bash +cat /tmp/jyotisha-staging-known-hosts +``` + +Paste the full line beginning with `118.26.111.127` or `[118.26.111.127]:22`; do not paste only the fingerprint. + +- [ ] **Step 4: Verify the Environment has no production credentials** + +Expected Environment inventory: + +```text +Secrets (1): STAGING_SSH_PRIVATE_KEY +Variables (6): STAGING_HOST, STAGING_PORT, STAGING_USER, STAGING_PATH, STAGING_URL, STAGING_KNOWN_HOSTS +Allowed controller branch: main +``` + +GitHub Environment secrets become available only to jobs that explicitly reference that Environment: . + +### Task 7: Infrastructure Readiness Gate + +**Interfaces:** +- Consumes: Tasks 1–6. +- Produces: A go/no-go result for repository deployment automation. + +- [ ] **Step 1: Run the local SSH and DNS checks** + +```bash +dig +short A staging.jyotisha.chat +ssh -i "$HOME/.ssh/jyotisha-staging" -o IdentitiesOnly=yes deploy@118.26.111.127 \ + 'hostname; free -h; df -h /; docker version --format "{{.Server.Version}}"; docker compose version; stat -c "%a %n" /opt/jyotisha-staging/.env.staging' +``` + +Expected: correct IP, hostname `jyotisha-staging`, Docker/Compose versions, and `.env.staging` mode `600`. + +- [ ] **Step 2: Confirm no unintended public ports** + +Run locally: + +```bash +nc -vz 118.26.111.127 22 +nc -vz 118.26.111.127 80 +nc -vz 118.26.111.127 443 +nc -vz -w 3 118.26.111.127 3000 +nc -vz -w 3 118.26.111.127 5200 +``` + +Expected: 22 is reachable. Before application deployment, 80/443 may refuse because nothing is listening; this is acceptable. Ports 3000 and 5200 must not connect. + +- [ ] **Step 3: Reboot once and verify the bootstrap survives** + +From the authenticated `ubuntu` admin-key session: + +```bash +sudo systemctl reboot +``` + +Wait for the provider console to report the VPS online, then run locally: + +```bash +ssh -i "$HOME/.ssh/jyotisha-staging" -o IdentitiesOnly=yes deploy@118.26.111.127 \ + 'hostname; swapon --show; systemctl is-active docker; systemctl is-active ufw' +``` + +Expected: deploy-key login works after reboot, swap is present, Docker is `active`, and UFW is active. + +- [ ] **Step 4: Record the go/no-go decision** + +Go only if all are true: + +```text +ubuntu admin key works +deploy key works +root/password SSH is disabled +host-key fingerprints match +Docker and Compose work as deploy +4 GB swap exists +DNS resolves only to 118.26.111.127 +staging Supabase project reference differs from production +migrations applied successfully to staging +.env.staging exists with mode 0600 +GitHub staging Environment contains only staging credentials +``` + +If any item is false, stop before implementing or running the deployment workflow. diff --git a/docs/superpowers/specs/2026-07-20-staging-server-design.md b/docs/superpowers/specs/2026-07-20-staging-server-design.md new file mode 100644 index 00000000..31da798d --- /dev/null +++ b/docs/superpowers/specs/2026-07-20-staging-server-design.md @@ -0,0 +1,132 @@ +# Jyotisha 香港 Staging 服务器设计 + +日期:2026-07-20 + +## 目标 + +把香港服务器 `118.26.111.127` 建设成与生产隔离的 staging 环境,用于验证 GitHub 自动部署、空数据库重建、Supabase 解耦和迁出演练。迁移阶段完成后,这台服务器可停止 staging 服务并转为第三方文字模型 generation worker。 + +本设计不修改当前生产服务器、`jyotisha.chat` DNS 或生产 Supabase 项目。 + +## 环境边界 + +| 项目 | Production | Staging | +| --- | --- | --- | +| 域名 | `jyotisha.chat` | `staging.jyotisha.chat` | +| 服务器 | 现有生产 VPS | `118.26.111.127` | +| GitHub Environment | `production` | `staging` | +| 部署密钥 | production 专用 | staging 专用 | +| 应用配置 | `.env.production` | `.env.staging` | +| Supabase | 生产项目 | 独立 staging 项目 | +| 部署触发 | 手动 production workflow | `staging` CI 成功或手动触发 | + +staging 不得写入生产数据库,不得复用 service-role key、数据库密码、SSH 私钥或模型计费密钥。模型接口优先使用独立测试 key、低额度或 provider sandbox。 + +## 服务器基础设计 + +- 操作系统:Ubuntu 24.04 LTS x86_64。 +- 访问:供应商默认 SSH 用户为 `ubuntu`;本机管理密钥只授权给 `ubuntu`,GitHub deploy 密钥只授权给 `deploy`。两个 key 登录都验证成功后再关闭 SSH 密码登录和直接 root 登录。 +- 内存:2 vCPU / 4 GB RAM,增加 4 GB swap;staging 部署串行执行,避免构建峰值并发。 +- 防火墙:只开放 SSH、80、443;Python API 5200 和 Next.js 3000 只在 Docker 网络暴露。 +- 运行时:Docker Engine、Buildx 和 Compose plugin,从 Docker 官方 apt repository 安装。 +- 目录:应用位于 `/opt/jyotisha-staging`,`.env.staging` 权限为 `0600`,不由 rsync 或 Git 覆盖。 +- 运维:启用安全更新、日志轮转、磁盘/内存监控;部署前后记录 Docker 状态和健康检查。 + +## DNS 与 TLS + +在域名供应商创建: + +```text +A staging.jyotisha.chat 118.26.111.127 +``` + +Compose 通过 `SITE_ADDRESS=https://staging.jyotisha.chat` 配置 Caddy。DNS 生效后由 Caddy申请和续期 TLS 证书。生产根域名记录保持不变。 + +## 应用配置 + +现有 Compose 固定引用 `.env.production`。实施时应使同一份 Compose 接受一个显式配置文件参数,production 默认行为保持不变,staging 指向 `/opt/jyotisha-staging/.env.staging`。不复制一整份长期漂移的 Compose 文件。 + +staging 配置包含: + +- 独立 Supabase URL、anon key、service-role key、DB URL; +- `SITE_ADDRESS=https://staging.jyotisha.chat`; +- staging 专用 admin email 和模型 key; +- 明确的 environment 标识,防止邮件、计费或任务误指向 production; +- 与生产不同的 cookie/session 名称或域范围,避免浏览器 session 混淆。 + +## GitHub 部署设计 + +新增 `staging` GitHub Environment: + +- Secret:`STAGING_SSH_PRIVATE_KEY`; +- Variable:`STAGING_HOST=118.26.111.127`、SSH port/user/path、staging URL; +- GitHub Environment 只允许控制器分支 `main` 使用;`workflow_run` 另外强制上游成功运行来自 `staging`,并部署其 `head_sha`; +- staging 部署使用独立 concurrency group,不能阻塞或取消 production。 + +部署流: + +```text +push staging + -> Jyotish Skill CI + -> checkout 已测试 SHA + -> 记录旧 SHA 和镜像 ID + -> rsync 到 /opt/jyotisha-staging(排除所有 .env*) + -> 校验 .env.staging 权限、固定选择器和 Compose 配置,并在 Compose 进程上显式钉死 staging 选择器 + -> docker compose build/up + -> login、401 account、Python health smoke tests + -> 记录部署 SHA +``` + +数据库 migration 不隐式混入普通应用部署。迁移必须是单独、可见、可审计的步骤,先在 staging DB 执行并验证,再决定 production 运行窗口。 + +## 错误处理与回滚 + +- SSH、rsync、build 或 health check 任一步失败,workflow 必须失败并保留日志。 +- 新容器健康检查未通过时,不宣告部署成功。 +- 部署前记录当前 SHA 和镜像;应用回滚恢复到上一已验证 SHA。 +- 数据库 migration 必须有独立备份和恢复演练。应用回滚不能被误认为数据库回滚。 +- Caddy、web、api 中任一服务不健康时,保留 SSH 故障排查通道,不自动删除 volumes 或环境文件。 + +## 验收测试 + +服务器基础验收: + +- ubuntu admin key 与 deploy key 分别登录成功,密码/root 登录按设计受限; +- UFW 与云防火墙只开放预期端口; +- Docker/Compose 正常;swap 生效;重启后容器能恢复。 + +部署验收: + +- `https://staging.jyotisha.chat/login` 返回成功; +- 未登录 `/api/account` 返回 401; +- web 容器能访问私有 Python `/api/health`,且 Swiss Ephemeris 可用; +- staging 页面与 cookie 不影响 production; +- GitHub 显示 staging deployment 和部署 SHA; +- 故意部署一个失败健康检查的测试 revision 时,workflow 能阻止其被标记为成功。 + +数据隔离验收: + +- staging 注册用户只出现在 staging Auth; +- profile、chat、credits、redemption、jobs 均只写 staging DB; +- staging service role 无法连接 production project; +- staging migration 可从空库重建到当前版本。 + +## 分阶段实施 + +1. 通过云厂商控制台确认系统、架构、磁盘、网络和救援入口。 +2. 初始化 SSH、安全更新、deploy 用户、swap、防火墙和 Docker。 +3. 建立独立 Supabase staging 项目和 `.env.staging`。 +4. 配置 `staging.jyotisha.chat` DNS 与 Caddy TLS。 +5. 参数化 Compose 配置文件选择,不改变 production 默认路径。 +6. 新增 GitHub staging Environment 和 deployment workflow。 +7. 手动首部署并验收,再启用分支自动部署。 +8. 完成空库 migration、恢复和 Supabase 解耦演练。 +9. staging 使命完成后,重新设计并切换为私有 generation worker;不直接把公开 staging 容器当作生产 worker。 + +## 非目标 + +- 本阶段不迁移生产用户或生产数据库。 +- 不改变 production 自动部署。 +- 不购买或部署国内后端服务器。 +- 不在本机运行大模型。 +- 不在 staging 和 production 之间做应用双写。 diff --git a/frontend/tests/health-deployment.test.ts b/frontend/tests/health-deployment.test.ts index 4888e47b..f4f15bf4 100644 --- a/frontend/tests/health-deployment.test.ts +++ b/frontend/tests/health-deployment.test.ts @@ -1,9 +1,24 @@ import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; import test from "node:test"; +import { fileURLToPath } from "node:url"; test("health endpoint exposes deployment identity for production verification", () => { - const source = readFileSync(new URL("../src/app/api/health/route.ts", import.meta.url), "utf8"); + const source = readFileSync( + new URL("../src/app/api/health/route.ts", import.meta.url), + "utf8", + ); assert.match(source, /deployment:/); assert.match(source, /GITHUB_SHA/); @@ -11,12 +26,20 @@ test("health endpoint exposes deployment identity for production verification", assert.match(source, /gitCommit/); }); -test("production deployment passes the tested revision into the web runtime", () => { - const compose = readFileSync(new URL("../../deploy/docker-compose.server.yml", import.meta.url), "utf8"); - const workflow = readFileSync(new URL("../../.github/workflows/deploy-production.yml", import.meta.url), "utf8"); +test("manual production deployment passes the selected revision into the web runtime", () => { + const compose = readFileSync( + new URL("../../deploy/docker-compose.server.yml", import.meta.url), + "utf8", + ); + const workflow = readFileSync( + new URL("../../.github/workflows/deploy-production.yml", import.meta.url), + "utf8", + ); assert.match(compose, /GITHUB_SHA: \$\{GITHUB_SHA\}/); - assert.match(workflow, /DEPLOY_GIT_SHA: \$\{\{ github\.event\.workflow_run\.head_sha \|\| github\.sha \}\}/); + assert.match(workflow, /workflow_dispatch:/); + assert.doesNotMatch(workflow, /workflow_run:/); + assert.match(workflow, /DEPLOY_GIT_SHA: \$\{\{ github\.sha \}\}/); assert.match(workflow, /GITHUB_SHA='\$DEPLOY_GIT_SHA'/); assert.match(workflow, /get\("deployment", \{\}\)\.get\("gitCommit"/); assert.match(workflow, /DEPLOY_GIT_SHA/); @@ -24,3 +47,239 @@ test("production deployment passes the tested revision into the web runtime", () assert.match(workflow, /git ls-remote origin refs\/heads\/main/); assert.match(workflow, /steps\.revision\.outputs\.deploy == 'true'/); }); + +test("server compose accepts staging paths while preserving production defaults", () => { + const compose = readFileSync( + new URL("../../deploy/docker-compose.server.yml", import.meta.url), + "utf8", + ); + + assert.match( + compose, + /env_file:\s*\n\s*- \$\{APP_ENV_FILE:-\.\.\/\.env\.production\}/, + ); + assert.match( + compose, + /\$\{CADDYFILE_PATH:-\.\/Caddyfile\}:\/etc\/caddy\/Caddyfile:ro/, + ); + assert.match( + compose, + /SITE_ADDRESS: \$\{SITE_ADDRESS:-https:\/\/jyotisha\.chat\}/, + ); +}); + +test("staging Caddy configuration serves only the configured staging address", () => { + const caddy = readFileSync( + new URL("../../deploy/Caddyfile.staging", import.meta.url), + "utf8", + ); + + assert.match(caddy, /\{\$SITE_ADDRESS:https:\/\/staging\.jyotisha\.chat\}/); + assert.match(caddy, /reverse_proxy web:3000/); + assert.doesNotMatch(caddy, /www\.jyotisha\.chat/); +}); + +test("staging deploy consumes only the isolated staging environment and tested revision", () => { + const ci = readFileSync( + new URL("../../.github/workflows/ci.yml", import.meta.url), + "utf8", + ); + const workflow = readFileSync( + new URL("../../.github/workflows/deploy-staging.yml", import.meta.url), + "utf8", + ); + + assert.match(ci, /push:\s*\n\s*branches: \[staging\]/); + assert.match(workflow, /workflows: \["Jyotish Skill CI"\]/); + assert.match( + workflow, + /github\.event\.workflow_run\.head_branch == 'staging'/, + ); + assert.match(workflow, /actions: read/); + assert.match(workflow, /environment:\s*\n\s*name: staging/); + assert.match(workflow, /git_sha:/); + assert.doesNotMatch(workflow, /default: staging/); + assert.match(workflow, /test "\$\{#REQUESTED_SHA\}" -eq 40/); + assert.match(workflow, /actions\/workflows\/ci\.yml\/runs\?head_sha=/); + assert.match(workflow, /STAGING_SSH_PRIVATE_KEY/); + assert.match(workflow, /vars\.STAGING_HOST/); + assert.match(workflow, /vars\.STAGING_KNOWN_HOSTS/); + assert.match(workflow, /test "\$DEPLOY_HOST" = "118\.26\.111\.127"/); + assert.match(workflow, /test "\$DEPLOY_USER" = "deploy"/); + assert.match(workflow, /test "\$DEPLOY_PATH" = "\/opt\/jyotisha-staging"/); + assert.match(workflow, /--exclude='\.env\*'/); + assert.match(workflow, /docker compose --env-file \.env\.staging/); + assert.match( + workflow, + /bash deploy\/validate-staging-env\.sh \.env\.staging/, + ); + assert.match( + workflow, + /docker compose --env-file \.env\.staging -f deploy\/docker-compose\.server\.yml config --quiet/, + ); + assert.match(workflow, /deployment\.gitCommit/); + assert.doesNotMatch(workflow, /PRODUCTION_SSH_PRIVATE_KEY/); + assert.doesNotMatch(workflow, /103\.117\.123\.53/); + + const composeLines = workflow + .split("\n") + .filter((line) => line.includes("docker compose")); + assert.equal(workflow.match(/docker compose/g)?.length, 4); + assert.equal(composeLines.length, 3); + for (const line of composeLines) { + assert.match(line, /APP_ENV_FILE='\.\.\/\.env\.staging'/); + assert.match(line, /CADDYFILE_PATH='\.\/Caddyfile\.staging'/); + assert.match(line, /SITE_ADDRESS='https:\/\/staging\.jyotisha\.chat'/); + } +}); + +test("staging rsync preserves every destination env variant during delete", () => { + const workflow = readFileSync( + new URL("../../.github/workflows/deploy-staging.yml", import.meta.url), + "utf8", + ); + const envExclusion = workflow.match(/--exclude='([^']*\.env[^']*)'/)?.[1]; + + assert.equal(envExclusion, ".env*"); + + const root = mkdtempSync(join(tmpdir(), "jyotisha-staging-rsync-")); + const source = join(root, "source"); + const destination = join(root, "destination"); + mkdirSync(source); + mkdirSync(destination); + writeFileSync(join(source, "app.txt"), "new revision\n"); + for (const name of [ + ".env", + ".env.local", + ".env.staging", + ".env.staging.backup", + ]) { + writeFileSync(join(destination, name), "preserve\n"); + } + + try { + const result = spawnSync( + "rsync", + [ + "-a", + "--delete", + `--exclude=${envExclusion}`, + `${source}/`, + `${destination}/`, + ], + { encoding: "utf8" }, + ); + assert.equal(result.status, 0, result.stderr); + for (const name of [ + ".env", + ".env.local", + ".env.staging", + ".env.staging.backup", + ]) { + assert.equal( + existsSync(join(destination, name)), + true, + `${name} was deleted`, + ); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("staging env validator rejects selector drift, duplicates, and unsafe permissions", () => { + const validator = fileURLToPath( + new URL("../../deploy/validate-staging-env.sh", import.meta.url), + ); + const root = mkdtempSync(join(tmpdir(), "jyotisha-staging-env-")); + const envFile = join(root, ".env.staging"); + const composeFile = join(root, "compose.yml"); + const validSelectors = [ + "APP_ENV_FILE=../.env.staging", + "CADDYFILE_PATH=./Caddyfile.staging", + "SITE_ADDRESS=https://staging.jyotisha.chat", + ]; + const run = () => + spawnSync("bash", [validator, envFile], { encoding: "utf8" }); + const writeEnv = (lines: string[], mode = 0o600) => { + writeFileSync(envFile, `${lines.join("\n")}\n`); + chmodSync(envFile, mode); + }; + + try { + writeFileSync( + composeFile, + [ + "services:", + " probe:", + " image: alpine", + " environment:", + " SELECTED: ${APP_ENV_FILE}", + "", + ].join("\n"), + ); + writeEnv(validSelectors); + assert.equal(run().status, 0); + const shellOverride = spawnSync( + "docker", + [ + "compose", + "--env-file", + envFile, + "-f", + composeFile, + "config", + "--format", + "json", + ], + { + encoding: "utf8", + env: { ...process.env, APP_ENV_FILE: "../.env.production" }, + }, + ); + assert.equal(shellOverride.status, 0, shellOverride.stderr); + assert.equal( + JSON.parse(shellOverride.stdout).services.probe.environment.SELECTED, + "../.env.production", + ); + + writeEnv(["APP_ENV_FILE=../.env.production", ...validSelectors.slice(1)]); + assert.notEqual(run().status, 0); + + writeEnv([...validSelectors, "SITE_ADDRESS=https://example.invalid"]); + assert.notEqual(run().status, 0); + + writeEnv([...validSelectors, "APP_ENV_FILE = ../.env.production"]); + assert.notEqual(run().status, 0); + const rendered = spawnSync( + "docker", + [ + "compose", + "--env-file", + envFile, + "-f", + composeFile, + "config", + "--format", + "json", + ], + { encoding: "utf8" }, + ); + assert.equal(rendered.status, 0, rendered.stderr); + assert.equal( + JSON.parse(rendered.stdout).services.probe.environment.SELECTED, + "../.env.production", + ); + + writeEnv([...validSelectors, "export CADDYFILE_PATH=./Caddyfile"]); + assert.notEqual(run().status, 0); + + writeEnv([...validSelectors, "SITE_ADDRESS"]); + assert.notEqual(run().status, 0); + + writeEnv(validSelectors, 0o644); + assert.notEqual(run().status, 0); + } finally { + rmSync(root, { recursive: true, force: true }); + } +});