"""Contracts for the closed-beta safety net. Fixtures are fictional.""" from __future__ import annotations import json import os import shutil import subprocess import sys import threading import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path ROOT = Path(__file__).resolve().parents[1] WORKFLOW = ROOT / ".gitea" / "workflows" / "backup-production-daily.yml" RECOVERY_WORKFLOW = ROOT / ".gitea" / "workflows" / "create-production-recovery.yml" RECOVERY_SCRIPT = ROOT / "deploy" / "run-production-recovery.sh" PRUNE_SCRIPT = ROOT / "deploy" / "prune-production-daily-backups.sh" HEALTH_SCRIPT = ROOT / "deploy" / "monitor" / "healthcheck-alert.sh" DIGEST_SCRIPT = ROOT / "deploy" / "monitor" / "error-digest.sh" FIND_SCRIPT = ROOT / "deploy" / "find-request-log.sh" NOTIFY_SCRIPT = ROOT / "deploy" / "monitor" / "notify-feishu.sh" SERVER_COMPOSE = ROOT / "deploy" / "docker-compose.server.yml" POSTGRES_COMPOSE = ROOT / "deploy" / "docker-compose.postgres.yml" def resolve_bash() -> str: found = shutil.which("bash") git_bash = Path(r"D:\Program Files\Git\bin\bash.exe") if os.name == "nt" and git_bash.exists(): lowered = (found or "").lower() if not found or "system32" in lowered or "windowsapps" in lowered: return str(git_bash) if found: return found if git_bash.exists(): return str(git_bash) return "bash" BASH = resolve_bash() def bash_path(path: Path) -> str: if os.name != "nt": return str(path) completed = subprocess.run( [BASH, "-lc", 'cygpath -u "$1"', "cygpath", str(path)], check=True, capture_output=True, text=True, encoding="utf-8", ) return completed.stdout.strip() def run_bash(script: Path, extra: dict[str, str], args: list[str] | None = None) -> subprocess.CompletedProcess[str]: env = os.environ.copy() env.pop("FEISHU_WEBHOOK_URL", None) env.pop("FEISHU_SIGN_SECRET", None) env.update(extra) env["JYOTISHA_PYTHON"] = bash_path(Path(sys.executable)) return subprocess.run( [BASH, bash_path(script), *(args or [])], cwd=ROOT, env=env, stdin=subprocess.DEVNULL, capture_output=True, text=True, encoding="utf-8", errors="replace", ) def write_text(path: Path, text: str) -> None: path.write_text(text, encoding="utf-8", newline="\n") class HealthHandler(BaseHTTPRequestHandler): def do_GET(self) -> None: if self.path != "/health": self.send_response(404) self.end_headers() return kind = getattr(self.server, "kind", "ok") if kind == "timeout": time.sleep(3) commit = getattr(self.server, "commit", "a" * 40) if kind == "500": status, body = 500, b'{"status":"ok"}' elif kind == "missing": status, body = 200, b"{}" elif kind == "degraded": status, body = 200, json.dumps({"status": "degraded", "deployment": {"gitCommit": commit}}).encode() else: status, body = 200, json.dumps({"status": "ok", "deployment": {"gitCommit": commit}}).encode() self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, fmt: str, *args: object) -> None: return class HookHandler(BaseHTTPRequestHandler): def do_POST(self) -> None: length = int(self.headers.get("Content-Length", "0")) body = self.rfile.read(length) self.server.bodies.append(body) self.close_connection = True self.send_response(200) self.send_header("Content-Length", "2") self.send_header("Connection", "close") self.end_headers() self.wfile.write(b"ok") def log_message(self, fmt: str, *args: object) -> None: return def serve(handler: type[BaseHTTPRequestHandler]) -> tuple[ThreadingHTTPServer, threading.Thread]: server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() return server, thread def alert_lines(state: Path) -> list[str]: log = state / "alerts.log" if not log.exists(): return [] return [line for line in log.read_text(encoding="utf-8").splitlines() if line] def test_daily_workflow_is_scheduled_from_main_and_reuses_recovery() -> None: text = WORKFLOW.read_text(encoding="utf-8") assert "\tif:" not in text assert "if: gitea.ref == 'refs/heads/main'" in text assert 'cron: "0 20 * * *"' in text assert "workflow_dispatch:" in text assert 'git show "$GITEA_SHA:deploy/run-production-recovery.sh"' in text assert "RECOVERY_BASENAME_PREFIX='production-daily'" in text assert "prune-production-daily-backups.sh" in text assert "production-daily-" in text assert 'artifact_retention="${ARTIFACT_RETENTION_DAYS:-14}"' in text assert 'local_retention="${LOCAL_RETENTION_DAYS:-7}"' in text assert 'process.env["INPUT_RETENTION-DAYS"]=process.env.ARTIFACT_RETENTION_DAYS' in text assert "set +x" in text assert "pg_dump" not in text assert "CHECK_RELEASE_GATE" not in text assert "observed_staging_sha" not in text assert "STAGING_URL" not in text assert "echo \"$SSH_PRIVATE_KEY" not in text assert "PRODUCTION_BACKUP_ENCRYPTION_KEY" not in text recovery = RECOVERY_WORKFLOW.read_text(encoding="utf-8") assert "observed_staging_sha" in recovery assert 'process.env["INPUT_RETENTION-DAYS"]="30"' in recovery def test_recovery_script_keeps_manual_names_unless_daily_prefix() -> None: text = RECOVERY_SCRIPT.read_text(encoding="utf-8") assert 'backup_prefix="production-pre-migration"' in text assert 'production-daily) backup_prefix="production-daily"' in text assert "openssl enc -aes-256-cbc -salt -pbkdf2" in text assert "usage_percent < 70" in text assert "flock -n 9" in text def test_prune_deletes_only_expired_daily_files(tmp_path: Path) -> None: backup = tmp_path / "backups" nested = backup / "nested" nested.mkdir(parents=True) now = 1_700_000_000 limit = 7 * 86400 names = { "production-daily-20261001T000000Z.dump.enc": now - limit - 60, "production-daily-20261001T000000Z-restore-verify.json": now - limit - 60, "production-daily-20261007T000000Z.dump.enc": now - limit, "production-daily-20261007T000000Z-restore-verify.json": now - limit, "production-daily-20261014T000000Z.dump.enc": now - 3600, "production-daily-20261014T000000Z-restore-verify.json": now - 3600, "production-pre-migration-20261001T000000Z.dump.enc": now - limit - 3600, "production-pre-migration-20261001T000000Z-restore-verify.json": now - limit - 3600, "production-daily-notes.txt": now - limit - 3600, "production-daily-20261001T000000Z.dump.enc.bak": now - limit - 3600, "unrelated.txt": now - limit - 3600, } for name, epoch in names.items(): path = backup / name write_text(path, "fictional\n") subprocess.run( [BASH, "-lc", 'touch -d "@$1" "$2"', "touch", str(epoch), bash_path(path)], check=True, ) write_text(nested / "production-daily-20261001T000000Z.dump.enc", "nested\n") completed = run_bash(PRUNE_SCRIPT, {}, [bash_path(backup), "7", str(now)]) assert completed.returncode == 0, completed.stderr deleted = completed.stdout.splitlines() assert deleted == ["production-daily-20261001T000000Z.dump.enc"] assert not (backup / "production-daily-20261001T000000Z.dump.enc").exists() assert not (backup / "production-daily-20261001T000000Z-restore-verify.json").exists() for name in names: if name.startswith("production-daily-20261001T000000Z.dump") or name.startswith( "production-daily-20261001T000000Z-restore" ): continue assert (backup / name).exists(), name assert (nested / "production-daily-20261001T000000Z.dump.enc").exists() def test_health_alerts_after_three_failures_and_recovery_once(tmp_path: Path) -> None: server, _thread = serve(HealthHandler) server.kind = "500" state = tmp_path / "state" env = { "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), "JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{server.server_address[1]}/health", "JYOTISHA_HEALTH_TIMEOUT": "2", } try: for _ in range(2): completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) assert completed.returncode == 0, completed.stderr assert alert_lines(state) == [] completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) assert completed.returncode == 0, completed.stderr lines = alert_lines(state) assert len(lines) == 1 assert "errorCode=health_down" in lines[0] assert "count=3" in lines[0] assert "route=/api/health" in lines[0] server.kind = "ok" server.commit = "b" * 40 completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) assert completed.returncode == 0, completed.stderr lines = alert_lines(state) assert len(lines) == 2 assert "errorCode=recovered" in lines[1] completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) assert completed.returncode == 0, completed.stderr assert len(alert_lines(state)) == 2 finally: server.shutdown() def test_health_timeout_and_missing_fields_count_as_failures(tmp_path: Path) -> None: server, _thread = serve(HealthHandler) server.kind = "timeout" state = tmp_path / "state" env = { "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), "JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{server.server_address[1]}/health", "JYOTISHA_HEALTH_TIMEOUT": "1", } try: completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) assert completed.returncode == 0, completed.stderr assert alert_lines(state) == [] server.kind = "missing" run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) assert completed.returncode == 0, completed.stderr assert any("errorCode=health_down" in line for line in alert_lines(state)) finally: server.shutdown() def test_commit_change_alerts_once_outside_release_window(tmp_path: Path) -> None: server, _thread = serve(HealthHandler) server.kind = "ok" server.commit = "a" * 40 state = tmp_path / "state" env = { "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), "JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{server.server_address[1]}/health", "JYOTISHA_HEALTH_TIMEOUT": "2", } try: assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0 assert alert_lines(state) == [] server.commit = "b" * 40 assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0 lines = alert_lines(state) assert len(lines) == 1 assert "errorCode=commit_changed" in lines[0] assert "b" * 40 not in lines[0] assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0 assert len(alert_lines(state)) == 1 marker = state / "release-window" state.mkdir(parents=True, exist_ok=True) write_text(marker, "open\n") server.commit = "c" * 40 assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0 assert len(alert_lines(state)) == 1 finally: server.shutdown() def test_disk_alerts_once_and_no_webhook_stays_local(tmp_path: Path) -> None: bin_dir = tmp_path / "bin" bin_dir.mkdir() write_text( bin_dir / "df", "#!/usr/bin/env bash\nprintf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted' \"overlay 100 90 10 ${FAKE_DISK_PERCENT:?}% /\"\n", ) subprocess.run([BASH, "-lc", 'chmod +x "$1/df"', "chmod", bash_path(bin_dir)], check=True) state = tmp_path / "state" env = { "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), "FAKE_DISK_PERCENT": "81", } env.pop("FEISHU_WEBHOOK_URL", None) env.pop("FEISHU_SIGN_SECRET", None) completed = subprocess.run( [BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)], cwd=ROOT, env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))}, capture_output=True, text=True, encoding="utf-8", errors="replace", ) assert completed.returncode == 0, completed.stderr assert "errorCode=disk_high" in alert_lines(state)[0] assert "count=81" in alert_lines(state)[0] env["FAKE_DISK_PERCENT"] = "81" again = subprocess.run( [BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)], cwd=ROOT, env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))}, capture_output=True, text=True, encoding="utf-8", errors="replace", ) assert again.returncode == 0, again.stderr assert len(alert_lines(state)) == 1 env["FAKE_DISK_PERCENT"] = "10" recovered = subprocess.run( [BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)], cwd=ROOT, env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))}, capture_output=True, text=True, encoding="utf-8", errors="replace", ) assert recovered.returncode == 0, recovered.stderr assert len(alert_lines(state)) == 2 assert "errorCode=recovered" in alert_lines(state)[1] def test_feishu_payload_keeps_only_allowed_fields(tmp_path: Path) -> None: hook, _thread = serve(HookHandler) hook.bodies = [] health, _thread = serve(HealthHandler) health.kind = "500" state = tmp_path / "state" secret = "fixture-sign-secret" env_file = tmp_path / "monitor.env" write_text(env_file, f"FEISHU_WEBHOOK_URL=http://127.0.0.1:{hook.server_address[1]}/hook\nFEISHU_SIGN_SECRET={secret}\n") env = { "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), "JYOTISHA_MONITOR_ENV": bash_path(env_file), "JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{health.server_address[1]}/health", "JYOTISHA_HEALTH_TIMEOUT": "2", } try: for _ in range(3): completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) assert completed.returncode == 0, completed.stderr assert len(hook.bodies) == 1 payload = json.loads(hook.bodies[0].decode("utf-8")) assert set(payload) == {"timestamp", "sign", "msg_type", "content"} assert payload["msg_type"] == "text" text = payload["content"]["text"] assert secret not in text assert secret not in hook.bodies[0].decode("utf-8") assert "errorCode=health_down" in text assert "@" not in text expected = subprocess.run( [ sys.executable, "-c", "import base64,hashlib,hmac,os,sys; ts=sys.argv[1]; secret=sys.argv[2]; key=f'{ts}\\n{secret}'.encode(); print(base64.b64encode(hmac.new(key, digestmod=hashlib.sha256).digest()).decode())", payload["timestamp"], secret, ], check=True, capture_output=True, text=True, ).stdout.strip() assert payload["sign"] == expected finally: hook.shutdown() health.shutdown() def uuid(index: int) -> str: return f"00000000-0000-4000-8000-{index:012d}" def test_error_digest_drops_personal_text_and_respects_thresholds(tmp_path: Path) -> None: safe = [ f"2026-10-15T00:00:{index:02d}Z web [agent-observability] {json.dumps({'requestId': uuid(index), 'errorCode': 'settlement_failed'})}" for index in range(1, 11) ] unsafe = [ "2026-10-15T00:01:00Z web person@example.com errorCode=should_drop", "2026-10-15T00:01:01Z web note 1900-01-01 birth_date", "2026-10-15T00:01:02Z web 对话内容 今天工作不顺", ] fixture = tmp_path / "logs.txt" write_text(fixture, "\n".join([*safe, *unsafe]) + "\n") state = tmp_path / "state" env = { "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), "JYOTISHA_LOG_FIXTURE": bash_path(fixture), "JYOTISHA_NOW": "2026-10-15T01:00:00Z", } below = tmp_path / "below.txt" write_text(below, "\n".join(safe[:9]) + "\n") quiet = run_bash(DIGEST_SCRIPT, {**env, "JYOTISHA_LOG_FIXTURE": bash_path(below)}, ["--window"]) assert quiet.returncode == 0, quiet.stderr assert alert_lines(state) == [] completed = run_bash(DIGEST_SCRIPT, env, ["--window"]) assert completed.returncode == 0, completed.stderr lines = alert_lines(state) assert len(lines) == 1 assert "errorCode=settlement_failed" in lines[0] assert "count=10" in lines[0] assert lines[0].count(",") == 4 assert uuid(6) not in lines[0] assert "person@example.com" not in lines[0] assert "1900-01-01" not in lines[0] assert "今天工作不顺" not in lines[0] assert "should_drop" not in lines[0] fivexx = tmp_path / "fivexx.txt" write_text(fivexx, "\n".join(f"2026-10-15T00:02:{index:02d}Z api status=502" for index in range(20)) + "\n") state_5xx = tmp_path / "state5" completed = run_bash( DIGEST_SCRIPT, {**env, "JYOTISHA_MONITOR_STATE_DIR": bash_path(state_5xx), "JYOTISHA_LOG_FIXTURE": bash_path(fivexx)}, ["--window"], ) assert completed.returncode == 0, completed.stderr joined = "\n".join(alert_lines(state_5xx)) assert "errorCode=http_5xx" in joined assert "count=20" in joined def test_daily_digest_summarizes_previous_shanghai_day(tmp_path: Path) -> None: fixture = tmp_path / "daily.txt" write_text( fixture, "\n".join( [ '2026-10-14T02:00:00Z api [agent-observability] {"requestId":"00000000-0000-4000-8000-000000000001","errorCode":"settlement_failed"}', '2026-10-15T01:30:00Z api [agent-observability] {"requestId":"00000000-0000-4000-8000-000000000002","errorCode":"settlement_failed"}', ] ) + "\n", ) state = tmp_path / "state" completed = run_bash( DIGEST_SCRIPT, { "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), "JYOTISHA_LOG_FIXTURE": bash_path(fixture), "JYOTISHA_NOW": "2026-10-15T01:00:00Z", }, ["--daily"], ) assert completed.returncode == 0, completed.stderr lines = alert_lines(state) assert len(lines) == 1 assert "count=1" in lines[0] assert "00000000-0000-4000-8000-000000000001" in lines[0] assert "00000000-0000-4000-8000-000000000002" not in lines[0] def test_find_request_log_sorts_and_rejects_injection(tmp_path: Path) -> None: log_dir = tmp_path / "logs" log_dir.mkdir() request_id = "11111111-2222-4333-8444-555555555555" write_text(log_dir / "caddy.log", f"2026-10-15T03:00:00Z caddy {request_id}\n") write_text(log_dir / "web.log", f"2026-10-15T01:00:00Z web {request_id}\nignore\n") write_text(log_dir / "api.log", f"2026-10-15T02:00:00Z api {request_id}\n") canary = tmp_path / "canary.txt" write_text(canary, "keep\n") found = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, [request_id, "--since", "24h"]) assert found.returncode == 0, found.stderr assert found.stdout.splitlines() == [ f"2026-10-15T01:00:00Z web {request_id}", f"2026-10-15T02:00:00Z api {request_id}", f"2026-10-15T03:00:00Z caddy {request_id}", ] rejected = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, ["; rm -rf /", "--since", "24h"]) assert rejected.returncode == 2 assert canary.read_text(encoding="utf-8") == "keep\n" injected = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, ["$(rm canary.txt)", "--since", "1h"]) assert injected.returncode == 2 assert canary.exists() bad_since = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, [request_id, "--since", "24h;rm"]) assert bad_since.returncode == 2 def service_logging_blocks(text: str) -> dict[str, str]: blocks: dict[str, str] = {} service: str | None = None in_services = False capturing = False buf: list[str] = [] for line in text.splitlines(): if line.startswith("services:"): in_services = True continue if in_services and line and not line.startswith(" "): in_services = False if not in_services: continue if line.startswith(" ") and not line.startswith(" ") and line.endswith(":"): if capturing and service is not None: blocks[service] = "\n".join(buf) service = line.strip()[:-1] capturing = False buf = [] continue if service and line == " logging:": capturing = True buf = [line] continue if capturing: if line.startswith(" ") and not line.startswith(" "): blocks[service] = "\n".join(buf) capturing = False buf = [] else: buf.append(line) if capturing and service is not None: blocks[service] = "\n".join(buf) return blocks def test_compose_logging_is_present_and_identical() -> None: expected = "\n".join( [ " logging:", " driver: json-file", " options:", ' max-size: "20m"', ' max-file: "10"', ] ) expected_names = { SERVER_COMPOSE: {"api", "web", "caddy"}, POSTGRES_COMPOSE: {"postgres", "migrator", "migration-checker"}, } for path, names in expected_names.items(): blocks = service_logging_blocks(path.read_text(encoding="utf-8")) assert set(blocks) == names for name in names: assert blocks[name] == expected, name def test_shell_scripts_pass_bash_n() -> None: for script in (PRUNE_SCRIPT, HEALTH_SCRIPT, DIGEST_SCRIPT, FIND_SCRIPT, NOTIFY_SCRIPT, RECOVERY_SCRIPT): completed = subprocess.run( [BASH, "-n", bash_path(script)], capture_output=True, text=True, encoding="utf-8", ) assert completed.returncode == 0, script.name + completed.stderr