root/Jyotisha
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(release): keep daily backups from stalling when the disk fills

Browse Source 
Expired daily files are pruned before a new backup, a verify older than 36 hours raises an alert, and the host monitors call sudo with the production env file. The Linux 0600 check is on the staging quick list.

Co-Authored-By: Grok (agent loop)
This commit is contained in:
jesse-ux
2026-10-15 17:24:26 +08:00
co-authored by Grok (agent loop)
parent 14897f0bf3
commit 9455fbfc4f
12 changed files with 453 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_beta_safety_net.py
+188 -29
View File
@@ -77,6 +77,30 @@ def write_text(path: Path, text: str) -> None:
path.write_text(text, encoding="utf-8", newline="\n")
def write_backup_record(directory: Path, verified_at: str) -> Path:
record = directory / "daily-backup-latest.env"
write_text(
record,
"BACKUP_BASENAME=production-daily-20261015T010000Z.dump.enc\n"
f"BACKUP_SHA256={'a' * 64}\n"
f"VERIFIED_AT={verified_at}\n"
"BACKUP_FORMAT=2\n",
)
return record
def run_disk_mode(bin_dir: Path, script_env: dict[str, str]) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)],
cwd=ROOT,
env={**os.environ, **script_env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))},
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
)
class HealthHandler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
if self.path != "/health":
@@ -111,11 +135,14 @@ class HookHandler(BaseHTTPRequestHandler):
body = self.rfile.read(length)
self.server.bodies.append(body)
self.close_connection = True
code = int(getattr(self.server, "feishu_code", 0))
reply = json.dumps({"code": code}).encode("utf-8")
self.send_response(200)
self.send_header("Content-Length", "2")
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(reply)))
self.send_header("Connection", "close")
self.end_headers()
self.wfile.write(b"ok")
self.wfile.write(reply)
def log_message(self, fmt: str, *args: object) -> None:
return
@@ -144,6 +171,9 @@ def test_daily_workflow_is_scheduled_from_main_and_reuses_recovery() -> None:
assert 'git show "$GITEA_SHA:deploy/run-production-recovery.sh"' in text
assert "RECOVERY_BASENAME_PREFIX='production-daily'" in text
assert "prune-production-daily-backups.sh" in text
prune_call = text.index("bash '$incoming/prune-production-daily-backups.sh'")
recovery_call = text.index("bash '$incoming/run-production-recovery.sh'")
assert prune_call < recovery_call
assert "production-daily-" in text
assert 'artifact_retention="${ARTIFACT_RETENTION_DAYS:-14}"' in text
assert 'local_retention="${LOCAL_RETENTION_DAYS:-7}"' in text
@@ -567,47 +597,26 @@ def test_disk_alerts_once_and_no_webhook_stays_local(tmp_path: Path) -> None:
)
subprocess.run([BASH, "-lc", 'chmod +x "$1/df"', "chmod", bash_path(bin_dir)], check=True)
state = tmp_path / "state"
record = write_backup_record(tmp_path, "2026-10-15T01:00:00Z")
env = {
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
"JYOTISHA_DAILY_BACKUP_RECORD": bash_path(record),
"JYOTISHA_NOW": "2026-10-15T01:00:00Z",
"FAKE_DISK_PERCENT": "81",
}
env.pop("FEISHU_WEBHOOK_URL", None)
env.pop("FEISHU_SIGN_SECRET", None)
completed = subprocess.run(
[BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)],
cwd=ROOT,
env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))},
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
)
completed = run_disk_mode(bin_dir, env)
assert completed.returncode == 0, completed.stderr
assert "errorCode=disk_high" in alert_lines(state)[0]
assert "count=81" in alert_lines(state)[0]
env["FAKE_DISK_PERCENT"] = "81"
again = subprocess.run(
[BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)],
cwd=ROOT,
env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))},
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
)
again = run_disk_mode(bin_dir, env)
assert again.returncode == 0, again.stderr
assert len(alert_lines(state)) == 1
env["FAKE_DISK_PERCENT"] = "10"
recovered = subprocess.run(
[BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)],
cwd=ROOT,
env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))},
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
)
recovered = run_disk_mode(bin_dir, env)
assert recovered.returncode == 0, recovered.stderr
assert len(alert_lines(state)) == 2
assert "errorCode=recovered" in alert_lines(state)[1]
@@ -622,6 +631,12 @@ def test_feishu_payload_keeps_only_allowed_fields(tmp_path: Path) -> None:
secret = "fixture-sign-secret"
env_file = tmp_path / "monitor.env"
write_text(env_file, f"FEISHU_WEBHOOK_URL=http://127.0.0.1:{hook.server_address[1]}/hook\nFEISHU_SIGN_SECRET={secret}\n")
os.chmod(env_file, 0o600)
if os.name != "nt":
assert (env_file.stat().st_mode & 0o777) == 0o600
notify = NOTIFY_SCRIPT.read_text(encoding="utf-8")
assert '[ "$mode" = "600" ]' in notify
assert "curl --silent --show-error --fail" in notify
env = {
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
"JYOTISHA_MONITOR_ENV": bash_path(env_file),
@@ -659,6 +674,35 @@ def test_feishu_payload_keeps_only_allowed_fields(tmp_path: Path) -> None:
health.shutdown()
def test_feishu_rejection_is_not_recorded_as_sent(tmp_path: Path) -> None:
hook, _thread = serve(HookHandler)
hook.bodies = []
hook.feishu_code = 19001
health, _thread = serve(HealthHandler)
health.kind = "500"
state = tmp_path / "state"
env_file = tmp_path / "monitor.env"
write_text(env_file, f"FEISHU_WEBHOOK_URL=http://127.0.0.1:{hook.server_address[1]}/hook\n")
os.chmod(env_file, 0o600)
env = {
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
"JYOTISHA_MONITOR_ENV": bash_path(env_file),
"JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{health.server_address[1]}/health",
"JYOTISHA_HEALTH_TIMEOUT": "2",
}
try:
assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0
assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0
rejected = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
assert rejected.returncode != 0
assert len(hook.bodies) == 1
assert alert_lines(state) == []
assert "health_down" not in rejected.stdout
finally:
hook.shutdown()
health.shutdown()
def uuid(index: int) -> str:
return f"00000000-0000-4000-8000-{index:012d}"
@@ -699,6 +743,13 @@ def test_error_digest_drops_personal_text_and_respects_thresholds(tmp_path: Path
assert "1900-01-01" not in lines[0]
assert "今天工作不顺" not in lines[0]
assert "should_drop" not in lines[0]
soon = run_bash(DIGEST_SCRIPT, {**env, "JYOTISHA_NOW": "2026-10-15T01:29:59Z"}, ["--window"])
assert soon.returncode == 0, soon.stderr
assert len(alert_lines(state)) == 1
later = run_bash(DIGEST_SCRIPT, {**env, "JYOTISHA_NOW": "2026-10-15T01:30:00Z"}, ["--window"])
assert later.returncode == 0, later.stderr
assert len(alert_lines(state)) == 2
assert "errorCode=settlement_failed" in alert_lines(state)[1]
fivexx = tmp_path / "fivexx.txt"
write_text(fivexx, "\n".join(f"2026-10-15T00:02:{index:02d}Z api status=502" for index in range(20)) + "\n")
@@ -769,6 +820,30 @@ def test_find_request_log_sorts_and_rejects_injection(tmp_path: Path) -> None:
assert canary.exists()
bad_since = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, [request_id, "--since", "24h;rm"])
assert bad_since.returncode == 2
compose_log = tmp_path / "compose.log"
write_text(
compose_log,
"\n".join(
[
f"web-1 | 2026-10-15T01:00:00Z web {request_id}",
f"api-1 | 2026-10-15T03:00:00Z api {request_id}",
f"caddy-1 | 2026-10-15T02:00:00Z caddy {request_id}",
f"api-1 | ignore {request_id}",
]
)
+ "\n",
)
composed = run_bash(
FIND_SCRIPT,
{"JYOTISHA_COMPOSE_LOG_FIXTURE": bash_path(compose_log)},
[request_id, "--since", "24h"],
)
assert composed.returncode == 0, composed.stderr
assert composed.stdout.splitlines() == [
f"2026-10-15T01:00:00Z web {request_id}",
f"2026-10-15T02:00:00Z caddy {request_id}",
f"2026-10-15T03:00:00Z api {request_id}",
]
def service_logging_blocks(text: str) -> dict[str, str]:
@@ -829,6 +904,90 @@ def test_compose_logging_is_present_and_identical() -> None:
assert blocks[name] == expected, name
def test_monitor_commands_use_production_docker() -> None:
for script in (DIGEST_SCRIPT, FIND_SCRIPT):
text = script.read_text(encoding="utf-8")
assert "sudo -n docker compose -p jyotisha-production" in text
assert "export DATABASE_ENV_FILE='../.env.production.database'" in text
assert "export APP_ENV_FILE='../.env.production'" in text
assert "docker-compose.server.yml" in text
assert "docker-compose.postgres.yml" in text
assert "docker-compose.production.yml" in text
for line in text.splitlines():
stripped = line.strip()
if stripped.startswith("docker ") or stripped.startswith("docker compose"):
raise AssertionError(stripped)
finder = FIND_SCRIPT.read_text(encoding="utf-8")
strip_at = finder.index("sed -E 's/^[^|]*\\| //'")
sort_at = finder.index("sort", strip_at)
assert strip_at < sort_at
def test_safety_net_is_on_the_quick_gate() -> None:
text = (ROOT / "scripts" / "run_quality_gate.py").read_text(encoding="utf-8")
assert '"tests/test_beta_safety_net.py"' in text
def install_fake_df(bin_dir: Path) -> None:
bin_dir.mkdir()
write_text(
bin_dir / "df",
"#!/usr/bin/env bash\nprintf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted' \"overlay 100 10 90 ${FAKE_DISK_PERCENT:?}% /\"\n",
)
subprocess.run([BASH, "-lc", 'chmod +x "$1/df"', "chmod", bash_path(bin_dir)], check=True)
def test_stale_daily_backup_alerts_once_until_a_fresh_record(tmp_path: Path) -> None:
bin_dir = tmp_path / "bin"
install_fake_df(bin_dir)
state = tmp_path / "state"
now = "2026-10-15T12:00:00Z"
env = {
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
"JYOTISHA_NOW": now,
"JYOTISHA_DAILY_BACKUP_RECORD": bash_path(tmp_path / "missing-record.env"),
"FAKE_DISK_PERCENT": "10",
}
missing = run_disk_mode(bin_dir, env)
assert missing.returncode == 0, missing.stderr
assert alert_lines(state) == [
line for line in alert_lines(state) if "errorCode=backup_stale" in line and "count=36" in line
]
assert len(alert_lines(state)) == 1
assert "production-daily" not in alert_lines(state)[0]
assert ".env" not in alert_lines(state)[0]
again = run_disk_mode(bin_dir, env)
assert again.returncode == 0, again.stderr
assert len(alert_lines(state)) == 1
exact = tmp_path / "exact"
exact.mkdir()
env["JYOTISHA_MONITOR_STATE_DIR"] = bash_path(exact)
env["JYOTISHA_DAILY_BACKUP_RECORD"] = bash_path(write_backup_record(exact, "2026-10-14T00:00:00Z"))
boundary = run_disk_mode(bin_dir, env)
assert boundary.returncode == 0, boundary.stderr
assert alert_lines(exact) == []
stale_state = tmp_path / "stale-state"
old_dir = tmp_path / "old"
old_dir.mkdir()
env["JYOTISHA_MONITOR_STATE_DIR"] = bash_path(stale_state)
env["JYOTISHA_DAILY_BACKUP_RECORD"] = bash_path(write_backup_record(old_dir, "2026-10-13T23:00:00Z"))
old = run_disk_mode(bin_dir, env)
assert old.returncode == 0, old.stderr
assert len(alert_lines(stale_state)) == 1
assert "errorCode=backup_stale" in alert_lines(stale_state)[0]
assert "count=37" in alert_lines(stale_state)[0]
fresh_dir = tmp_path / "fresh"
fresh_dir.mkdir()
env["JYOTISHA_DAILY_BACKUP_RECORD"] = bash_path(write_backup_record(fresh_dir, now))
recovered = run_disk_mode(bin_dir, env)
assert recovered.returncode == 0, recovered.stderr
assert len(alert_lines(stale_state)) == 2
assert "errorCode=recovered" in alert_lines(stale_state)[1]
def test_shell_scripts_pass_bash_n() -> None:
for script in (PRUNE_SCRIPT, HEALTH_SCRIPT, DIGEST_SCRIPT, FIND_SCRIPT, NOTIFY_SCRIPT, RECOVERY_SCRIPT):
completed = subprocess.run(