root/Jyotisha
1
0
Fork 0
Code Issues 5 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

docs(rectification): record v3 sealed holdout as evaluated not missing

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Jesse_Chen
2026-08-29 00:25:26 +08:00
parent fe0afdaddb
commit 9d80498316
4 changed files with 60 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/tests/rectification-confirmation-gate.test.ts
+29 -9
View File
@@ -33,11 +33,17 @@ import {
const THIRD_CANDIDATE_ID = "88888888-8888-4888-8888-888888888883";
const holdoutReport = JSON.parse(readFileSync(
new URL("../../references/real_case_calibration/minute_rectification_holdout_v2_pilot_report.json", import.meta.url),
new URL("../../references/real_case_calibration/minute_rectification_holdout_v3_report.json", import.meta.url),
"utf8",
)) as {
case_gate: { valid_public_aa_cases: number; minimum_public_aa_cases: number };
metrics: { top_1_rate: number; confirmation_coverage_rate: number };
benchmark_id: string;
validation: { valid_public_aa_cases: number; minimum_public_aa_cases: number };
release_metrics: {
top_1_rate: { actual: number };
top_3_rate: { actual: number };
mean_absolute_minute_error: { actual: number };
confirmation_coverage_rate: number;
};
};
const skill = readFileSync(
new URL("../../skills/jyotish-birth-time-rectification/SKILL.md", import.meta.url),
@@ -80,19 +86,33 @@ function blocker(
return found;
}
test("sealed holdout aggregates match the v2 report and stay below the case gate", () => {
test("sealed holdout aggregates match the v3 report and still block confirmation", () => {
const productHoldout = JSON.parse(readFileSync(
new URL("../../references/rectification_sealed_holdout.v1.json", import.meta.url),
"utf8",
)) as typeof SEALED_MINUTE_HOLDOUT;
assert.equal(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases, holdoutReport.case_gate.valid_public_aa_cases);
assert.equal(SEALED_MINUTE_HOLDOUT.required_cases, holdoutReport.case_gate.minimum_public_aa_cases);
assert.equal(SEALED_MINUTE_HOLDOUT.top_1_rate, holdoutReport.metrics.top_1_rate);
assert.equal(SEALED_MINUTE_HOLDOUT.confirmation_coverage_rate, holdoutReport.metrics.confirmation_coverage_rate);
assert.equal(SEALED_MINUTE_HOLDOUT.sealed_benchmark_id, holdoutReport.benchmark_id);
assert.equal(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases, holdoutReport.validation.valid_public_aa_cases);
assert.equal(SEALED_MINUTE_HOLDOUT.required_cases, holdoutReport.validation.minimum_public_aa_cases);
assert.equal(SEALED_MINUTE_HOLDOUT.top_1_rate, holdoutReport.release_metrics.top_1_rate.actual);
assert.equal(
SEALED_MINUTE_HOLDOUT.confirmation_coverage_rate,
holdoutReport.release_metrics.confirmation_coverage_rate,
);
assert.equal(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases, productHoldout.valid_public_aa_cases);
assert.equal(SEALED_MINUTE_HOLDOUT.status, productHoldout.status);
assert.ok(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases < SEALED_MINUTE_HOLDOUT.required_cases);
assert.ok(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases >= SEALED_MINUTE_HOLDOUT.required_cases);
assert.equal(SEALED_MINUTE_HOLDOUT.status, "not_ready");
assert.equal(SEALED_MINUTE_HOLDOUT.confirmation_coverage_rate, 0);
const gate = buildConfirmationGate({
engineConfirmationAllowed: true,
candidates: UNIQUE_MINUTE,
decisionReceipt: {
gates: { exact_confirmation: { external_validation_status: "passed" } },
},
});
assert.equal(gate.confirmation_allowed, false);
assert.equal(blocker(gate, "public_aa_holdout").status, "not_ready");
});
test("VedAstro not_evaluated is not described as fail and still forbids confirmation", () => {
progress.md
+7
View File
@@ -1023,3 +1023,10 @@
- main/staging 同步后，migration run 30324787560 成功：应用 `20260727000000_admin_viewer_identity.sql` 与 `20260727010000_refine_admin_redemption_audit.sql`，并自动 dispatch deploy run 30324940917；该 deploy 成功，health exact SHA 为 `f950bfd...`。
- 线上 `/admin` 冒烟发现反向代理 Location 错用容器 URL `https://0.0.0.0:3000/admin/codes`；改为相对 `/admin/codes`，形成最终 SHA `5206fb2323c87c877e5d361dc64c77d0497e14dd`，同步 fast-forward 到 main/staging。
- 最终 SHA gate run 30325219367 success，自动 deploy run 30325788303 success；health 返回同一 SHA，`/admin` 307 到相对 `/admin/codes`，Refine shell 200，5 个匿名资源与匿名写请求均 401。未触发 production deploy。
## 2026-08-29 - C1: 密封 holdout 合同回写 v3 已评估结果，不是缺案例
- `references/rectification_sealed_holdout.v1.json` 此前仍是 v2 Obama 单例试点：`valid_public_aa_cases=1` / `required_cases=20` / `top_1_rate=0`。该 1/20 从未在 v3 的 20 例盲测回放后被覆盖，会让读者误以为“还缺案例”。
- 真实状态来自 `minute_rectification_holdout_v3.json` 与 `minute_rectification_holdout_v3_report.json`（evaluated_on 2026-07-21）：20 个公开 AA 案例已存在；source audit 失败（`invalidated_after_replay`）；`top_1_rate=0.15`、`top_3_rate=0.25`、`mean_absolute_minute_error=6.95` 均未过门；`confirmation_coverage_rate=0.0`。
- 合同六键已改为 v3 数字，`status` 仍为 `not_ready`。确认门保持关闭：案例数已满，但 status 非 ready 且 confirmation_coverage_rate 为 0。
- 防复发：holdout 集一旦回放，必须把真实 status / 案例数 / 指标写回 `rectification_sealed_holdout.v1.json`；不得把合同留在上一版的计数上。
references/rectification_sealed_holdout.v1.json
+13 -4
View File
@@ -1,8 +1,17 @@
{
"sealed_benchmark_id": "minute_rectification_holdout_v2",
"sealed_benchmark_id": "minute_rectification_fact_ranker_v4_holdout_v3",
"status": "not_ready",
"valid_public_aa_cases": 1,
"valid_public_aa_cases": 20,
"required_cases": 20,
"top_1_rate": 0,
"confirmation_coverage_rate": 0
"top_1_rate": 0.15,
"confirmation_coverage_rate": 0.0,
"previous_pilot_id": "minute_rectification_holdout_v2",
"dataset_path": "references/real_case_calibration/minute_rectification_holdout_v3.json",
"report_path": "references/real_case_calibration/minute_rectification_holdout_v3_report.json",
"source_audit_status": "invalidated_after_replay",
"evaluated_on": "2026-07-21",
"report_status": "invalidated_after_source_audit",
"trial_count": 20,
"top_3_rate": 0.25,
"mean_absolute_minute_error": 6.95
}
tests/test_rectification_confirmation_and.py
+11 -7
View File
@@ -10,13 +10,15 @@ from scripts.rectification.decision_policy import (
indistinguishable_width_minutes,
)
from scripts.rectification.sealed_holdout import (
PILOT_REPORT_PATH,
holdout_passed,
load_sealed_minute_holdout,
)
ROOT = Path(__file__).resolve().parents[1]
V3_REPORT_PATH = (
ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v3_report.json"
)
def _request(*, extra_events=()):
@@ -78,16 +80,18 @@ def _rows():
]
def test_sealed_holdout_contract_matches_v2_pilot_and_stays_closed() -> None:
def test_sealed_holdout_contract_matches_v3_report_and_stays_closed() -> None:
holdout = load_sealed_minute_holdout()
report = json.loads(PILOT_REPORT_PATH.read_text(encoding="utf-8"))
report = json.loads(V3_REPORT_PATH.read_text(encoding="utf-8"))
product = json.loads((ROOT / "references" / "rectification_sealed_holdout.v1.json").read_text(encoding="utf-8"))
assert holdout["sealed_benchmark_id"] == report["benchmark_id"] == product["sealed_benchmark_id"]
assert holdout["valid_public_aa_cases"] == report["case_gate"]["valid_public_aa_cases"]
assert holdout["required_cases"] == report["case_gate"]["minimum_public_aa_cases"]
assert holdout["top_1_rate"] == report["metrics"]["top_1_rate"]
assert holdout["confirmation_coverage_rate"] == report["metrics"]["confirmation_coverage_rate"]
assert holdout["valid_public_aa_cases"] == report["validation"]["valid_public_aa_cases"]
assert holdout["required_cases"] == report["validation"]["minimum_public_aa_cases"]
assert holdout["top_1_rate"] == report["release_metrics"]["top_1_rate"]["actual"]
assert holdout["confirmation_coverage_rate"] == report["release_metrics"]["confirmation_coverage_rate"]
assert holdout["status"] == "not_ready"
assert holdout["valid_public_aa_cases"] >= holdout["required_cases"]
assert holdout["confirmation_coverage_rate"] == 0.0
assert holdout_passed(holdout) is False