Jesse_Chen
381 lines
19 KiB
TypeScript
381 lines
19 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { cpSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import test from "node:test";
|
|
|
|
import { handleAdminModelMutation } from "../src/lib/admin/model-mutation-handler.ts";
|
|
import { assertAllowedModelProviderUrl } from "../src/lib/epay/gateway-policy.ts";
|
|
import { runMigrations } from "../scripts/db-migrate.mjs";
|
|
import { startPostgresFixture } from "./helpers/postgres-fixture.ts";
|
|
|
|
const migrationPath = fileURLToPath(new URL("../supabase/migrations/20260806040000_model_configuration.sql", import.meta.url));
|
|
const routePath = fileURLToPath(new URL("../src/app/api/admin/models/route.ts", import.meta.url));
|
|
const catalogPath = fileURLToPath(new URL("../src/lib/model-catalog.ts", import.meta.url));
|
|
const componentPath = fileURLToPath(new URL("../src/components/admin/model-management.tsx", import.meta.url));
|
|
const actorId = "10000000-0000-4000-8000-000000000003";
|
|
const sessionOneId = "20000000-0000-4000-8000-000000000001";
|
|
const sessionTwoId = "20000000-0000-4000-8000-000000000002";
|
|
|
|
const publicLookup = async () => [{ address: "93.184.216.34", family: 4 }] as const;
|
|
|
|
async function migrateModelConfigurationFixture(connectionString: string) {
|
|
const root = fileURLToPath(new URL("..", import.meta.url));
|
|
const temporaryRoot = mkdtempSync(join(tmpdir(), "jyotisha-model-migrations-"));
|
|
const directories = ["db/migrations", "supabase/migrations"].map((relative) => {
|
|
const source = join(root, relative);
|
|
const target = join(temporaryRoot, relative.replace("/", "-"));
|
|
mkdirSync(target, { recursive: true });
|
|
for (const filename of readdirSync(source)) {
|
|
if (!filename.endsWith(".sql") || filename > "20260806040000_model_configuration.sql") continue;
|
|
if (filename === "20260806030000_settle_order_usage_authorization.sql") continue;
|
|
cpSync(join(source, filename), join(target, filename));
|
|
}
|
|
return target;
|
|
});
|
|
try {
|
|
await runMigrations({
|
|
connectionString,
|
|
migrationsDirectory: undefined,
|
|
migrationsDirectories: directories,
|
|
logger: console,
|
|
});
|
|
} finally {
|
|
rmSync(temporaryRoot, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
test("model provider URLs require a server-owned public allowlist", async () => {
|
|
await assertAllowedModelProviderUrl(
|
|
"https://models.example.com/v1",
|
|
{ MODEL_PROVIDER_BASE_URL_ALLOWLIST: "https://models.example.com" },
|
|
publicLookup,
|
|
);
|
|
await assert.rejects(
|
|
assertAllowedModelProviderUrl("https://attacker.example/v1", {}, publicLookup),
|
|
/允许列表/,
|
|
);
|
|
await assert.rejects(
|
|
assertAllowedModelProviderUrl(
|
|
"https://models.example.com/v1",
|
|
{ MODEL_PROVIDER_BASE_URL_ALLOWLIST: "https://models.example.com" },
|
|
async () => [{ address: "169.254.169.254", family: 4 }],
|
|
),
|
|
/内网|保留地址/,
|
|
);
|
|
await assert.rejects(
|
|
assertAllowedModelProviderUrl(
|
|
"https://service.internal/v1",
|
|
{ MODEL_PROVIDER_BASE_URL_ALLOWLIST: "https://service.internal" },
|
|
publicLookup,
|
|
),
|
|
/内部域名/,
|
|
);
|
|
});
|
|
|
|
test("admin and runtime source keep refs private and test exact model versions", () => {
|
|
const route = readFileSync(routePath, "utf8");
|
|
const catalog = readFileSync(catalogPath, "utf8");
|
|
const component = readFileSync(componentPath, "utf8");
|
|
assert.doesNotMatch(route, /secretRef:\s*provider\.secret_ref,\s*enabled:/);
|
|
assert.doesNotMatch(component, /name="secretRef"|providerId:\s*item\.id/);
|
|
assert.match(component, /action: "test", versionId: item\.id/);
|
|
assert.match(route, /sanitizeModelSettings\(row\.settings\)/);
|
|
assert.match(route, /requireHighRiskAdminMutation/);
|
|
assert.match(route, /handleAdminModelMutation/);
|
|
assert.match(catalog, /models\.database_catalog/);
|
|
assert.match(catalog, /models\.circuit_breaker/);
|
|
assert.match(catalog, /assertAllowedModelProviderUrl/);
|
|
assert.match(catalog, /resolveSessionLanguageModel/);
|
|
});
|
|
|
|
test("mutation handler rejects arbitrary env refs and persists failed connection evidence", async () => {
|
|
let queryCount = 0;
|
|
const rejected = await handleAdminModelMutation({
|
|
action: "saveProvider",
|
|
code: "openai",
|
|
name: "OpenAI",
|
|
providerType: "openai",
|
|
secretRef: "env:DATABASE_URL",
|
|
enabled: true,
|
|
reason: "unsafe ref",
|
|
}, { actorUserId: actorId, requestId: "provider-dangerous-ref" }, {
|
|
queryRows: async () => { queryCount += 1; return []; },
|
|
assertAllowedUrl: async () => undefined,
|
|
probeAllowed: async () => 200,
|
|
invalidateCatalog: () => undefined,
|
|
environment: { OPENAI_API_KEY: "model-key" },
|
|
});
|
|
assert.equal(rejected.status, 400);
|
|
assert.equal(queryCount, 0);
|
|
|
|
const calls: Array<{ sql: string; values?: readonly unknown[] }> = [];
|
|
const failed = await handleAdminModelMutation({
|
|
action: "test",
|
|
versionId: "30000000-0000-4000-8000-000000000001",
|
|
}, { actorUserId: actorId, requestId: "test-unauthorized" }, {
|
|
queryRows: async (sql, values) => {
|
|
calls.push({ sql, values });
|
|
if (sql.includes("from public.model_config_versions")) {
|
|
return [{
|
|
id: "40000000-0000-4000-8000-000000000001",
|
|
code: "openai",
|
|
provider_type: "openai",
|
|
base_url: null,
|
|
secret_ref: "env:OPENAI_API_KEY",
|
|
enabled: true,
|
|
version_id: "30000000-0000-4000-8000-000000000001",
|
|
version_enabled: true,
|
|
}];
|
|
}
|
|
return [{ id: "50000000-0000-4000-8000-000000000001" }];
|
|
},
|
|
assertAllowedUrl: async () => undefined,
|
|
probeAllowed: async () => 401,
|
|
invalidateCatalog: () => undefined,
|
|
environment: { OPENAI_API_KEY: "model-key" },
|
|
});
|
|
assert.equal(failed.status, 409);
|
|
assert.equal(calls.length, 2);
|
|
assert.match(calls[1]!.sql, /admin_record_model_connection_test/);
|
|
assert.deepEqual(calls[1]!.values, [actorId, "30000000-0000-4000-8000-000000000001", 401, "test-unauthorized"]);
|
|
});
|
|
|
|
test("provider mutation returns a clear immutable-version conflict", async () => {
|
|
const response = await handleAdminModelMutation({
|
|
action: "saveProvider",
|
|
id: "40000000-0000-4000-8000-000000000001",
|
|
code: "openai-next",
|
|
name: "OpenAI Next",
|
|
providerType: "openai",
|
|
enabled: true,
|
|
reason: "rotate provider",
|
|
}, { actorUserId: actorId, requestId: "provider-immutable" }, {
|
|
queryRows: async () => {
|
|
throw Object.assign(new Error("model_provider_runtime_immutable"), { code: "23514" });
|
|
},
|
|
assertAllowedUrl: async () => undefined,
|
|
probeAllowed: async () => 200,
|
|
invalidateCatalog: () => undefined,
|
|
environment: { OPENAI_API_KEY: "model-key" },
|
|
});
|
|
|
|
assert.equal(response.status, 409);
|
|
assert.deepEqual(await response.json(), {
|
|
error: "已发布或已退役版本使用的供应商连接配置不可修改，请新建供应商和模型版本后重新测试并发布",
|
|
code: "model_provider_runtime_immutable",
|
|
});
|
|
});
|
|
|
|
test("rollback handler probes and records same-request evidence before changing publication", async () => {
|
|
const events: string[] = [];
|
|
const providerRow = {
|
|
id: "40000000-0000-4000-8000-000000000001",
|
|
code: "openai",
|
|
provider_type: "openai" as const,
|
|
base_url: null,
|
|
secret_ref: "env:OPENAI_API_KEY",
|
|
enabled: true,
|
|
version_id: "30000000-0000-4000-8000-000000000001",
|
|
version_enabled: true,
|
|
};
|
|
const dependencies = (status: number) => ({
|
|
queryRows: async (sql: string, values?: readonly unknown[]) => {
|
|
if (sql.includes("from public.model_config_versions")) {
|
|
events.push("lookup");
|
|
return [providerRow];
|
|
}
|
|
if (sql.includes("admin_record_model_connection_test")) {
|
|
events.push(`evidence:${String(values?.[3])}`);
|
|
return [{ id: "50000000-0000-4000-8000-000000000001" }];
|
|
}
|
|
if (sql.includes("admin_rollback_model")) {
|
|
events.push("rollback");
|
|
return [{ id: providerRow.version_id }];
|
|
}
|
|
return [];
|
|
},
|
|
assertAllowedUrl: async () => undefined,
|
|
probeAllowed: async () => { events.push(`probe:${status}`); return status; },
|
|
invalidateCatalog: () => events.push("invalidate"),
|
|
environment: { OPENAI_API_KEY: "model-key" },
|
|
});
|
|
|
|
const action = {
|
|
action: "rollback" as const,
|
|
configId: "60000000-0000-4000-8000-000000000001",
|
|
targetVersion: 1,
|
|
reason: "runtime rollback",
|
|
};
|
|
const failed = await handleAdminModelMutation(
|
|
action,
|
|
{ actorUserId: actorId, requestId: "rollback-runtime-check" },
|
|
dependencies(403),
|
|
);
|
|
assert.equal(failed.status, 409);
|
|
assert.deepEqual(events, ["lookup", "probe:403", "evidence:rollback-runtime-check"]);
|
|
|
|
events.length = 0;
|
|
const passed = await handleAdminModelMutation(
|
|
action,
|
|
{ actorUserId: actorId, requestId: "rollback-runtime-check" },
|
|
dependencies(200),
|
|
);
|
|
assert.equal(passed.status, 200);
|
|
assert.deepEqual(events, ["lookup", "probe:200", "evidence:rollback-runtime-check", "rollback", "invalidate"]);
|
|
});
|
|
|
|
test("database enforces fixed secrets, fresh evidence, rollback viability, and session version pinning", async () => {
|
|
const fixture = startPostgresFixture();
|
|
const sql = (statement: string) => fixture.psql(statement);
|
|
const sqlAsOwner = (statement: string) => fixture.psqlAs("schema_owner", "schema-owner-test-password", statement);
|
|
const expectSqlError = (statement: string, pattern: RegExp) => assert.throws(() => sqlAsOwner(statement), pattern);
|
|
|
|
try {
|
|
await migrateModelConfigurationFixture(
|
|
fixture.connectionUrl("schema_owner", "schema-owner-test-password"),
|
|
);
|
|
|
|
assert.equal(sql("select public.expected_model_provider_secret_ref('openai','openai')"), "env:OPENAI_API_KEY");
|
|
assert.equal(sql("select public.expected_model_provider_secret_ref('deepseek','openai-compatible')"), "env:DEEPSEEK_API_KEY");
|
|
assert.equal(sql("select public.expected_model_provider_secret_ref('trusted-edge','openai-compatible')"), "env:MODEL_PROVIDER_TRUSTED_EDGE_API_KEY");
|
|
assert.equal(sql("select public.model_provider_base_url_is_safe('https://api.example.com/v1')"), "t");
|
|
assert.equal(sql("select public.model_provider_base_url_is_safe('https://127.0.0.1/v1')"), "f");
|
|
assert.equal(sql("select public.model_provider_base_url_is_safe('https://metadata.google.internal/v1')"), "f");
|
|
assert.equal(sql("select public.model_settings_contain_secrets('{\"nested\":{\"authorization\":\"Bearer nope\"}}')"), "t");
|
|
|
|
fixture.psqlAs("identity_runtime", "identity-runtime-test-password", `
|
|
insert into identity.users (id,name,email,email_verified,email_verified_at,role)
|
|
values ('${actorId}','Model Admin','model-admin@example.com',true,now(),'user')
|
|
`);
|
|
sql(`
|
|
insert into public.admin_users(user_id,created_by) values ('${actorId}','${actorId}');
|
|
insert into public.admin_user_roles(admin_user_id,role_id,assigned_by)
|
|
select '${actorId}',id,'${actorId}' from public.admin_roles where code='model_admin'
|
|
`);
|
|
|
|
for (const [index, secretRef] of ["env:DATABASE_URL", "env:ADMIN_DATABASE_URL", "env:SUPABASE_SERVICE_ROLE_KEY", "env:RESEND_API_KEY", "env:EPAY_KEY"].entries()) {
|
|
expectSqlError(
|
|
`select public.admin_save_model_provider('${actorId}',null,'openai','OpenAI','openai',null,'${secretRef}',false,'reject dangerous ref','danger-${index}')`,
|
|
/model_provider_secret_ref_forbidden/,
|
|
);
|
|
}
|
|
expectSqlError(
|
|
`select public.admin_save_model_provider('${actorId}',null,'unsafe-provider','Unsafe','openai-compatible','https://127.0.0.1/v1','env:MODEL_PROVIDER_UNSAFE_PROVIDER_API_KEY',false,'reject unsafe','unsafe-provider')`,
|
|
/model_provider_url_unsafe/,
|
|
);
|
|
|
|
const provider = sql(`select public.admin_save_model_provider('${actorId}',null,'openai','OpenAI','openai',null,'env:OPENAI_API_KEY',true,'create provider','provider-create')`);
|
|
expectSqlError(
|
|
`select public.admin_save_model_draft('${actorId}','default-model',null,'${provider}','Default','', 'model-v1','standard',1,64000,0,0,false,true,null,'{}','disabled default','disabled-default')`,
|
|
/default_model_disabled/,
|
|
);
|
|
expectSqlError(
|
|
`select public.admin_save_model_draft('${actorId}','secret-model',null,'${provider}','Secret','', 'model-v1','standard',1,64000,0,0,true,false,null,'{\"nested\":{\"apiKey\":\"nope\"}}','secret settings','secret-settings')`,
|
|
/model_settings_secret_forbidden/,
|
|
);
|
|
|
|
const v1 = sql(`select public.admin_save_model_draft('${actorId}','default-model',null,'${provider}','Default v1','', 'model-v1','standard',1,64000,0,0,true,true,null,'{}','create v1','draft-v1')`);
|
|
expectSqlError(
|
|
`select public.admin_publish_model('${actorId}','${v1}','untested publish','publish-untested')`,
|
|
/model_connection_test_required/,
|
|
);
|
|
sql(`select public.admin_record_model_connection_test('${actorId}','${v1}',401,'test-v1-401')`);
|
|
expectSqlError(
|
|
`select public.admin_publish_model('${actorId}','${v1}','401 publish','publish-401')`,
|
|
/model_connection_test_required/,
|
|
);
|
|
|
|
sql(`select public.admin_record_model_connection_test('${actorId}','${v1}',200,'test-v1-before-change')`);
|
|
sql(`update public.model_config_versions set label='Default v1 changed' where id='${v1}'`);
|
|
expectSqlError(
|
|
`select public.admin_publish_model('${actorId}','${v1}','changed config publish','publish-changed')`,
|
|
/model_connection_test_required/,
|
|
);
|
|
|
|
const expiringEvidence = sql(`select public.admin_record_model_connection_test('${actorId}','${v1}',200,'test-v1-expired')`);
|
|
sql(`update public.model_connection_test_evidence set tested_at=clock_timestamp()-interval '20 minutes',expires_at=clock_timestamp()-interval '10 minutes' where id='${expiringEvidence}'`);
|
|
expectSqlError(
|
|
`select public.admin_publish_model('${actorId}','${v1}','expired publish','publish-expired')`,
|
|
/model_connection_test_required/,
|
|
);
|
|
|
|
sql(`select public.admin_record_model_connection_test('${actorId}','${v1}',200,'test-v1-fresh')`);
|
|
sql(`select public.admin_save_model_provider('${actorId}','${provider}','openai','OpenAI renamed','openai',null,'env:OPENAI_API_KEY',true,'rename provider','provider-rename')`);
|
|
assert.equal(sql(`select public.model_connection_test_is_fresh('${v1}',null)`), "t");
|
|
sql(`select public.admin_publish_model('${actorId}','${v1}','publish v1','publish-v1')`);
|
|
sql(`insert into public.chat_sessions(id,user_id,title,theme,messages,model_id) values('${sessionOneId}','${actorId}','v1 session','general','[]','default-model')`);
|
|
assert.equal(sql(`select model_id||':'||model_config_version from public.chat_sessions where id='${sessionOneId}'`), "default-model:1");
|
|
|
|
for (const mutation of [
|
|
"code='openai-mutated'",
|
|
"provider_type='openai-compatible'",
|
|
"base_url='https://models.example.com/v1'",
|
|
"secret_ref='env:MODEL_PROVIDER_OPENAI_MUTATED_API_KEY'",
|
|
"enabled=false",
|
|
]) {
|
|
expectSqlError(
|
|
`update public.model_providers set ${mutation} where id='${provider}'`,
|
|
/model_provider_runtime_immutable/,
|
|
);
|
|
}
|
|
expectSqlError(
|
|
`select public.admin_save_model_provider('${actorId}','${provider}','openai','OpenAI','openai',null,'env:OPENAI_API_KEY',false,'disable published provider','provider-disable')`,
|
|
/model_provider_runtime_immutable/,
|
|
);
|
|
|
|
const providerV2 = sql(`select public.admin_save_model_provider('${actorId}',null,'openai-next','OpenAI Next','openai',null,'env:OPENAI_API_KEY',true,'create replacement provider','provider-v2')`);
|
|
const v2 = sql(`select public.admin_save_model_draft('${actorId}','default-model',null,'${providerV2}','Default v2','', 'model-v2','standard',1,64000,0,0,true,true,null,'{}','create v2','draft-v2')`);
|
|
sql(`select public.admin_record_model_connection_test('${actorId}','${v2}',403,'test-v2-403')`);
|
|
expectSqlError(
|
|
`select public.admin_publish_model('${actorId}','${v2}','403 publish','publish-v2-403')`,
|
|
/model_connection_test_required/,
|
|
);
|
|
sql(`select public.admin_record_model_connection_test('${actorId}','${v2}',204,'test-v2-fresh')`);
|
|
sql(`select public.admin_publish_model('${actorId}','${v2}','publish v2','publish-v2')`);
|
|
expectSqlError(
|
|
`update public.model_providers set enabled=false where id='${provider}'`,
|
|
/model_provider_runtime_immutable/,
|
|
);
|
|
|
|
sql(`update public.chat_sessions set messages='[{\"role\":\"user\",\"content\":\"hello\"}]',model_id='default-model',model_config_version=999 where id='${sessionOneId}'`);
|
|
assert.equal(sql(`select model_config_version from public.chat_sessions where id='${sessionOneId}'`), "1");
|
|
assert.equal(sql(`select p.code||':'||coalesce(p.base_url,'native') from public.chat_sessions s join public.model_configs c on c.model_id=s.model_id join public.model_config_versions v on v.config_id=c.id and v.version=s.model_config_version join public.model_providers p on p.id=v.provider_id where s.id='${sessionOneId}'`), "openai:native");
|
|
sql(`insert into public.chat_sessions(id,user_id,title,theme,messages,model_id) values('${sessionTwoId}','${actorId}','v2 session','general','[]','default-model')`);
|
|
assert.equal(sql(`select model_config_version from public.chat_sessions where id='${sessionTwoId}'`), "2");
|
|
assert.equal(sql(`select p.code from public.chat_sessions s join public.model_configs c on c.model_id=s.model_id join public.model_config_versions v on v.config_id=c.id and v.version=s.model_config_version join public.model_providers p on p.id=v.provider_id where s.id='${sessionTwoId}'`), "openai-next");
|
|
assert.equal(sql("select has_column_privilege('authenticated','public.chat_sessions','model_config_version','INSERT')"), "f");
|
|
assert.equal(sql("select has_column_privilege('authenticated','public.chat_sessions','model_config_version','UPDATE')"), "f");
|
|
|
|
const configId = sql("select id from public.model_configs where model_id='default-model'");
|
|
expectSqlError(
|
|
`select public.admin_rollback_model('${actorId}','${configId}',1,'rollback without same request evidence','rollback-v1')`,
|
|
/model_connection_test_required/,
|
|
);
|
|
sql(`select public.admin_record_model_connection_test('${actorId}','${v1}',403,'rollback-v1')`);
|
|
expectSqlError(
|
|
`select public.admin_rollback_model('${actorId}','${configId}',1,'rollback with failed evidence','rollback-v1')`,
|
|
/model_connection_test_required/,
|
|
);
|
|
sql(`select public.admin_record_model_connection_test('${actorId}','${v1}',200,'rollback-v1')`);
|
|
sql(`select public.admin_rollback_model('${actorId}','${configId}',1,'rollback with fresh evidence','rollback-v1')`);
|
|
assert.equal(sql(`select version from public.model_config_versions where config_id='${configId}' and status='published'`), "1");
|
|
assert.equal(sql(`select model_config_version from public.chat_sessions where id='${sessionTwoId}'`), "2");
|
|
} finally {
|
|
fixture.stop();
|
|
}
|
|
});
|
|
|
|
test("migration contains database-enforced secret, evidence, and version-pin contracts", () => {
|
|
const migration = readFileSync(migrationPath, "utf8");
|
|
assert.match(migration, /expected_model_provider_secret_ref/);
|
|
assert.match(migration, /model_connection_test_evidence/);
|
|
assert.match(migration, /model_version_config_hash/);
|
|
assert.match(migration, /model_connection_test_is_fresh/);
|
|
assert.match(migration, /pin_chat_session_model_config_version/);
|
|
assert.match(migration, /model_connection_test_required/);
|
|
assert.match(migration, /model_provider_runtime_immutable/);
|
|
assert.match(migration, /where p\.id=v_draft\.provider_id[\s\S]*for update/);
|
|
});
|