import assert from "node:assert/strict"; import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; import { splitAfterSentencePunctuation, type SentenceSplitOptions, } from "../src/lib/sentence-split.ts"; const sourceRoot = fileURLToPath(new URL("../src/", import.meta.url)); function walk(dir: string): string[] { const files: string[] = []; for (const entry of readdirSync(dir, { withFileTypes: true })) { const path = join(dir, entry.name); if (entry.isDirectory()) files.push(...walk(path)); else files.push(path); } return files; } test("frontend/src has no lookbehind assertions", () => { const textExtensions = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".css", ".json", ".md"]); const hits: string[] = []; for (const file of walk(sourceRoot)) { const dot = file.lastIndexOf("."); if (!textExtensions.has(file.slice(dot))) continue; const source = readFileSync(file, "utf8"); if (source.includes("(?<=")) hits.push(file); } assert.deepEqual(hits, []); }); type HabitCase = { habit: string; input: string; punctuation: string; options?: SentenceSplitOptions; expected: string[]; }; const habits: HabitCase[] = [ { habit: "chinese-period", input: "你好。世界", punctuation: "。!?", expected: ["你好。", "世界"] }, { habit: "chinese-period", input: "甲。乙。丙", punctuation: "。!?", expected: ["甲。", "乙。", "丙"] }, { habit: "question", input: "你好?世界", punctuation: "。!?", expected: ["你好?", "世界"] }, { habit: "question", input: "真的吗?是的。", punctuation: "。!?", expected: ["真的吗?", "是的。"] }, { habit: "exclamation", input: "你好!世界", punctuation: "。!?", expected: ["你好!", "世界"] }, { habit: "exclamation", input: "停!别走。", punctuation: "。!?", expected: ["停!", "别走。"] }, { habit: "semicolon", input: "你好;世界", punctuation: "。!?;;", options: { consumeFollowingWhitespace: true }, expected: ["你好;", "世界"], }, { habit: "semicolon", input: "甲;乙;丙", punctuation: "。!?;;", options: { consumeFollowingWhitespace: true }, expected: ["甲;", "乙;", "丙"], }, { habit: "english-punctuation", input: "Hello! World", punctuation: "。!?!?.", expected: ["Hello!", " World"], }, { habit: "english-punctuation", input: "Hello. World", punctuation: "。!?!?.", expected: ["Hello.", " World"], }, { habit: "newline", input: "你好\n世界", punctuation: "。!?!?;;", options: { consumeFollowingWhitespace: true, splitOnNewlines: true }, expected: ["你好", "世界"], }, { habit: "newline", input: "甲。\n乙", punctuation: "。!?!?;;", options: { consumeFollowingWhitespace: true, splitOnNewlines: true }, expected: ["甲。", "乙"], }, { habit: "repeated-punctuation", input: "你好。。世界", punctuation: "。!?", expected: ["你好。", "。", "世界"] }, { habit: "repeated-punctuation", input: "啊!!停", punctuation: "。!?!?", expected: ["啊!", "!", "停"] }, { habit: "no-trailing-punctuation", input: "没有标点的一段", punctuation: "。!?", expected: ["没有标点的一段"] }, { habit: "no-trailing-punctuation", input: "结尾无句号就这样", punctuation: "。!?", expected: ["结尾无句号就这样"] }, ]; test("sentence splits keep punctuation and match each call-site habit", () => { const seen = new Map(); for (const sample of habits) { seen.set(sample.habit, (seen.get(sample.habit) ?? 0) + 1); assert.deepEqual( splitAfterSentencePunctuation(sample.input, sample.punctuation, sample.options), sample.expected, sample.habit, ); } for (const [habit, count] of seen) { assert.ok(count >= 2, `${habit} needs two cases, has ${count}`); } }); test("sentence splits stay character-identical to the removed lookbehind patterns", () => { const presets: Array<{ punctuation: string; options?: SentenceSplitOptions; legacy: RegExp; }> = [ { punctuation: "。!?", legacy: /(?<=[。!?])/ }, { punctuation: "。!?;;", options: { consumeFollowingWhitespace: true }, legacy: /(?<=[。!?;;])\s*/ }, { punctuation: "。!?!?;;", options: { consumeFollowingWhitespace: true, splitOnNewlines: true }, legacy: /(?<=[。!?!?;;])\s*|\n+/u, }, { punctuation: "。!?!?.", legacy: /(?<=[。!?!?.])/ }, { punctuation: "。!??\n", legacy: /(?<=[。!??\n])/ }, { punctuation: "。!??", options: { consumeFollowingWhitespace: true }, legacy: /(?<=[。!??])\s*/u }, { punctuation: "。!", options: { consumeFollowingWhitespace: true }, legacy: /(?<=[。!])\s*/u }, ]; const atoms = ["", "甲", "Hello", "1", "。", "!", "?", ";", ";", "!", "?", ".", "\n", "\n\n", " ", " ", "\t", "\r\n", "\u3000", "没有标点"]; const samples = new Set(["", "你好。", "你好。 ", "你好。 "]); for (const left of atoms) { for (const right of atoms) samples.add(left + right); } for (const left of atoms) { for (const mark of ["。", "!", "\n", " "]) { for (const right of ["乙", "", "\n", "."]) samples.add(left + mark + right); } } for (const preset of presets) { for (const sample of samples) { assert.deepEqual( splitAfterSentencePunctuation(sample, preset.punctuation, preset.options), sample.split(preset.legacy), `${preset.punctuation} :: ${JSON.stringify(sample)}`, ); } } });