Jyotisha/frontend/tests/sentence-split.test.ts

import assert from "node:assert/strict";
import { readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";
import test from "node:test";
import { fileURLToPath } from "node:url";

import {
  splitAfterSentencePunctuation,
  type SentenceSplitOptions,
} from "../src/lib/sentence-split.ts";

const sourceRoot = fileURLToPath(new URL("../src/", import.meta.url));

function walk(dir: string): string[] {
  const files: string[] = [];
  for (const entry of readdirSync(dir, { withFileTypes: true })) {
    const path = join(dir, entry.name);
    if (entry.isDirectory()) files.push(...walk(path));
    else files.push(path);
  }
  return files;
}

test("frontend/src has no lookbehind assertions", () => {
  const textExtensions = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".css", ".json", ".md"]);
  const hits: string[] = [];
  for (const file of walk(sourceRoot)) {
    const dot = file.lastIndexOf(".");
    if (!textExtensions.has(file.slice(dot))) continue;
    const source = readFileSync(file, "utf8");
    if (source.includes("(?<=")) hits.push(file);
  }
  assert.deepEqual(hits, []);
});

type HabitCase = {
  habit: string;
  input: string;
  punctuation: string;
  options?: SentenceSplitOptions;
  expected: string[];
};

const habits: HabitCase[] = [
  { habit: "chinese-period", input: "你好。世界", punctuation: "。！？", expected: ["你好。", "世界"] },
  { habit: "chinese-period", input: "甲。乙。丙", punctuation: "。！？", expected: ["甲。", "乙。", "丙"] },
  { habit: "question", input: "你好？世界", punctuation: "。！？", expected: ["你好？", "世界"] },
  { habit: "question", input: "真的吗？是的。", punctuation: "。！？", expected: ["真的吗？", "是的。"] },
  { habit: "exclamation", input: "你好！世界", punctuation: "。！？", expected: ["你好！", "世界"] },
  { habit: "exclamation", input: "停！别走。", punctuation: "。！？", expected: ["停！", "别走。"] },
  {
    habit: "semicolon",
    input: "你好；世界",
    punctuation: "。！？;；",
    options: { consumeFollowingWhitespace: true },
    expected: ["你好；", "世界"],
  },
  {
    habit: "semicolon",
    input: "甲;乙；丙",
    punctuation: "。！？;；",
    options: { consumeFollowingWhitespace: true },
    expected: ["甲;", "乙；", "丙"],
  },
  {
    habit: "english-punctuation",
    input: "Hello! World",
    punctuation: "。！？!?.",
    expected: ["Hello!", " World"],
  },
  {
    habit: "english-punctuation",
    input: "Hello. World",
    punctuation: "。！？!?.",
    expected: ["Hello.", " World"],
  },
  {
    habit: "newline",
    input: "你好\n世界",
    punctuation: "。！？!?；;",
    options: { consumeFollowingWhitespace: true, splitOnNewlines: true },
    expected: ["你好", "世界"],
  },
  {
    habit: "newline",
    input: "甲。\n乙",
    punctuation: "。！？!?；;",
    options: { consumeFollowingWhitespace: true, splitOnNewlines: true },
    expected: ["甲。", "乙"],
  },
  { habit: "repeated-punctuation", input: "你好。。世界", punctuation: "。！？", expected: ["你好。", "。", "世界"] },
  { habit: "repeated-punctuation", input: "啊！！停", punctuation: "。！？!?", expected: ["啊！", "！", "停"] },
  { habit: "no-trailing-punctuation", input: "没有标点的一段", punctuation: "。！？", expected: ["没有标点的一段"] },
  { habit: "no-trailing-punctuation", input: "结尾无句号就这样", punctuation: "。！？", expected: ["结尾无句号就这样"] },
];

test("sentence splits keep punctuation and match each call-site habit", () => {
  const seen = new Map<string, number>();
  for (const sample of habits) {
    seen.set(sample.habit, (seen.get(sample.habit) ?? 0) + 1);
    assert.deepEqual(
      splitAfterSentencePunctuation(sample.input, sample.punctuation, sample.options),
      sample.expected,
      sample.habit,
    );
  }
  for (const [habit, count] of seen) {
    assert.ok(count >= 2, `${habit} needs two cases, has ${count}`);
  }
});

test("sentence splits stay character-identical to the removed lookbehind patterns", () => {
  const presets: Array<{
    punctuation: string;
    options?: SentenceSplitOptions;
    legacy: RegExp;
  }> = [
    { punctuation: "。！？", legacy: /(?<=[。！？])/ },
    { punctuation: "。！？;；", options: { consumeFollowingWhitespace: true }, legacy: /(?<=[。！？;；])\s*/ },
    {
      punctuation: "。！？!?；;",
      options: { consumeFollowingWhitespace: true, splitOnNewlines: true },
      legacy: /(?<=[。！？!?；;])\s*|\n+/u,
    },
    { punctuation: "。！？!?.", legacy: /(?<=[。！？!?.])/ },
    { punctuation: "。！？?\n", legacy: /(?<=[。！？?\n])/ },
    { punctuation: "。！？?", options: { consumeFollowingWhitespace: true }, legacy: /(?<=[。！？?])\s*/u },
    { punctuation: "。！", options: { consumeFollowingWhitespace: true }, legacy: /(?<=[。！])\s*/u },
  ];
  const atoms = ["", "甲", "Hello", "1", "。", "！", "？", "；", ";", "!", "?", ".", "\n", "\n\n", " ", "  ", "\t", "\r\n", "\u3000", "没有标点"];
  const samples = new Set<string>(["", "你好。", "你好。 ", "你好。  "]);
  for (const left of atoms) {
    for (const right of atoms) samples.add(left + right);
  }
  for (const left of atoms) {
    for (const mark of ["。", "!", "\n", " "]) {
      for (const right of ["乙", "", "\n", "."]) samples.add(left + mark + right);
    }
  }
  for (const preset of presets) {
    for (const sample of samples) {
      assert.deepEqual(
        splitAfterSentencePunctuation(sample, preset.punctuation, preset.options),
        sample.split(preset.legacy),
        `${preset.punctuation} :: ${JSON.stringify(sample)}`,
      );
    }
  }
});