import type { Lang, Token } from '../core/types'; import { matchENKeyword, normalizeENKeyword, normalizeWhitespace, tokenizeEN } from '../core/index'; import { segmentGraphemes } from '../grapheme/index'; export type PhraseSpan = { start: number; end: number; length: number }; function toNonEmptyArray(parts: string[]): string[] { return parts.map((s) => s.trim()).filter((s) => s.length > 0); } export function preparePhraseTokens(phrase: string, lang: Lang): string[] { const input = String(phrase ?? ''); if (input === '') return []; if (lang === 'EN') { const { normalizedText } = normalizeWhitespace(input, 'NORMALIZE'); const words = normalizedText.split(' '); // EN:按“全词等值匹配”口径做归一化(小写 + 两端去常见标点) return toNonEmptyArray(words).map((w) => normalizeENKeyword(w)); } // TC:按 grapheme clusters const { clusters } = segmentGraphemes(input, 'PREFERRED'); return clusters; } function tokenEqualsPhraseToken(tokenText: string, phraseToken: string, lang: Lang): boolean { if (lang === 'EN') return matchENKeyword(tokenText, phraseToken); return tokenText === phraseToken; } /** * 在 tokens 中寻找 phraseTokens 的“连续区间完全匹配”(必须)。 * * 输出顺序(确定性): * - start 升序 * - start 相同:length 降序(更长优先) */ export function findPhraseSpans(tokens: Token[], phraseTokens: string[], lang: Lang): PhraseSpan[] { const n = tokens.length; const m = phraseTokens.length; if (n === 0 || m === 0 || m > n) return []; const spans: PhraseSpan[] = []; for (let i = 0; i <= n - m; i++) { let ok = true; for (let j = 0; j < m; j++) { const t = tokens[i + j]?.text ?? ''; const p = phraseTokens[j] ?? ''; if (!tokenEqualsPhraseToken(t, p, lang)) { ok = false; break; } } if (ok) spans.push({ start: i, end: i + m, length: m }); } spans.sort((a, b) => { if (a.start !== b.start) return a.start - b.start; return b.length - a.length; }); return spans; } /** * 判断某个 span 是否被断点拆分到不同的行(即:span 内存在 break)。 * * 说明: * - span=[start,end) 是 token 索引区间 * - breaks[i] 是 token 边界索引(1..N-1) * - 若存在 start < b < end,则 span 被拆分 */ export function isSpanSplitByBreaks(span: PhraseSpan, breaks: number[]): boolean { const s = span.start; const e = span.end; if (e - s <= 1) return false; for (const b of breaks) { if (b > s && b < e) return true; if (b >= e) break; // breaks 升序:可提前退出 } return false; } /** * 仅用于测试:把 EN 文本按 core-contract 口径 tokenize。 */ export function tokenizePhraseLikeInputEN(text: string): Token[] { const { normalizedText } = normalizeWhitespace(text, 'NORMALIZE'); return tokenizeEN(normalizedText); }