93 lines
2.8 KiB
TypeScript
93 lines
2.8 KiB
TypeScript
import type { Lang, Token } from '../core/types';
|
||
import { matchENKeyword, normalizeENKeyword, normalizeWhitespace, tokenizeEN } from '../core/index';
|
||
import { segmentGraphemes } from '../grapheme/index';
|
||
|
||
export type PhraseSpan = { start: number; end: number; length: number };
|
||
|
||
function toNonEmptyArray(parts: string[]): string[] {
|
||
return parts.map((s) => s.trim()).filter((s) => s.length > 0);
|
||
}
|
||
|
||
export function preparePhraseTokens(phrase: string, lang: Lang): string[] {
|
||
const input = String(phrase ?? '');
|
||
if (input === '') return [];
|
||
|
||
if (lang === 'EN') {
|
||
const { normalizedText } = normalizeWhitespace(input, 'NORMALIZE');
|
||
const words = normalizedText.split(' ');
|
||
// EN：按“全词等值匹配”口径做归一化（小写 + 两端去常见标点）
|
||
return toNonEmptyArray(words).map((w) => normalizeENKeyword(w));
|
||
}
|
||
|
||
// TC：按 grapheme clusters
|
||
const { clusters } = segmentGraphemes(input, 'PREFERRED');
|
||
return clusters;
|
||
}
|
||
|
||
function tokenEqualsPhraseToken(tokenText: string, phraseToken: string, lang: Lang): boolean {
|
||
if (lang === 'EN') return matchENKeyword(tokenText, phraseToken);
|
||
return tokenText === phraseToken;
|
||
}
|
||
|
||
/**
|
||
* 在 tokens 中寻找 phraseTokens 的“连续区间完全匹配”（必须）。
|
||
*
|
||
* 输出顺序（确定性）：
|
||
* - start 升序
|
||
* - start 相同：length 降序（更长优先）
|
||
*/
|
||
export function findPhraseSpans(tokens: Token[], phraseTokens: string[], lang: Lang): PhraseSpan[] {
|
||
const n = tokens.length;
|
||
const m = phraseTokens.length;
|
||
if (n === 0 || m === 0 || m > n) return [];
|
||
|
||
const spans: PhraseSpan[] = [];
|
||
|
||
for (let i = 0; i <= n - m; i++) {
|
||
let ok = true;
|
||
for (let j = 0; j < m; j++) {
|
||
const t = tokens[i + j]?.text ?? '';
|
||
const p = phraseTokens[j] ?? '';
|
||
if (!tokenEqualsPhraseToken(t, p, lang)) {
|
||
ok = false;
|
||
break;
|
||
}
|
||
}
|
||
if (ok) spans.push({ start: i, end: i + m, length: m });
|
||
}
|
||
|
||
spans.sort((a, b) => {
|
||
if (a.start !== b.start) return a.start - b.start;
|
||
return b.length - a.length;
|
||
});
|
||
return spans;
|
||
}
|
||
|
||
/**
|
||
* 判断某个 span 是否被断点拆分到不同的行（即：span 内存在 break）。
|
||
*
|
||
* 说明：
|
||
* - span=[start,end) 是 token 索引区间
|
||
* - breaks[i] 是 token 边界索引（1..N-1）
|
||
* - 若存在 start < b < end，则 span 被拆分
|
||
*/
|
||
export function isSpanSplitByBreaks(span: PhraseSpan, breaks: number[]): boolean {
|
||
const s = span.start;
|
||
const e = span.end;
|
||
if (e - s <= 1) return false;
|
||
for (const b of breaks) {
|
||
if (b > s && b < e) return true;
|
||
if (b >= e) break; // breaks 升序：可提前退出
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* 仅用于测试：把 EN 文本按 core-contract 口径 tokenize。
|
||
*/
|
||
export function tokenizePhraseLikeInputEN(text: string): Token[] {
|
||
const { normalizedText } = normalizeWhitespace(text, 'NORMALIZE');
|
||
return tokenizeEN(normalizedText);
|
||
}
|
||
|