更新换行算法和APP-PUSH

This commit is contained in:
吕新雨
2026-02-10 11:39:33 +08:00
parent f03d36b5e9
commit ee2d9f44ea
105 changed files with 9967 additions and 233 deletions

View File

@@ -0,0 +1,65 @@
import { describe, expect, it } from 'vitest';
import { segmentGraphemes } from '../index';
const REQUIRED_SINGLE_CLUSTER_CASES: Array<{ name: string; input: string }> = [
{ name: 'ZWJ family', input: '👨‍👩‍👧‍👦' },
{ name: 'flag', input: '🇸🇬' },
{ name: 'skin tone', input: '👍🏽' },
{ name: 'ZWJ + variation', input: '😮‍💨' },
{ name: 'combining mark', input: 'e\u0301' }, // é组合形式
];
describe('textWrap grapheme-segmentation', () => {
it('基础性质:空字符串 -> []', () => {
const out = segmentGraphemes('', 'PREFERRED');
expect(out.clusters).toEqual([]);
expect(out.clusters.join('')).toBe('');
});
it('基础性质可逆性clusters.join("") === input', () => {
const input = '我好累😮‍💨';
const out = segmentGraphemes(input, 'PREFERRED');
expect(out.clusters.join('')).toBe(input);
});
it('基础性质:确定性(同输入同输出)', () => {
const input = '👨👩👧👦🇸🇬👍🏽😮💨e\u0301';
const a = segmentGraphemes(input, 'PREFERRED');
const b = segmentGraphemes(input, 'PREFERRED');
expect(a).toEqual(b);
});
it('必测样例:在 PREFERRED 下必须“不拆”为单个 cluster', () => {
for (const c of REQUIRED_SINGLE_CLUSTER_CASES) {
const out = segmentGraphemes(c.input, 'PREFERRED');
expect(out.clusters.join('')).toBe(c.input);
expect(out.clusters.length).toBe(1);
expect(out.clusters[0]).toBe(c.input);
}
});
it('必测样例:在强制 FALLBACK 下必须“不拆”为单个 cluster', () => {
for (const c of REQUIRED_SINGLE_CLUSTER_CASES) {
const out = segmentGraphemes(c.input, 'FALLBACK');
expect(out.meta.strategy).toBe('FALLBACK');
expect(out.meta.hadFallback).toBe(true);
expect(out.clusters.join('')).toBe(c.input);
expect(out.clusters.length).toBe(1);
expect(out.clusters[0]).toBe(c.input);
}
});
it('跨策略一致性:若 Intl.Segmenter 可用,则 PREFERRED 与 FALLBACK 输出应一致', () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const hasIntl = typeof (globalThis as any)?.Intl?.Segmenter === 'function';
if (!hasIntl) return;
for (const c of REQUIRED_SINGLE_CLUSTER_CASES) {
const preferred = segmentGraphemes(c.input, 'PREFERRED');
const fallback = segmentGraphemes(c.input, 'FALLBACK');
expect(preferred.clusters).toEqual(fallback.clusters);
}
});
});

View File

@@ -0,0 +1,9 @@
export type {
GraphemeSegmentationMeta,
GraphemeSegmentationMode,
GraphemeSegmentationResult,
GraphemeSegmentationStrategy,
} from './types';
export { segmentGraphemes } from './segmentGraphemes';

View File

@@ -0,0 +1,42 @@
import type { GraphemeSegmentationMode, GraphemeSegmentationResult } from './types';
import { canUseIntlSegmenter, segmentWithIntlSegmenter } from './strategies/intlSegmenter';
import { segmentWithFallback } from './strategies/fallback';
/**
* 把字符串切分为 grapheme clusters字符簇
*
* 契约(必须):
* - clusters.join('') === text不允许丢字符/改顺序)
* - text=='' 时 clusters==[]
* - 任何异常都必须降级到 fallback保证可用性
*/
export function segmentGraphemes(text: string, mode: GraphemeSegmentationMode = 'PREFERRED'): GraphemeSegmentationResult {
const input = String(text ?? '');
if (input === '') {
return { clusters: [], meta: { strategy: 'FALLBACK', hadFallback: mode === 'FALLBACK' } };
}
// 强制 fallback
if (mode === 'FALLBACK') {
const clusters = segmentWithFallback(input);
return { clusters, meta: { strategy: 'FALLBACK', hadFallback: true } };
}
// 优先 Intl.Segmenter
if (canUseIntlSegmenter()) {
try {
const clusters = segmentWithIntlSegmenter(input);
// 防御:必须可逆
if (clusters.length > 0 && clusters.join('') === input) {
return { clusters, meta: { strategy: 'INTL_SEGMENTER', hadFallback: false } };
}
// 若结果异常(空/丢字符),走 fallback
} catch {
// ignore -> fallback
}
}
const clusters = segmentWithFallback(input);
return { clusters, meta: { strategy: 'FALLBACK', hadFallback: true } };
}

View File

@@ -0,0 +1,81 @@
import GraphemeSplitter from 'grapheme-splitter';
/**
* fallback 策略:使用 grapheme-splitter 分割 grapheme clusters。
*
* 选择原因:
* - 避免手写不完整的 Unicode 规则导致“漏拆/误拆”
* - 作为 Intl.Segmenter 不可用时的稳定兜底
*
* 注意:
* - grapheme-splitter 对少数较新的 emoji ZWJ 序列支持可能不完整(例如 `😮‍💨`)。
* - 为满足本项目“至少不拆 ZWJ sequence/VS16/肤色修饰符/组合字符”的最低要求,这里做一层确定性的后处理合并。
*/
export function segmentWithFallback(text: string): string[] {
const splitter = new GraphemeSplitter();
const raw = splitter.splitGraphemes(text);
return mergeFallbackClusters(raw);
}
const ZWJ = '\u200D';
function firstCodePoint(s: string): number | null {
if (!s) return null;
const cp = s.codePointAt(0);
return typeof cp === 'number' ? cp : null;
}
function isVariationSelector(cp: number): boolean {
// VS15/VS16
return cp === 0xfe0e || cp === 0xfe0f;
}
function isSkinToneModifier(cp: number): boolean {
return cp >= 0x1f3fb && cp <= 0x1f3ff;
}
function isCombiningMark(cp: number): boolean {
// 常见 combining marks 范围(覆盖 `e\u0301` 等)
return (
(cp >= 0x0300 && cp <= 0x036f) ||
(cp >= 0x1ab0 && cp <= 0x1aff) ||
(cp >= 0x1dc0 && cp <= 0x1dff) ||
(cp >= 0x20d0 && cp <= 0x20ff) ||
(cp >= 0xfe20 && cp <= 0xfe2f)
);
}
/**
* 对 fallback 输出做“最低可用”合并:
* - 若上一个 cluster 以 ZWJ 结尾则必须与下一个合并ZWJ sequence
* - 若下一个 cluster 以 VS/肤色修饰符/combining mark 开头,也与上一个合并
*
* 目标:满足文档列出的“不拆”组合要求,且行为完全确定性。
*/
function mergeFallbackClusters(raw: string[]): string[] {
if (!raw.length) return raw;
const out: string[] = [];
let buf = raw[0] ?? '';
for (let i = 1; i < raw.length; i++) {
const next = raw[i] ?? '';
const nextCp = firstCodePoint(next);
const shouldMerge =
buf.endsWith(ZWJ) ||
(nextCp !== null && (isVariationSelector(nextCp) || isSkinToneModifier(nextCp) || isCombiningMark(nextCp)));
if (shouldMerge) {
buf += next;
continue;
}
out.push(buf);
buf = next;
}
out.push(buf);
return out;
}

View File

@@ -0,0 +1,28 @@
/**
* Intl.Segmenter 策略(优先)
*
* 注意:不同 JS 引擎/版本对 Intl.Segmenter 的支持可能不同,调用方必须捕获异常并降级。
*/
export function canUseIntlSegmenter(): boolean {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const Seg = (globalThis as any)?.Intl?.Segmenter;
return typeof Seg === 'function';
}
export function segmentWithIntlSegmenter(text: string): string[] {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const Segmenter = (globalThis as any).Intl.Segmenter as new (locales?: string | string[], options?: any) => any;
// 用 zh-Hant 仅用于选择合适的 localegrapheme 分割应与语言本身关系不大,但保持固定输入更易对齐跨端。
const seg = new Segmenter('zh-Hant', { granularity: 'grapheme' });
const it = seg.segment(text);
const clusters: string[] = [];
for (const part of it) {
// part: { segment: string, index: number, input: string, isWordLike?: boolean }
clusters.push(part.segment);
}
return clusters;
}

View File

@@ -0,0 +1,21 @@
/**
* Text Wrap - grapheme-segmentation
*
* 本模块只负责把字符串切成“字符簇grapheme clusters”数组。
* 后续 TC 换行断点只能发生在 clusters 边界。
*/
export type GraphemeSegmentationMode = 'PREFERRED' | 'FALLBACK';
export type GraphemeSegmentationStrategy = 'INTL_SEGMENTER' | 'FALLBACK';
export type GraphemeSegmentationMeta = {
strategy: GraphemeSegmentationStrategy;
hadFallback: boolean;
};
export type GraphemeSegmentationResult = {
clusters: string[];
meta: GraphemeSegmentationMeta;
};