fix(chunk): keep surrogate pairs whole when hard-splitting an over-lo… · openclaw/openclaw@2968004
Bartok9
·
2026-06-27
·
via Recent Commits to openclaw:main
| Original file line number | Diff line number | Diff line change |
|---|
@@ -513,6 +513,19 @@ describe("chunkByNewline", () => {
|
513 | 513 | it.each(["", " \n\n "] as const)("returns empty array for input %j", (text) => { |
514 | 514 | expect(chunkByNewline(text, 100)).toStrictEqual([]); |
515 | 515 | }); |
| 516 | + |
| 517 | +it("does not split surrogate pairs when hard-splitting an over-long line", () => { |
| 518 | +// An emoji-dense line with no break point forces the raw head cut at an odd code-unit offset; |
| 519 | +// it must back off to a code-point boundary so no chunk ends in a high (or starts with a low) |
| 520 | +// surrogate — the same contract the recursive chunkText path already honors. |
| 521 | +const text = "😀".repeat(30); |
| 522 | +const chunks = chunkByNewline(text, 11); |
| 523 | + |
| 524 | +expect(chunks.join("")).toBe(text); |
| 525 | +expect(chunks.length).toBeGreaterThan(1); |
| 526 | +expect(chunks.every((chunk) => !/[\uD800-\uDBFF]$/u.test(chunk))).toBe(true); |
| 527 | +expect(chunks.every((chunk) => !/^[\uDC00-\uDFFF]/u.test(chunk))).toBe(true); |
| 528 | +}); |
516 | 529 | }); |
517 | 530 | |
518 | 531 | describe("chunkTextWithMode", () => { |
|
| Original file line number | Diff line number | Diff line change |
|---|
@@ -163,7 +163,10 @@ export function chunkByNewline(
|
163 | 163 | continue; |
164 | 164 | } |
165 | 165 | |
166 | | -const firstLimit = Math.max(1, maxLineLength - prefix.length); |
| 166 | +// Back the head cut off to a code-point boundary so an over-long line never splits a surrogate |
| 167 | +// pair; the recursive chunkText below is already surrogate-safe, only this first cut was raw. |
| 168 | +const rawLimit = Math.max(1, maxLineLength - prefix.length); |
| 169 | +const firstLimit = avoidTrailingHighSurrogateBreak(lineValue, 0, rawLimit); |
167 | 170 | const first = lineValue.slice(0, firstLimit); |
168 | 171 | chunks.push(prefix + first); |
169 | 172 | const remaining = lineValue.slice(firstLimit); |
|
此内容由惯性聚合(RSS阅读器)自动聚合整理,仅供阅读参考。 原文来自 — 版权归原作者所有。