fix(context): count fullwidth chars in token estimates (#...
lin-hongkuan
·
2026-06-25
·
via Recent Commits to openclaw:main
| Original file line number | Diff line number | Diff line change |
|---|
@@ -46,6 +46,13 @@ describe("estimateStringChars", () => {
|
46 | 46 | expect(estimateStringChars("안녕하세요")).toBe(20); |
47 | 47 | }); |
48 | 48 | |
| 49 | +it("handles East Asian fullwidth letters, numbers, and punctuation", () => { |
| 50 | +expect(estimateStringChars("ＡＢＣ１２３")).toBe(6 * CHARS_PER_TOKEN_ESTIMATE); |
| 51 | +expect(estimateStringChars("hello，world")).toBe( |
| 52 | +"helloworld".length + CHARS_PER_TOKEN_ESTIMATE, |
| 53 | +); |
| 54 | +}); |
| 55 | + |
49 | 56 | it("handles CJK punctuation and symbols in the extended range", () => { |
50 | 57 | // "⺀" (U+2E80) is in CJK Radicals Supplement range |
51 | 58 | expect(estimateStringChars("⺀")).toBe(CHARS_PER_TOKEN_ESTIMATE); |
|
| Original file line number | Diff line number | Diff line change |
|---|
@@ -20,9 +20,10 @@ export const CHARS_PER_TOKEN_ESTIMATE = 4;
|
20 | 20 | /** |
21 | 21 | * Matches CJK Unified Ideographs, CJK Extension A/B, CJK Compatibility |
22 | 22 | * Ideographs, Hangul Syllables, Hiragana, Katakana, and other non-Latin |
23 | | - * scripts that typically use ~1 token per character. |
| 23 | + * scripts and East Asian fullwidth forms that typically use ~1 token per character. |
24 | 24 | */ |
25 | | -const NON_LATIN_RE = /[\u2E80-\u9FFF\uA000-\uA4FF\uAC00-\uD7AF\uF900-\uFAFF\u{20000}-\u{2FA1F}]/gu; |
| 25 | +const NON_LATIN_RE = |
| 26 | +/[\u2E80-\u9FFF\uA000-\uA4FF\uAC00-\uD7AF\uF900-\uFAFF\uFF01-\uFF60\uFFE0-\uFFE6\u{20000}-\u{2FA1F}]/gu; |
26 | 27 | |
27 | 28 | /** |
28 | 29 | * Return an adjusted character length that accounts for non-Latin (CJK, etc.) |
|
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。
