expect(estimateStringChars("안녕하세요")).toBe(20);

});

it("handles East Asian fullwidth letters, numbers, and punctuation", () => {

expect(estimateStringChars("ＡＢＣ１２３")).toBe(6 * CHARS_PER_TOKEN_ESTIMATE);

expect(estimateStringChars("hello，world")).toBe(

"helloworld".length + CHARS_PER_TOKEN_ESTIMATE,

);

});

it("handles CJK punctuation and symbols in the extended range", () => {

// "⺀" (U+2E80) is in CJK Radicals Supplement range

expect(estimateStringChars("⺀")).toBe(CHARS_PER_TOKEN_ESTIMATE);

/**

* Matches CJK Unified Ideographs, CJK Extension A/B, CJK Compatibility

* Ideographs, Hangul Syllables, Hiragana, Katakana, and other non-Latin

* scripts that typically use ~1 token per character.

* scripts and East Asian fullwidth forms that typically use ~1 token per character.

*/

const NON_LATIN_RE = /[\u2E80-\u9FFF\uA000-\uA4FF\uAC00-\uD7AF\uF900-\uFAFF\u{20000}-\u{2FA1F}]/gu;

const NON_LATIN_RE =

/[\u2E80-\u9FFF\uA000-\uA4FF\uAC00-\uD7AF\uF900-\uFAFF\uFF01-\uFF60\uFFE0-\uFFE6\u{20000}-\u{2FA1F}]/gu;

/**

* Return an adjusted character length that accounts for non-Latin (CJK, etc.)