);

}

function normalizeConceptToken(rawToken: string): string | null {

function normalizeConceptToken(rawToken: string, fromGlossary = false): string | null {

const normalized = normalizeLowercaseStringOrEmpty(

rawToken

.normalize("NFKC")

return null;

}

const script = classifyConceptTagScript(normalized);

if (normalized.length < minimumTokenLengthForScript(script)) {

// Glossary entries are an explicit allowlist of short technical terms (e.g. "kv", "s3"); they

// bypass the per-script minimum length that would otherwise discard them.

if (!fromGlossary && normalized.length < minimumTokenLengthForScript(script)) {

return null;

}

if (isKanaOnlyToken(normalized) && normalized.length < 3) {

return normalized;

}

// Only entries shorter than their script's minimum token length rely on the glossary bypass, and

// only those need whole-word matching so they don't fire inside longer words ("kv" in "mkv"). Longer

// entries keep substring containment (the shipped behavior, e.g. "backup" tagging inside "backups").

// Precomputed so derive() does not reclassify on every call.

const GLOSSARY_ENTRIES = PROTECTED_GLOSSARY.map((entry) => ({

entry,

wholeWord: entry.length < minimumTokenLengthForScript(classifyConceptTagScript(entry)),

}));

function isAlphanumericAt(source: string, index: number): boolean {

const ch = source[index];

return ch !== undefined && LETTER_OR_NUMBER_RE.test(ch);

}

// True when `entry` occurs as a delimiter-bounded token, not inside a longer word. Keeps short

// glossary entries like "kv"/"s3" from firing inside "mkv"/"css3" once they bypass the length gate.

function includesStandaloneTerm(source: string, entry: string): boolean {

let from = source.indexOf(entry);

while (from !== -1) {

if (!isAlphanumericAt(source, from - 1) && !isAlphanumericAt(source, from + entry.length)) {

return true;

}

from = source.indexOf(entry, from + 1);

}

return false;

}

function collectGlossaryMatches(source: string): string[] {

const normalizedSource = normalizeLowercaseStringOrEmpty(source.normalize("NFKC"));

const matches: string[] = [];

for (const entry of PROTECTED_GLOSSARY) {

if (!normalizedSource.includes(entry)) {

continue;

for (const { entry, wholeWord } of GLOSSARY_ENTRIES) {

const present = wholeWord

? includesStandaloneTerm(normalizedSource, entry)

: normalizedSource.includes(entry);

if (present) {

matches.push(entry);

}

matches.push(entry);

}

return matches;

}

return source.split(/[^\p{L}\p{N}]+/u).filter(Boolean);

}

function pushNormalizedTag(tags: string[], rawToken: string, limit: number): void {

const normalized = normalizeConceptToken(rawToken);

function pushNormalizedTag(

tags: string[],

rawToken: string,

limit: number,

fromGlossary = false,

): void {

const normalized = normalizeConceptToken(rawToken, fromGlossary);

if (!normalized || tags.includes(normalized)) {

return;

}

}

const tags: string[] = [];

for (const rawToken of [

...collectGlossaryMatches(source),

...collectCompoundTokens(source),

...collectSegmentTokens(source),

]) {

pushNormalizedTag(tags, rawToken, limit);

if (tags.length >= limit) {

break;

const tokenSources: Array<{ tokens: string[]; fromGlossary: boolean }> = [

{ tokens: collectGlossaryMatches(source), fromGlossary: true },

{ tokens: collectCompoundTokens(source), fromGlossary: false },

{ tokens: collectSegmentTokens(source), fromGlossary: false },

];

for (const { tokens, fromGlossary } of tokenSources) {

for (const rawToken of tokens) {

pushNormalizedTag(tags, rawToken, limit, fromGlossary);

if (tags.length >= limit) {

return tags;

}

}

}

return tags;