import { describe } from './_harness.mjs';
import { tokenize, normalize } from '../../../src/core/tokenize.mjs';
describe('tokenize', (it)=>{
it('removes english stop words & normalizes full-width', ()=>{
const tokens = tokenize('TEST The quick brown 狀態 fox 的 jump');
if(!tokens.includes('test')) throw new Error('missing normalized token');
if(tokens.includes('the')) throw new Error('stop word not removed');
});
it('handles empty', ()=>{
const t = tokenize('');
if(t.length!==0) throw new Error('empty should yield empty array');
});
it('normalize full-width digits', ()=>{
if(normalize('123')!=='123') throw new Error('full-width digits not normalized');
});
});