prepare-tokens.js•1.09 kB
import { tokenize } from '../tokenizer/index.js';
const astToTokens = {
decorator(handlers) {
const tokens = [];
let curNode = null;
return {
...handlers,
node(node) {
const tmp = curNode;
curNode = node;
handlers.node.call(this, node);
curNode = tmp;
},
emit(value, type, auto) {
tokens.push({
type,
value,
node: auto ? null : curNode
});
},
result() {
return tokens;
}
};
}
};
function stringToTokens(str) {
const tokens = [];
tokenize(str, (type, start, end) =>
tokens.push({
type,
value: str.slice(start, end),
node: null
})
);
return tokens;
}
export default function(value, syntax) {
if (typeof value === 'string') {
return stringToTokens(value);
}
return syntax.generate(value, astToTokens);
};