tiktoken_bg.cjs•16.6 kB
let wasm;
module.exports.__wbg_set_wasm = function(val) {
wasm = val;
};
const lTextDecoder = typeof TextDecoder === 'undefined' ? (0, module.require)('util').TextDecoder : TextDecoder;
let cachedTextDecoder = new lTextDecoder('utf-8', { ignoreBOM: true, fatal: true });
cachedTextDecoder.decode();
let cachedUint8ArrayMemory0 = null;
function getUint8ArrayMemory0() {
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
}
return cachedUint8ArrayMemory0;
}
function getStringFromWasm0(ptr, len) {
ptr = ptr >>> 0;
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
}
const heap = new Array(128).fill(undefined);
heap.push(undefined, null, true, false);
let heap_next = heap.length;
function addHeapObject(obj) {
if (heap_next === heap.length) heap.push(heap.length + 1);
const idx = heap_next;
heap_next = heap[idx];
heap[idx] = obj;
return idx;
}
function handleError(f, args) {
try {
return f.apply(this, args);
} catch (e) {
wasm.__wbindgen_export_0(addHeapObject(e));
}
}
function getObject(idx) { return heap[idx]; }
function dropObject(idx) {
if (idx < 132) return;
heap[idx] = heap_next;
heap_next = idx;
}
function takeObject(idx) {
const ret = getObject(idx);
dropObject(idx);
return ret;
}
let WASM_VECTOR_LEN = 0;
const lTextEncoder = typeof TextEncoder === 'undefined' ? (0, module.require)('util').TextEncoder : TextEncoder;
let cachedTextEncoder = new lTextEncoder('utf-8');
const encodeString = (typeof cachedTextEncoder.encodeInto === 'function'
? function (arg, view) {
return cachedTextEncoder.encodeInto(arg, view);
}
: function (arg, view) {
const buf = cachedTextEncoder.encode(arg);
view.set(buf);
return {
read: arg.length,
written: buf.length
};
});
function passStringToWasm0(arg, malloc, realloc) {
if (realloc === undefined) {
const buf = cachedTextEncoder.encode(arg);
const ptr = malloc(buf.length, 1) >>> 0;
getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
WASM_VECTOR_LEN = buf.length;
return ptr;
}
let len = arg.length;
let ptr = malloc(len, 1) >>> 0;
const mem = getUint8ArrayMemory0();
let offset = 0;
for (; offset < len; offset++) {
const code = arg.charCodeAt(offset);
if (code > 0x7F) break;
mem[ptr + offset] = code;
}
if (offset !== len) {
if (offset !== 0) {
arg = arg.slice(offset);
}
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
const ret = encodeString(arg, view);
offset += ret.written;
ptr = realloc(ptr, len, offset, 1) >>> 0;
}
WASM_VECTOR_LEN = offset;
return ptr;
}
function isLikeNone(x) {
return x === undefined || x === null;
}
let cachedDataViewMemory0 = null;
function getDataViewMemory0() {
if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
}
return cachedDataViewMemory0;
}
let cachedUint32ArrayMemory0 = null;
function getUint32ArrayMemory0() {
if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) {
cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer);
}
return cachedUint32ArrayMemory0;
}
function getArrayU32FromWasm0(ptr, len) {
ptr = ptr >>> 0;
return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
}
function passArray8ToWasm0(arg, malloc) {
const ptr = malloc(arg.length * 1, 1) >>> 0;
getUint8ArrayMemory0().set(arg, ptr / 1);
WASM_VECTOR_LEN = arg.length;
return ptr;
}
function passArray32ToWasm0(arg, malloc) {
const ptr = malloc(arg.length * 4, 4) >>> 0;
getUint32ArrayMemory0().set(arg, ptr / 4);
WASM_VECTOR_LEN = arg.length;
return ptr;
}
function getArrayU8FromWasm0(ptr, len) {
ptr = ptr >>> 0;
return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len);
}
module.exports.get_encoding = function(encoding, extend_special_tokens) {
if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(encoding, wasm.__wbindgen_export_1, wasm.__wbindgen_export_2);
const len0 = WASM_VECTOR_LEN;
wasm.get_encoding(retptr, ptr0, len0, addHeapObject(extend_special_tokens));
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
if (r2) {
throw takeObject(r1);
}
return Tiktoken.__wrap(r0);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
};
module.exports.encoding_for_model = function(model, extend_special_tokens) {
if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(model, wasm.__wbindgen_export_1, wasm.__wbindgen_export_2);
const len0 = WASM_VECTOR_LEN;
wasm.encoding_for_model(retptr, ptr0, len0, addHeapObject(extend_special_tokens));
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
if (r2) {
throw takeObject(r1);
}
return Tiktoken.__wrap(r0);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
};
module.exports.get_encoding_name_for_model = function(model) {
if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
let deferred3_0;
let deferred3_1;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(model, wasm.__wbindgen_export_1, wasm.__wbindgen_export_2);
const len0 = WASM_VECTOR_LEN;
wasm.get_encoding_name_for_model(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true);
var ptr2 = r0;
var len2 = r1;
if (r3) {
ptr2 = 0; len2 = 0;
throw takeObject(r2);
}
deferred3_0 = ptr2;
deferred3_1 = len2;
return getStringFromWasm0(ptr2, len2);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_export_3(deferred3_0, deferred3_1, 1);
}
};
const TiktokenFinalization = (typeof FinalizationRegistry === 'undefined')
? { register: () => {}, unregister: () => {} }
: new FinalizationRegistry(ptr => wasm.__wbg_tiktoken_free(ptr >>> 0, 1));
class Tiktoken {
/**
* @param {string} tiktoken_bfe
* @param {any} special_tokens
* @param {string} pat_str
*/
constructor(tiktoken_bfe, special_tokens, pat_str) {
if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
const ptr0 = passStringToWasm0(tiktoken_bfe, wasm.__wbindgen_export_1, wasm.__wbindgen_export_2);
const len0 = WASM_VECTOR_LEN;
const ptr1 = passStringToWasm0(pat_str, wasm.__wbindgen_export_1, wasm.__wbindgen_export_2);
const len1 = WASM_VECTOR_LEN;
const ret = wasm.tiktoken_new(ptr0, len0, addHeapObject(special_tokens), ptr1, len1);
this.__wbg_ptr = ret >>> 0;
TiktokenFinalization.register(this, this.__wbg_ptr, this);
return this;
}
/** @returns {string | undefined} */
get name() {
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.tiktoken_name(retptr, this.__wbg_ptr);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
let v1;
if (r0 !== 0) {
v1 = getStringFromWasm0(r0, r1).slice();
wasm.__wbindgen_export_3(r0, r1 * 1, 1);
}
return v1;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
static __wrap(ptr) {
ptr = ptr >>> 0;
const obj = Object.create(Tiktoken.prototype);
obj.__wbg_ptr = ptr;
TiktokenFinalization.register(obj, obj.__wbg_ptr, obj);
return obj;
}
__destroy_into_raw() {
const ptr = this.__wbg_ptr;
this.__wbg_ptr = 0;
TiktokenFinalization.unregister(this);
return ptr;
}
free() {
if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
const ptr = this.__destroy_into_raw();
wasm.__wbg_tiktoken_free(ptr, 0);
}
/**
* @param {string} text
* @param {any} allowed_special
* @param {any} disallowed_special
* @returns {Uint32Array}
*/
encode(text, allowed_special, disallowed_special) {
if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export_1, wasm.__wbindgen_export_2);
const len0 = WASM_VECTOR_LEN;
wasm.tiktoken_encode(retptr, this.__wbg_ptr, ptr0, len0, addHeapObject(allowed_special), addHeapObject(disallowed_special));
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true);
if (r3) {
throw takeObject(r2);
}
var v2 = getArrayU32FromWasm0(r0, r1).slice();
wasm.__wbindgen_export_3(r0, r1 * 4, 4);
return v2;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/**
* @param {string} text
* @returns {Uint32Array}
*/
encode_ordinary(text) {
if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export_1, wasm.__wbindgen_export_2);
const len0 = WASM_VECTOR_LEN;
wasm.tiktoken_encode_ordinary(retptr, this.__wbg_ptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var v2 = getArrayU32FromWasm0(r0, r1).slice();
wasm.__wbindgen_export_3(r0, r1 * 4, 4);
return v2;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/**
* @param {string} text
* @param {any} allowed_special
* @param {any} disallowed_special
* @returns {any}
*/
encode_with_unstable(text, allowed_special, disallowed_special) {
if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export_1, wasm.__wbindgen_export_2);
const len0 = WASM_VECTOR_LEN;
wasm.tiktoken_encode_with_unstable(retptr, this.__wbg_ptr, ptr0, len0, addHeapObject(allowed_special), addHeapObject(disallowed_special));
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
if (r2) {
throw takeObject(r1);
}
return takeObject(r0);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/**
* @param {Uint8Array} bytes
* @returns {number}
*/
encode_single_token(bytes) {
if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
const ptr0 = passArray8ToWasm0(bytes, wasm.__wbindgen_export_1);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.tiktoken_encode_single_token(this.__wbg_ptr, ptr0, len0);
return ret >>> 0;
}
/**
* @param {Uint32Array} tokens
* @returns {Uint8Array}
*/
decode(tokens) {
if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passArray32ToWasm0(tokens, wasm.__wbindgen_export_1);
const len0 = WASM_VECTOR_LEN;
wasm.tiktoken_decode(retptr, this.__wbg_ptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var v2 = getArrayU8FromWasm0(r0, r1).slice();
wasm.__wbindgen_export_3(r0, r1 * 1, 1);
return v2;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/**
* @param {number} token
* @returns {Uint8Array}
*/
decode_single_token_bytes(token) {
if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.tiktoken_decode_single_token_bytes(retptr, this.__wbg_ptr, token);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var v1 = getArrayU8FromWasm0(r0, r1).slice();
wasm.__wbindgen_export_3(r0, r1 * 1, 1);
return v1;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/** @returns {any} */
token_byte_values() {
if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
const ret = wasm.tiktoken_token_byte_values(this.__wbg_ptr);
return takeObject(ret);
}
}
module.exports.Tiktoken = Tiktoken;
module.exports.__wbg_parse_def2e24ef1252aff = function() {
return handleError(function (arg0, arg1) {
const ret = JSON.parse(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
}, arguments) };
;
module.exports.__wbg_stringify_f7ed6987935b4a24 = function() {
return handleError(function (arg0) {
const ret = JSON.stringify(getObject(arg0));
return addHeapObject(ret);
}, arguments) };
;
module.exports.__wbindgen_error_new = function(arg0, arg1) {
const ret = new Error(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
};
;
module.exports.__wbindgen_is_undefined = function(arg0) {
const ret = getObject(arg0) === undefined;
return ret;
};
;
module.exports.__wbindgen_object_drop_ref = function(arg0) {
takeObject(arg0);
};
;
module.exports.__wbindgen_string_get = function(arg0, arg1) {
if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
const obj = getObject(arg1);
const ret = typeof(obj) === 'string' ? obj : undefined;
var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_export_1, wasm.__wbindgen_export_2);
var len1 = WASM_VECTOR_LEN;
getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
};
;
module.exports.__wbindgen_throw = function(arg0, arg1) {
throw new Error(getStringFromWasm0(arg0, arg1));
};
;