import { describe, it, expect } from 'bun:test';
import {
calculateObservationTokens,
calculateTokenEconomics,
} from '../../src/core/context/index.js';
import type { Observation } from '../../src/core/context/types.js';
import { CHARS_PER_TOKEN_ESTIMATE } from '../../src/core/context/types.js';
// Helper to create a minimal observation for testing
function createTestObservation(overrides: Partial<Observation> = {}): Observation {
return {
id: 1,
memory_session_id: 'session-123',
type: 'discovery',
title: null,
subtitle: null,
narrative: null,
facts: null,
concepts: null,
files_read: null,
files_modified: null,
discovery_tokens: null,
created_at: '2025-01-01T12:00:00.000Z',
created_at_epoch: 1735732800000,
...overrides,
};
}
describe('TokenCalculator', () => {
describe('CHARS_PER_TOKEN_ESTIMATE constant', () => {
it('should be 4 characters per token', () => {
expect(CHARS_PER_TOKEN_ESTIMATE).toBe(4);
});
});
describe('calculateObservationTokens', () => {
it('should return 0 for an observation with no content', () => {
const obs = createTestObservation();
const tokens = calculateObservationTokens(obs);
// Even empty observations have facts as "[]" when stringified
// null facts becomes '[]' = 2 chars / 4 = 0.5 -> ceil = 1
expect(tokens).toBe(1);
});
it('should estimate tokens based on title length', () => {
const title = 'A'.repeat(40); // 40 chars = 10 tokens
const obs = createTestObservation({ title });
const tokens = calculateObservationTokens(obs);
// title (40) + facts stringified (null -> '[]' = 2) = 42 / 4 = 10.5 -> 11
expect(tokens).toBe(11);
});
it('should estimate tokens based on subtitle length', () => {
const subtitle = 'B'.repeat(20); // 20 chars = 5 tokens
const obs = createTestObservation({ subtitle });
const tokens = calculateObservationTokens(obs);
// subtitle (20) + facts (2) = 22 / 4 = 5.5 -> 6
expect(tokens).toBe(6);
});
it('should estimate tokens based on narrative length', () => {
const narrative = 'C'.repeat(80); // 80 chars = 20 tokens
const obs = createTestObservation({ narrative });
const tokens = calculateObservationTokens(obs);
// narrative (80) + facts (2) = 82 / 4 = 20.5 -> 21
expect(tokens).toBe(21);
});
it('should estimate tokens based on facts JSON length', () => {
// When facts is a string, JSON.stringify adds quotes around it
// '["fact"]' as string becomes '"[\\"fact\\"]"' when stringified
// But in practice, obs.facts is a string that gets stringified
const facts = '["fact one", "fact two", "fact three"]'; // 38 chars
const obs = createTestObservation({ facts });
const tokens = calculateObservationTokens(obs);
// JSON.stringify of string adds quotes: 38 + 2 = 40, plus escaping
// Actually becomes: '"[\"fact one\", \"fact two\", \"fact three\"]"' = 46 chars
// 46 / 4 = 11.5 -> 12
expect(tokens).toBe(12);
});
it('should combine all fields for total token estimate', () => {
const obs = createTestObservation({
title: 'A'.repeat(20), // 20 chars
subtitle: 'B'.repeat(20), // 20 chars
narrative: 'C'.repeat(40), // 40 chars
facts: '["test"]', // 8 chars, but JSON.stringify adds quotes = 10 chars
});
const tokens = calculateObservationTokens(obs);
// 20 + 20 + 40 + 10 (stringified) = 90 / 4 = 22.5 -> 23
expect(tokens).toBe(23);
});
it('should handle large observations correctly', () => {
const largeNarrative = 'X'.repeat(4000); // 4000 chars = 1000 tokens
const obs = createTestObservation({ narrative: largeNarrative });
const tokens = calculateObservationTokens(obs);
// 4000 + 2 (null facts) = 4002 / 4 = 1000.5 -> 1001
expect(tokens).toBe(1001);
});
it('should round up fractional tokens using ceil', () => {
// 9 chars / 4 = 2.25 -> should be 3
const obs = createTestObservation({ title: 'ABCDEFGHI' }); // 9 chars
const tokens = calculateObservationTokens(obs);
// 9 + 2 = 11 / 4 = 2.75 -> 3
expect(tokens).toBe(3);
});
});
describe('calculateTokenEconomics', () => {
it('should return zeros for empty observations array', () => {
const economics = calculateTokenEconomics([]);
expect(economics.totalObservations).toBe(0);
expect(economics.totalReadTokens).toBe(0);
expect(economics.totalDiscoveryTokens).toBe(0);
expect(economics.savings).toBe(0);
expect(economics.savingsPercent).toBe(0);
});
it('should count total observations', () => {
const observations = [
createTestObservation({ id: 1 }),
createTestObservation({ id: 2 }),
createTestObservation({ id: 3 }),
];
const economics = calculateTokenEconomics(observations);
expect(economics.totalObservations).toBe(3);
});
it('should sum read tokens from all observations', () => {
const observations = [
createTestObservation({ title: 'A'.repeat(40) }), // ~11 tokens
createTestObservation({ title: 'B'.repeat(40) }), // ~11 tokens
];
const economics = calculateTokenEconomics(observations);
expect(economics.totalReadTokens).toBe(22);
});
it('should sum discovery tokens from all observations', () => {
const observations = [
createTestObservation({ discovery_tokens: 100 }),
createTestObservation({ discovery_tokens: 200 }),
createTestObservation({ discovery_tokens: 300 }),
];
const economics = calculateTokenEconomics(observations);
expect(economics.totalDiscoveryTokens).toBe(600);
});
it('should handle null discovery_tokens as 0', () => {
const observations = [
createTestObservation({ discovery_tokens: 100 }),
createTestObservation({ discovery_tokens: null }),
createTestObservation({ discovery_tokens: 50 }),
];
const economics = calculateTokenEconomics(observations);
expect(economics.totalDiscoveryTokens).toBe(150);
});
it('should calculate savings as discovery minus read tokens', () => {
const observations = [
createTestObservation({
title: 'A'.repeat(40), // ~11 read tokens
discovery_tokens: 500,
}),
];
const economics = calculateTokenEconomics(observations);
expect(economics.savings).toBe(500 - 11);
expect(economics.savings).toBe(489);
});
it('should calculate savings percent correctly', () => {
// If discovery = 1000 and read = 100, savings = 900, percent = 90%
const observations = [
createTestObservation({
title: 'A'.repeat(396), // 396 + 2 = 398 / 4 = 99.5 -> 100 read tokens
discovery_tokens: 1000,
}),
];
const economics = calculateTokenEconomics(observations);
expect(economics.totalReadTokens).toBe(100);
expect(economics.totalDiscoveryTokens).toBe(1000);
expect(economics.savings).toBe(900);
expect(economics.savingsPercent).toBe(90);
});
it('should return 0% savings when discovery tokens is 0', () => {
const observations = [
createTestObservation({ discovery_tokens: 0 }),
createTestObservation({ discovery_tokens: null }),
];
const economics = calculateTokenEconomics(observations);
expect(economics.savingsPercent).toBe(0);
});
it('should handle negative savings correctly', () => {
// When read tokens > discovery tokens, savings is negative
const observations = [
createTestObservation({
narrative: 'X'.repeat(400), // ~101 read tokens
discovery_tokens: 50,
}),
];
const economics = calculateTokenEconomics(observations);
expect(economics.savings).toBeLessThan(0);
});
it('should round savings percent to nearest integer', () => {
// Create a scenario where savings percent is fractional
// discovery = 100, read = 33, savings = 67, percent = 67%
const observations = [
createTestObservation({
title: 'A'.repeat(130), // 130 + 2 = 132 / 4 = 33 read tokens
discovery_tokens: 100,
}),
];
const economics = calculateTokenEconomics(observations);
expect(economics.totalReadTokens).toBe(33);
expect(economics.savingsPercent).toBe(67);
});
it('should aggregate correctly with multiple observations', () => {
const observations = [
createTestObservation({
id: 1,
title: 'A'.repeat(20),
narrative: 'X'.repeat(60),
discovery_tokens: 500,
}),
createTestObservation({
id: 2,
title: 'B'.repeat(40),
subtitle: 'Y'.repeat(40),
discovery_tokens: 300,
}),
createTestObservation({
id: 3,
narrative: 'Z'.repeat(100),
facts: '["fact1", "fact2"]',
discovery_tokens: 200,
}),
];
const economics = calculateTokenEconomics(observations);
expect(economics.totalObservations).toBe(3);
expect(economics.totalDiscoveryTokens).toBe(1000);
expect(economics.totalReadTokens).toBeGreaterThan(0);
expect(economics.savings).toBe(economics.totalDiscoveryTokens - economics.totalReadTokens);
});
});
});
