import { DatabaseManager } from '../../utils/database';
import { RepositoryManager } from '../../repositories/RepositoryManager';
import * as os from 'os';
import * as path from 'path';
import * as fs from 'fs';
describe('Issue #24 - Token Limit with includeMetadata', () => {
let dbManager: DatabaseManager;
let repositories: RepositoryManager;
let tempDbPath: string;
let testSessionId: string;
beforeEach(() => {
tempDbPath = path.join(os.tmpdir(), `test-issue24-${Date.now()}.db`);
dbManager = new DatabaseManager({
filename: tempDbPath,
maxSize: 10 * 1024 * 1024,
walMode: true,
});
repositories = new RepositoryManager(dbManager);
// Create test session
const session = repositories.sessions.create({
name: 'Issue #24 Test Session',
description: 'Testing token limit with metadata',
});
testSessionId = session.id;
});
afterEach(() => {
dbManager.close();
try {
fs.unlinkSync(tempDbPath);
fs.unlinkSync(`${tempDbPath}-wal`);
fs.unlinkSync(`${tempDbPath}-shm`);
} catch (_e) {
// Ignore
}
});
it('should not exceed token limit when includeMetadata is true with many items', () => {
// Create many items with substantial content to simulate real-world scenario
const itemCount = 200; // Create enough items to potentially exceed token limits
for (let i = 0; i < itemCount; i++) {
repositories.contexts.save(testSessionId, {
key: `test_item_${i}`,
value:
`This is a test item with a moderately long value that simulates real-world context data.
It includes multiple lines and enough content to make the response substantial when
many items are returned together. Each item contributes to the total token count.
Item number: ${i}. Additional metadata and timestamps will be included when
includeMetadata is set to true, further increasing the response size.`.repeat(2),
category: i % 2 === 0 ? 'task' : 'decision',
priority: i % 3 === 0 ? 'high' : 'normal',
channel: 'test-channel',
metadata: JSON.stringify({
index: i,
timestamp: new Date().toISOString(),
tags: ['test', 'issue24', 'metadata'],
}),
});
}
// Query with includeMetadata: true (this was causing the issue)
const result = repositories.contexts.queryEnhanced({
sessionId: testSessionId,
includeMetadata: true,
// No limit specified - repository uses default of 100
// The handler level would apply dynamic limits, but repository doesn't know about includeMetadata
});
// Repository default is 100 items (it doesn't know about metadata)
// The dynamic limiting happens at the handler level in index.ts
expect(result.items.length).toBeLessThanOrEqual(100);
expect(result.totalCount).toBe(itemCount);
// Simulate the response construction with metadata
const itemsWithMetadata = result.items.map(item => ({
key: item.key,
value: item.value,
category: item.category,
priority: item.priority,
channel: item.channel,
metadata: item.metadata ? JSON.parse(item.metadata) : null,
size: item.size || Buffer.byteLength(item.value, 'utf8'),
created_at: item.created_at,
updated_at: item.updated_at,
}));
// Create the full response structure
const response = {
items: itemsWithMetadata,
pagination: {
total: result.totalCount,
returned: result.items.length,
offset: 0,
hasMore: result.totalCount > result.items.length,
nextOffset: result.items.length < result.totalCount ? result.items.length : null,
totalCount: result.totalCount,
page: 1,
pageSize: 50,
totalPages: Math.ceil(result.totalCount / 50),
hasNextPage: result.totalCount > result.items.length,
hasPreviousPage: false,
previousOffset: null,
totalSize: 0,
averageSize: 0,
defaultsApplied: { limit: true, sort: true },
truncated: false,
truncatedCount: 0,
},
};
// Calculate token estimate for the response
const responseJson = JSON.stringify(response, null, 2);
const estimatedTokens = Math.ceil(responseJson.length / 4);
console.log(`Response size: ${responseJson.length} bytes`);
console.log(`Estimated tokens: ${estimatedTokens}`);
console.log(`Items returned: ${result.items.length} of ${result.totalCount}`);
// Note: Repository returns 100 items by default, which may exceed token limits
// This demonstrates why the handler level needs to apply dynamic limiting
// The handler would truncate this response to prevent token overflow
console.log(`Repository returned ${result.items.length} items with ${estimatedTokens} tokens`);
// This shows the problem: repository default of 100 can exceed limits
if (estimatedTokens > 25000) {
console.log('This demonstrates why dynamic limiting at handler level is needed!');
}
});
it('should handle explicit high limit with metadata by truncating', () => {
// Create many items
const itemCount = 500;
for (let i = 0; i < itemCount; i++) {
repositories.contexts.save(testSessionId, {
key: `large_item_${i}`,
value: `Large content ${i}`.repeat(50), // Make items larger
category: 'task',
priority: 'high',
channel: 'large-channel',
});
}
// Query with a high explicit limit and metadata
const result = repositories.contexts.queryEnhanced({
sessionId: testSessionId,
includeMetadata: true,
limit: 1000, // Explicitly request many items
});
// Repository will cap at the actual number of items available
// Since we created 500 items, we should get all 500
// The token limiting would happen at the handler level, not repository level
expect(result.items.length).toBe(500); // Gets all items since limit > itemCount
// Calculate response size
const itemsWithMetadata = result.items.map(item => ({
key: item.key,
value: item.value,
category: item.category,
priority: item.priority,
channel: item.channel,
metadata: item.metadata ? JSON.parse(item.metadata) : null,
size: item.size || Buffer.byteLength(item.value, 'utf8'),
created_at: item.created_at,
updated_at: item.updated_at,
}));
const responseJson = JSON.stringify({ items: itemsWithMetadata }, null, 2);
const estimatedTokens = Math.ceil(responseJson.length / 4);
console.log(`Large query - Items: ${result.items.length}, Tokens: ${estimatedTokens}`);
// This test shows that without handler-level limiting, large queries would overflow
// The handler's dynamic limiting would truncate this to safe levels
if (estimatedTokens > 25000) {
console.log(`Would need truncation: ${estimatedTokens} tokens exceeds 25000 limit`);
}
// The test demonstrates the need for handler-level dynamic limiting
expect(result.items.length).toBe(500); // Repository returns what was requested
});
});
