/**
* Main Chat API Route - Streaming AI Responses
*
* Handles:
* - Quota validation
* - BYOK key detection
* - Context-aware prompt injection
* - Streaming responses via SSE
* - Token tracking and cost calculation
* - Automatic usage logging
*/
import { supabase } from '@/lib/supabase-admin';
import Anthropic from '@anthropic-ai/sdk';
import OpenAI from 'openai';
import crypto from 'crypto';
import type { ContextType } from '@/lib/types/chat';
import { auth } from '@/auth';
import { tools } from '@/lib/toolDefinitions';
import { executeToolCall, formatToolResult, extractNavigation, extractBillContext, BillContext } from '@/lib/toolExecutor';
import {
getCachedResponse,
storeCachedResponse,
isCacheEnabled,
} from '@/lib/llmCache';
// AI Provider clients (server-side) - lazy initialization to avoid build-time errors
function getAnthropicClient() {
return new Anthropic({
apiKey: process.env.ANTHROPIC_API_KEY,
});
}
function getOpenAIClient() {
return new OpenAI({
apiKey: process.env.OPENAI_API_KEY,
});
}
// Pricing constants (per 1M tokens)
const CLAUDE_SONNET_INPUT_PRICE = 3.00; // $3 per 1M input tokens
const CLAUDE_SONNET_OUTPUT_PRICE = 15.00; // $15 per 1M output tokens
const GPT4_TURBO_INPUT_PRICE = 10.00; // $10 per 1M input tokens
const GPT4_TURBO_OUTPUT_PRICE = 30.00; // $30 per 1M output tokens
// Encryption for BYOK keys
function decryptKey(encryptedKey: string, iv: string, tag: string): string {
const decipher = crypto.createDecipheriv(
'aes-256-gcm',
Buffer.from(process.env.ENCRYPTION_KEY!, 'hex'),
Buffer.from(iv, 'hex')
);
decipher.setAuthTag(Buffer.from(tag, 'hex'));
let decrypted = decipher.update(encryptedKey, 'hex', 'utf8');
decrypted += decipher.final('utf8');
return decrypted;
}
// ============================================
// CONVERSATION CONTEXT EXTRACTION
// ============================================
interface Message {
role: 'user' | 'assistant';
content: string;
}
/**
* Extract a concise summary from the last assistant response
* Focuses on the main topic/conclusion for follow-up context
*/
function extractLastResponseContext(messages: Message[]): string {
// Get the last assistant message
const lastAssistant = messages?.filter(m => m.role === 'assistant').pop();
if (!lastAssistant?.content) return '';
const content = lastAssistant.content;
// Extract first meaningful paragraph (skip greetings)
const paragraphs = content.split('\n\n').filter(p => p.trim().length > 50);
if (paragraphs.length === 0) return '';
// Take first 200 chars of the main content
const summary = paragraphs[0].substring(0, 200);
return summary.endsWith('.') ? summary : summary + '...';
}
/**
* Extract bill numbers mentioned in recent messages
*/
function extractMentionedBills(messages: Message[]): string[] {
const billPattern = /\b([CS]-\d+)\b/gi;
const bills = new Set<string>();
messages?.slice(-5).forEach(msg => {
const matches = msg.content?.match(billPattern) || [];
matches.forEach(m => bills.add(m.toUpperCase()));
});
return Array.from(bills);
}
/**
* Extract MP names mentioned in recent messages (simple pattern)
*/
function extractMentionedMPs(messages: Message[]): string[] {
// Common MP name patterns
const mpPatterns = [
/(?:MP|Minister|Hon\.?|Right Hon\.?)\s+([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)/g,
/(?:Pierre Poilievre|Mark Carney|Chrystia Freeland|Jagmeet Singh|Elizabeth May)/gi
];
const mps = new Set<string>();
messages?.slice(-5).forEach(msg => {
if (!msg.content) return;
mpPatterns.forEach(pattern => {
const matches = msg.content.matchAll(pattern);
for (const match of matches) {
mps.add(match[1] || match[0]);
}
});
});
return Array.from(mps).slice(0, 5); // Limit to 5 most recent
}
/**
* Build dynamic conversation context for system prompt injection
*/
function buildConversationContext(messages: Message[]): string {
if (!messages || messages.length < 2) return '';
const parts: string[] = [];
// Add last response summary
const lastSummary = extractLastResponseContext(messages);
if (lastSummary) {
parts.push(`RECENT CONTEXT: You just discussed: "${lastSummary}"`);
}
// Add mentioned entities
const bills = extractMentionedBills(messages);
const mps = extractMentionedMPs(messages);
if (bills.length > 0) {
parts.push(`Bills in this conversation: ${bills.join(', ')}`);
}
if (mps.length > 0) {
parts.push(`MPs mentioned: ${mps.join(', ')}`);
}
return parts.length > 0 ? '\n\n' + parts.join('\n') : '';
}
// Generate context-aware system prompt
function generateSystemPrompt(
context?: {
type: ContextType;
id?: string;
data?: Record<string, any>;
},
customPrompt?: string,
conversationContext?: string // Dynamic context from message history
): string {
const today = new Date().toLocaleDateString('en-CA', {
year: 'numeric',
month: 'long',
day: 'numeric',
timeZone: 'America/Toronto'
});
let basePrompt = `You are Gordie, a guide to Canadian Parliament. Today's date is ${today}.
You have tools to query parliamentary data from a Neo4j database.
**Key Tools:**
- search_hansard: Full-text search of House debates (PRIMARY - use liberally)
- search_mps, get_mp, get_mp_scorecard: MP data, voting, expenses, speeches
- search_bills, get_bill, get_bill_lobbying: Bill tracking and lobbying influence
- get_committees, get_committee: Committee work and testimony
- search_lobby_registrations: Track corporate lobbying
**Usage:**
- ALWAYS use tools for queries - never rely on training knowledge for parliamentary data
- For ANY question about a bill, ALWAYS call get_bill or search_bills first - do not answer from memory
- Always cite data sources
- Search results auto-show in a "View Results" card - don't mention it
- Help button (?) shows full tool documentation
**Bill Queries - CRITICAL RULES:**
- When a user asks about a bill WITHOUT specifying a session, call get_bill WITHOUT the session parameter
- The system will automatically return the LATEST version of that bill (most recent parliament)
- NEVER list or mention historical/previous versions of bills unless explicitly asked
- NEVER add "Previous versions of Bill X" sections from your training knowledge
- Only discuss the SINGLE bill returned by the tool - nothing else
- Example: "What is Bill C-10?" → call get_bill(billNumber="C-10"), then explain ONLY that one bill
- Only mention previous versions if the user explicitly asks "history of Bill C-X" or "all versions of Bill C-X"
**Conversation Flow:**
- Short follow-up questions (like "what about X?" or "what if I am Y?") ALWAYS refer to your previous response
- Interpret ambiguous questions in context of what you just explained
- Do NOT treat follow-ups as new standalone questions - they connect to the ongoing topic
- Example: If you explained a bill and user asks "what about researchers?", answer how THAT BILL affects researchers
Provide clear, data-backed answers about Canadian democracy.`;
// Add static entity context (bill, mp, etc.)
const contextPrompts: Record<ContextType, string> = {
general: '',
mp: `\n\nContext: MP ${context?.data?.name || 'Unknown'} (${context?.data?.party}, ${context?.data?.riding}). Focus on this MP's bills, expenses, committees, votes, petitions.`,
bill: `\n\nContext: You are discussing Bill ${context?.data?.number} from Parliamentary Session ${context?.data?.session}.
IMPORTANT: When calling tools like get_bill, get_bill_lobbying, or get_bill_debates, you MUST use session="${context?.data?.session}" to get the correct bill. Bills like "S-242" exist in multiple sessions with completely different content.
Bill Details:
- Number: ${context?.data?.number}
- Session: ${context?.data?.session}
- Title: "${context?.data?.title || context?.data?.name || 'Unknown'}"
- Status: ${context?.data?.status || 'Unknown'}
- Sponsor: ${context?.data?.sponsor || 'Unknown'}
- Type: ${context?.data?.bill_type || 'Unknown'}
Focus on this specific bill's progress, votes, committees, lobbying, and petitions. Always include the session when querying bill data.`,
dashboard: `\n\nContext: Dashboard view. Provide high-level insights across MPs, bills, committees, conflicts.`,
lobbying: `\n\nContext: Lobbying data. Focus on who lobbies whom, active orgs, legislation influence, DPOH meetings.`,
spending: `\n\nContext: Spending data. Focus on MP expenses, contracts, departments, outliers.`,
visualizer: context?.data?.view === 'equalization'
? `\n\nContext: The user is viewing the Equalization Payments Visualizer, currently on Step ${context?.data?.step || 1} of 7.
EQUALIZATION OVERVIEW (2024-25):
- Total envelope: $25.3B distributed to qualifying provinces
- National standard: $10,927 per capita fiscal capacity
- 6 provinces receive payments, 4 do not, 3 territories receive TFF instead
STEP-BY-STEP EXPLANATION:
Step 1 - What is Equalization: Constitutional program (Section 36(2), 1982) ensuring comparable public services. Federal→Province transfers only.
Step 2 - Revenue Sources: 5 categories measured - Personal Income (35%), Business Income (15%), Consumption (25%), Property (10%), Natural Resources (15%).
Step 3 - Fiscal Capacity: Each province's ability to raise revenue, indexed vs national average (100%).
Step 4 - National Standard: 10-province weighted average = $10,927/capita. Territories excluded (separate TFF program).
Step 5 - Above or Below: Provinces below 100% qualify. AB (156%), SK (112%), BC (107%), NL (104%) do not receive.
Step 6 - Calculate Payment: (National Standard - Province Capacity) × Population. Only positive gaps create payments.
Step 7 - Results: QC ($13.34B), MB ($3.51B), NS ($2.84B), NB ($2.68B), PE ($561M), ON ($421M).
TERRITORIES (TFF Program):
- Yukon: $1.24B ($28,136/capita)
- Northwest Territories: $1.57B ($34,800/capita)
- Nunavut: $2.02B ($50,550/capita)
${context?.data?.step ? `USER IS ON STEP ${context.data.step}: Provide detailed explanation for this step. Answer questions in context of equalization mechanics.` : ''}
Be educational and help users understand Canadian fiscal federalism. Use analogies if helpful.`
: `\n\nContext: The user is viewing the Seat Count Visualizer showing federal election results by province/territory.
Explain seat distributions, party representation, and electoral dynamics. Current seat counts by party are shown on the map.`,
};
basePrompt += contextPrompts[context?.type || 'general'] || '';
// Add dynamic conversation context (last response summary, mentioned entities)
if (conversationContext) {
basePrompt += conversationContext;
}
// Add custom user prompt if provided
if (customPrompt && customPrompt.trim().length > 0) {
basePrompt += `\n\nADDITIONAL USER PREFERENCES:\n${customPrompt.trim()}`;
}
return basePrompt;
}
// Calculate cost in USD based on tokens
function calculateCost(
provider: 'anthropic' | 'openai',
model: string,
inputTokens: number,
outputTokens: number
): number {
if (provider === 'anthropic') {
// Claude Sonnet 3.5
return (
(inputTokens / 1_000_000) * CLAUDE_SONNET_INPUT_PRICE +
(outputTokens / 1_000_000) * CLAUDE_SONNET_OUTPUT_PRICE
);
} else {
// GPT-4 Turbo
return (
(inputTokens / 1_000_000) * GPT4_TURBO_INPUT_PRICE +
(outputTokens / 1_000_000) * GPT4_TURBO_OUTPUT_PRICE
);
}
}
export async function POST(request: Request) {
try {
const body = await request.json();
const { conversation_id, message, context } = body;
if (!conversation_id || !message) {
return Response.json(
{ error: 'Missing conversation_id or message' },
{ status: 400 }
);
}
// Get user from NextAuth session
const session = await auth();
if (!session || !session.user) {
return Response.json(
{ error: 'Not authenticated' },
{ status: 401 }
);
}
const user = { id: session.user.id };
// Check if user is on BYOK plan and whether they have a key configured
const { data: userProfile } = await supabase
.from('user_profiles')
.select('uses_own_key, subscription_tier')
.eq('id', user.id)
.single();
const isBYOKUser = userProfile?.uses_own_key === true;
let hasBYOKKey = false;
if (isBYOKUser) {
// Check if BYOK user has an active API key
const { data: apiKeys } = await supabase
.from('user_api_keys')
.select('id')
.eq('user_id', user.id)
.eq('is_active', true)
.limit(1);
hasBYOKKey = (apiKeys?.length || 0) > 0;
console.log(`[Chat] BYOK user check: hasKey=${hasBYOKKey}`);
}
// BYOK users with a key get unlimited queries (skip quota check)
// BYOK users WITHOUT a key are restricted to FREE tier quota (10 lifetime)
let skipQuotaCheck = false;
if (isBYOKUser && hasBYOKKey) {
// BYOK with key = unlimited
console.log('[Chat] BYOK user with key - skipping quota check');
skipQuotaCheck = true;
} else if (isBYOKUser && !hasBYOKKey) {
// BYOK without key = enforce FREE tier quota (10 lifetime queries)
console.log('[Chat] BYOK user without key - enforcing FREE tier quota');
const { data: lifetimeUsage } = await supabase
.from('usage_logs')
.select('id')
.eq('user_id', user.id)
.eq('counted_against_quota', true);
const queriesUsed = lifetimeUsage?.length || 0;
const FREE_TIER_LIMIT = 10;
if (queriesUsed >= FREE_TIER_LIMIT) {
return Response.json(
{
error: 'You have reached the free tier limit (10 queries). Please add your Anthropic API key in Settings → API Keys to continue using the chatbot.',
requires_payment: false,
requires_api_key: true,
},
{ status: 429 }
);
}
}
// Check quota using PostgreSQL function (only for non-BYOK or BYOK without key under limit)
if (!skipQuotaCheck) {
const { data: quotaResult, error: quotaError } = await supabase.rpc(
'can_user_query',
{ p_user_id: user.id }
);
if (quotaError) {
console.error('Quota check error:', quotaError);
// In development, allow queries if quota check fails
console.log('Allowing query despite quota check error (development mode)');
} else if (quotaResult && !quotaResult.can_query) {
// Quota check succeeded but user cannot query
return Response.json(
{
error: quotaResult.reason || 'Quota exceeded',
requires_payment: quotaResult.requires_payment || false,
},
{ status: 429 }
);
}
}
// ============================================
// LLM RESPONSE CACHE CHECK
// ============================================
// Check cache before making LLM call (uses 'anthropic' as default model for cache key)
const defaultModel = 'claude-sonnet-4-5';
if (isCacheEnabled()) {
const cacheResult = await getCachedResponse(message, context, defaultModel);
if (cacheResult.hit && cacheResult.entry) {
console.log(`[Chat] Cache HIT - returning cached response (${cacheResult.latencyMs}ms)`);
// Save user message to database
const { error: userMsgError } = await supabase
.from('messages')
.insert({
conversation_id,
role: 'user',
content: message,
used_byo_key: false,
});
if (userMsgError) {
console.error('Error saving user message:', userMsgError);
}
// Save cached assistant message to database
const { data: assistantMessage, error: assistantMsgError } = await supabase
.from('messages')
.insert({
conversation_id,
role: 'assistant',
content: cacheResult.entry.response_content,
tokens_input: 0, // Cached - no actual tokens used
tokens_output: 0,
tokens_total: 0,
provider: 'anthropic',
model: defaultModel,
used_byo_key: false,
cost_usd: 0, // Cached - no cost
})
.select()
.single();
if (assistantMsgError) {
console.error('Error saving cached assistant message:', assistantMsgError);
}
// Return cached response as streaming (immediate delivery)
const encoder = new TextEncoder();
const stream = new ReadableStream({
start(controller) {
// Send cached content
const responseData = {
content: cacheResult.entry!.response_content,
cached: true,
};
controller.enqueue(encoder.encode(`data: ${JSON.stringify(responseData)}\n\n`));
// Send completion signal
controller.enqueue(
encoder.encode(
`data: ${JSON.stringify({
done: true,
message: assistantMessage,
cached: true,
cacheSavings: {
tokens: cacheResult.entry!.input_tokens + cacheResult.entry!.output_tokens,
cost: cacheResult.entry!.original_cost_usd,
},
})}\n\n`
)
);
controller.enqueue(encoder.encode('data: [DONE]\n\n'));
controller.close();
},
});
return new Response(stream, {
headers: {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
Connection: 'keep-alive',
},
});
}
}
// ============================================
// Check for BYOK keys
console.log('[Chat] Checking for user API keys for user:', user.id);
const { data: apiKeys, error: apiKeysError } = await supabase
.from('user_api_keys')
.select('*')
.eq('user_id', user.id)
.eq('is_active', true);
if (apiKeysError) {
console.error('[Chat] Error fetching API keys:', apiKeysError);
}
console.log('[Chat] Found API keys:', apiKeys?.map(k => k.provider) || []);
const anthropicKey = apiKeys?.find((k) => k.provider === 'anthropic');
const openaiKey = apiKeys?.find((k) => k.provider === 'openai');
// Determine which provider to use
let provider: 'anthropic' | 'openai' = 'anthropic';
let usedBYOKey = false;
let providerClient: Anthropic | OpenAI;
if (anthropicKey) {
// Use user's Anthropic key
console.log('[Chat] Using user Anthropic key');
try {
const decryptedKey = decryptKey(
anthropicKey.encrypted_key,
anthropicKey.encryption_iv,
anthropicKey.encryption_tag
);
console.log('[Chat] Key decrypted successfully, length:', decryptedKey?.length);
providerClient = new Anthropic({ apiKey: decryptedKey });
provider = 'anthropic';
usedBYOKey = true;
} catch (error) {
console.error('[Chat] Error decrypting user Anthropic key:', error);
throw new Error('Failed to decrypt your API key. Please re-save it in settings.');
}
} else if (openaiKey) {
// Use user's OpenAI key
console.log('[Chat] Using user OpenAI key');
try {
const decryptedKey = decryptKey(
openaiKey.encrypted_key,
openaiKey.encryption_iv,
openaiKey.encryption_tag
);
providerClient = new OpenAI({ apiKey: decryptedKey });
provider = 'openai';
usedBYOKey = true;
} catch (error) {
console.error('[Chat] Error decrypting user OpenAI key:', error);
throw new Error('Failed to decrypt your API key. Please re-save it in settings.');
}
} else {
// Use platform's Anthropic key
console.log('[Chat] No user API key found, using platform key');
const platformKey = process.env.ANTHROPIC_API_KEY;
if (!platformKey) {
console.error('[Chat] Platform ANTHROPIC_API_KEY not set!');
throw new Error('Platform API key not configured. Please add your own API key in Settings → API Keys.');
}
providerClient = getAnthropicClient();
provider = 'anthropic';
}
// Load conversation history
const { data: messages } = await supabase
.from('messages')
.select('*')
.eq('conversation_id', conversation_id)
.order('created_at', { ascending: true })
.limit(20); // Keep last 20 messages for context
// Fetch user's custom Gordie prompt if exists
const { data: userPreferences } = await supabase
.from('user_preferences')
.select('custom_gordie_prompt')
.eq('user_id', user.id)
.single();
const customPrompt = userPreferences?.custom_gordie_prompt || '';
// Build dynamic context from conversation history
const conversationMessages: Message[] = (messages || [])
.filter(m => typeof m.content === 'string')
.map(m => ({ role: m.role as 'user' | 'assistant', content: m.content }));
const conversationContext = buildConversationContext(conversationMessages);
// Build message history with all context layers
const systemPrompt = generateSystemPrompt(context, customPrompt, conversationContext);
// Save user message to database
const { data: userMessage, error: userMsgError } = await supabase
.from('messages')
.insert({
conversation_id,
role: 'user',
content: message,
used_byo_key: usedBYOKey,
})
.select()
.single();
if (userMsgError) {
console.error('Error saving user message:', userMsgError);
return Response.json(
{ error: 'Failed to save message' },
{ status: 500 }
);
}
// Create streaming response
const encoder = new TextEncoder();
const stream = new ReadableStream({
async start(controller) {
try {
let assistantContent = '';
let inputTokens = 0;
let outputTokens = 0;
if (provider === 'anthropic') {
// Anthropic Claude streaming
// Filter out any messages with tool_use blocks (incomplete/malformed messages)
// Only include completed text messages
const messageHistory = messages
?.filter((m) => {
// Skip messages that contain tool_use or tool_result in their content
// These are intermediate messages that shouldn't be in conversation history
let content = m.content;
// Log the content for debugging
console.log(`[Chat] Checking message ${m.id} (${m.role}):`, typeof content, content);
// Try to parse if it's a JSON string
if (typeof content === 'string') {
try {
const parsed = JSON.parse(content);
if (Array.isArray(parsed)) {
// If it's an array, check if any item has type tool_use or tool_result
const hasToolContent = parsed.some(
(item) => item.type === 'tool_use' || item.type === 'tool_result'
);
if (hasToolContent) {
console.log('[Chat] Filtering out message with tool content:', m.id);
return false;
}
}
} catch {
// Not JSON, check if string contains tool references
if (content.includes('tool_use') || content.includes('tool_result')) {
console.log('[Chat] Filtering out message with tool reference in string:', m.id);
return false;
}
}
} else if (typeof content === 'object') {
// If it's already an object/array
if (Array.isArray(content)) {
const hasToolContent = content.some(
(item) => item.type === 'tool_use' || item.type === 'tool_result'
);
if (hasToolContent) {
console.log('[Chat] Filtering out message with tool content object:', m.id);
return false;
}
}
console.log('[Chat] Filtering out message with structured content:', m.id);
return false; // Skip any other structured content
}
return true;
})
.map((m) => ({
role: m.role as 'user' | 'assistant',
content: m.content,
})) || [];
console.log(`[Chat] Loaded ${messages?.length || 0} messages, filtered to ${messageHistory.length} messages`);
messageHistory.push({
role: 'user',
content: message,
});
// Enable tool calling
let response = await (providerClient as Anthropic).messages.create({
model: 'claude-sonnet-4-5',
max_tokens: 4096,
system: systemPrompt,
messages: messageHistory,
tools: tools,
});
inputTokens = response.usage.input_tokens;
outputTokens = response.usage.output_tokens;
// Track navigation data from tools
let navigationData: { url: string; message: string } | null = null;
// Track bill context from tools
let billContextData: BillContext | null = null;
// Track all tool calls made during this response
const functionCalls: { name: string; input: any }[] = [];
// Handle tool calls (may need multiple rounds)
while (response.stop_reason === 'tool_use') {
// Get ALL tool_use blocks from the response
const toolUses = response.content.filter((block) => block.type === 'tool_use');
if (toolUses.length === 0) break;
console.log(`[Chat] ${toolUses.length} tool(s) called`);
// Add assistant's tool use to history (entire response with all tool calls)
messageHistory.push({
role: 'assistant',
content: response.content,
});
// Execute ALL tools and collect results
const toolResults: any[] = [];
for (const toolUse of toolUses) {
if (toolUse.type !== 'tool_use') continue;
console.log(`[Chat] Executing tool: ${toolUse.name}`, toolUse.input);
// Track this tool call
functionCalls.push({ name: toolUse.name, input: toolUse.input });
// Execute the tool
const toolResult = await executeToolCall(toolUse.name, toolUse.input as Record<string, any>);
// Check if tool returned navigation data
const nav = extractNavigation(toolResult);
if (nav) {
navigationData = nav;
}
// Check if tool returned bill context
const billCtx = extractBillContext(toolResult);
if (billCtx) {
billContextData = billCtx;
}
const formattedResult = formatToolResult(toolResult);
toolResults.push({
type: 'tool_result',
tool_use_id: toolUse.id,
content: formattedResult,
});
}
// Add ALL tool results in a single user message
messageHistory.push({
role: 'user',
content: toolResults,
});
// Get Claude's response with the tool results
response = await (providerClient as Anthropic).messages.create({
model: 'claude-sonnet-4-5',
max_tokens: 4096,
system: systemPrompt,
messages: messageHistory,
tools: tools,
});
inputTokens += response.usage.input_tokens;
outputTokens += response.usage.output_tokens;
}
// Extract final text response
for (const block of response.content) {
if (block.type === 'text') {
assistantContent += block.text;
}
}
// Stream the final response to client
const responseData: any = { content: assistantContent };
if (navigationData) {
responseData.navigation = navigationData;
}
if (billContextData) {
responseData.billContext = billContextData;
}
if (functionCalls.length > 0) {
responseData.function_calls = functionCalls;
}
const data = `data: ${JSON.stringify(responseData)}\n\n`;
// Check if controller is still open before enqueueing
try {
controller.enqueue(encoder.encode(data));
} catch (err) {
console.error('Controller already closed, response:', err);
return; // Exit early if controller is closed
}
} else {
// OpenAI GPT streaming
const messageHistory = [
{ role: 'system' as const, content: systemPrompt },
...(messages?.map((m) => ({
role: m.role as 'user' | 'assistant',
content: m.content,
})) || []),
{ role: 'user' as const, content: message },
];
const stream = await (providerClient as OpenAI).chat.completions.create({
model: 'gpt-4-turbo-preview',
messages: messageHistory,
stream: true,
});
for await (const chunk of stream) {
const text = chunk.choices[0]?.delta?.content || '';
assistantContent += text;
// Send chunk to client
const data = `data: ${JSON.stringify({ content: text })}\n\n`;
controller.enqueue(encoder.encode(data));
}
// Estimate tokens for OpenAI (rough estimate: 1 token ≈ 4 characters)
inputTokens = Math.ceil(
(systemPrompt.length + messageHistory.map((m) => m.content).join('').length) / 4
);
outputTokens = Math.ceil(assistantContent.length / 4);
}
// Calculate cost
const totalTokens = inputTokens + outputTokens;
const cost = calculateCost(provider, 'claude-sonnet-4-5', inputTokens, outputTokens);
// Save assistant message to database
const { data: assistantMessage, error: assistantMsgError } = await supabase
.from('messages')
.insert({
conversation_id,
role: 'assistant',
content: assistantContent,
tokens_input: inputTokens,
tokens_output: outputTokens,
tokens_total: totalTokens,
provider,
model: provider === 'anthropic' ? 'claude-sonnet-4-5' : 'gpt-4-turbo-preview',
used_byo_key: usedBYOKey,
cost_usd: cost,
})
.select()
.single();
if (assistantMsgError) {
console.error('Error saving assistant message:', assistantMsgError);
}
// Store response in cache (async, don't block response)
if (isCacheEnabled() && !usedBYOKey) {
// Only cache platform API responses (not BYOK) to avoid caching personalized responses
storeCachedResponse(
message,
context,
provider === 'anthropic' ? 'claude-sonnet-4-5' : 'gpt-4-turbo-preview',
provider,
{
content: assistantContent,
inputTokens,
outputTokens,
costUsd: cost,
}
).catch((err) => {
console.error('[Chat] Error storing response in cache:', err);
});
}
// Track usage in database (only if not using BYOK)
// NOTE: We're directly inserting into usage_logs instead of using the RPC function
// because there are conflicting function signatures in migrations
if (!usedBYOKey) {
const { error: usageError } = await supabase
.from('usage_logs')
.insert({
user_id: user.id,
conversation_id: conversation_id,
query_date: new Date().toISOString().split('T')[0],
tokens_total: inputTokens + outputTokens,
tokens_input: inputTokens,
tokens_output: outputTokens,
cost_usd: cost,
counted_against_quota: true,
model_used: provider === 'anthropic' ? 'claude-sonnet-4-5' : 'gpt-4-turbo-preview',
});
if (usageError) {
console.error('Error tracking usage:', usageError);
}
}
// Update conversation metadata
await supabase
.from('conversations')
.update({
message_count: (messages?.length || 0) + 2, // +2 for user and assistant messages
total_tokens: totalTokens,
last_message_at: new Date().toISOString(),
})
.eq('id', conversation_id);
// Send completion signal
const doneData = `data: ${JSON.stringify({
done: true,
message: assistantMessage,
})}\n\n`;
controller.enqueue(encoder.encode(doneData));
// End stream
controller.enqueue(encoder.encode('data: [DONE]\n\n'));
controller.close();
} catch (error) {
console.error('Streaming error:', error);
const errorData = `data: ${JSON.stringify({
error: error instanceof Error ? error.message : 'Unknown error',
})}\n\n`;
controller.enqueue(encoder.encode(errorData));
controller.close();
}
},
});
return new Response(stream, {
headers: {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
},
});
} catch (error) {
console.error('Chat API error:', error);
return Response.json(
{ error: error instanceof Error ? error.message : 'Internal server error' },
{ status: 500 }
);
}
}
