/**
* AI Gateway Client (Unified/Compat Endpoint)
*
* Uses AI Gateway's OpenAI-compatible endpoint for ALL providers.
* Single endpoint, single format - gateway handles conversion.
*
* Model format: "provider/model-name"
* e.g., "anthropic/claude-sonnet-4-5-20250929", "openai/gpt-4o"
*
* Authentication:
* - BYOK: Configure provider keys in AI Gateway dashboard
* - CF_AIG_TOKEN: Required when Authenticated Gateway is enabled
*/
import type { Env, ChatMessage, ChatOptions, ChatDelta, ToolMetadata } from '../../types';
import { PROVIDERS, TOOL_CAPABLE_MODELS, parseModelId } from './providers';
export { PROVIDERS, TOOL_CAPABLE_MODELS };
// Cloudflare account ID (from wrangler.jsonc)
const ACCOUNT_ID = '0460574641fdbb98159c98ebf593e2bd';
/**
* Get the Compat endpoint URL (OpenAI-compatible for all providers)
*/
function getCompatEndpoint(env: Env): string {
const gatewayId = env.AI_GATEWAY_ID || 'default';
const accountId = env.CF_ACCOUNT_ID || ACCOUNT_ID;
return `https://gateway.ai.cloudflare.com/v1/${accountId}/${gatewayId}/compat/chat/completions`;
}
/**
* Build headers for AI Gateway requests
*/
function buildHeaders(env: Env): Record<string, string> {
const headers: Record<string, string> = {
'Content-Type': 'application/json',
};
// Authenticated Gateway requires cf-aig-authorization header
if (env.CF_AIG_TOKEN) {
headers['cf-aig-authorization'] = `Bearer ${env.CF_AIG_TOKEN}`;
}
return headers;
}
/**
* Convert ChatMessage to OpenAI format (works for all providers via compat endpoint)
*/
function convertMessages(messages: ChatMessage[]): Array<Record<string, unknown>> {
return messages.map(m => {
// Tool result message
if (m.role === 'tool') {
return {
role: 'tool',
content: m.content,
tool_call_id: m.toolCallId || '',
};
}
// Assistant message with tool calls
if (m.role === 'assistant' && m.toolCalls && m.toolCalls.length > 0) {
return {
role: 'assistant',
content: m.content || null,
tool_calls: m.toolCalls.map(tc => ({
id: tc.id,
type: 'function',
function: {
name: tc.name,
arguments: JSON.stringify(tc.arguments),
},
})),
};
}
// Regular message
return {
role: m.role,
content: m.content,
};
});
}
/**
* Convert ToolMetadata to OpenAI tool format
*/
function convertTools(tools: ToolMetadata[]) {
return tools.map(tool => ({
type: 'function',
function: {
name: tool.name,
description: tool.description,
parameters: tool.inputSchema,
},
}));
}
/**
* Parse tool call from text response (fallback for models like Qwen)
*
* Some models output tool calls as JSON in their text response:
* - {"name":"hello","arguments":{"name":"Admin"}}
* - {"tool":"hello","params":{"name":"Admin"}}
*/
function parseToolCallFromText(
text: string,
tools: ToolMetadata[]
): { id: string; name: string; arguments: Record<string, unknown> } | null {
// Get valid tool names
const toolNames = new Set(tools.map(t => t.name));
// Try to find JSON in the text
const jsonMatch = text.match(/\{[\s\S]*\}/);
if (!jsonMatch) return null;
try {
const parsed = JSON.parse(jsonMatch[0]);
// Format 1: {"name":"tool","arguments":{...}}
if (parsed.name && toolNames.has(parsed.name)) {
return {
id: crypto.randomUUID(),
name: parsed.name,
arguments: parsed.arguments || parsed.params || parsed.args || {},
};
}
// Format 2: {"tool":"name","params":{...}}
if (parsed.tool && toolNames.has(parsed.tool)) {
return {
id: crypto.randomUUID(),
name: parsed.tool,
arguments: parsed.params || parsed.arguments || parsed.args || {},
};
}
// Format 3: {"function":{"name":"tool","arguments":{...}}}
if (parsed.function?.name && toolNames.has(parsed.function.name)) {
return {
id: crypto.randomUUID(),
name: parsed.function.name,
arguments: parsed.function.arguments || {},
};
}
} catch {
// Not valid JSON, ignore
}
return null;
}
/**
* Call AI Gateway Compat endpoint (works for ALL providers)
*
* For models fetched from OpenRouter (with IDs like "x-ai/grok-2"),
* we route through OpenRouter: "openrouter/x-ai/grok-2"
* This uses your OpenRouter BYOK key to access all providers.
*/
async function callCompatEndpoint(
env: Env,
provider: string,
model: string,
messages: ChatMessage[],
tools?: ToolMetadata[],
useOpenRouter: boolean = false
): Promise<ChatDelta> {
const endpoint = getCompatEndpoint(env);
// If model contains a slash (OpenRouter format like "x-ai/grok-2"),
// route through OpenRouter for better compatibility
let compatModel: string;
if (useOpenRouter || model.includes('/')) {
// Route through OpenRouter: "openrouter/x-ai/grok-2"
compatModel = `openrouter/${model}`;
} else {
// Native provider format: "provider/model"
compatModel = `${provider}/${model}`;
}
console.log(`[AI Gateway Compat] ${compatModel}`);
const body: Record<string, unknown> = {
model: compatModel,
messages: convertMessages(messages),
};
if (tools && tools.length > 0) {
body.tools = convertTools(tools);
}
const response = await fetch(endpoint, {
method: 'POST',
headers: buildHeaders(env),
body: JSON.stringify(body),
});
if (!response.ok) {
const error = await response.text();
throw new Error(`AI Gateway error (${response.status}): ${error}`);
}
const result = await response.json() as {
choices: Array<{
message: {
content?: string | null;
tool_calls?: Array<{
id: string;
function: {
name: string;
arguments: string;
};
}>;
};
finish_reason: string;
}>;
};
const choice = result.choices[0];
const toolCalls = choice.message.tool_calls?.map(tc => ({
id: tc.id,
name: tc.function.name,
arguments: JSON.parse(tc.function.arguments),
}));
return {
content: choice.message.content || '',
toolCalls,
finishReason: toolCalls ? 'tool_calls' : 'stop',
};
}
/**
* Chat with an AI model
*
* Uses AI Gateway Compat endpoint for all providers (including Workers AI).
* Model format: "provider/model-name" or OpenRouter format "x-ai/grok-2"
*/
export async function chat(
env: Env,
messages: ChatMessage[],
options: ChatOptions
): Promise<ChatDelta> {
// Parse provider and model
const parsed = parseModelId(options.model);
const provider = options.provider || parsed.provider;
const model = parsed.model;
// Check if this is an OpenRouter model (has slash in the original ID)
// Exclude Workers AI models which use @cf/ or @hf/ prefixes
const isOpenRouterModel = options.model.includes('/') &&
!options.model.startsWith('@cf/') &&
!options.model.startsWith('@hf/');
const fullModelId = options.model; // Keep original for OpenRouter routing
console.log(`[AI] Provider: ${provider}, Model: ${model}, OpenRouter: ${isOpenRouterModel}`);
// Workers AI models (@cf/ and @hf/ prefixes) - use native binding for better performance
if ((model.startsWith('@cf/') || model.startsWith('@hf/')) && env.AI) {
const aiMessages = messages.map(msg => ({
role: msg.role as 'user' | 'assistant' | 'system',
content: msg.content,
}));
const aiInput: AiTextGenerationInput = { messages: aiMessages };
if (options.tools && options.tools.length > 0 && TOOL_CAPABLE_MODELS.includes(model)) {
aiInput.tools = options.tools.map(tool => ({
type: 'function' as const,
function: {
name: tool.name,
description: tool.description,
parameters: tool.inputSchema,
},
}));
}
const response = await env.AI.run(model as Parameters<typeof env.AI.run>[0], aiInput);
if (typeof response === 'string') {
return { content: response, finishReason: 'stop' };
}
const result = response as {
response?: string | unknown;
tool_calls?: Array<{
id?: string;
name: string;
arguments: Record<string, unknown>;
}>;
};
console.log('[Workers AI] Raw response:', JSON.stringify(result));
// Ensure content is always a string (some models return objects)
let content = typeof result.response === 'string'
? result.response
: result.response ? JSON.stringify(result.response) : '';
// Check for tool calls in the structured field
let toolCalls = result.tool_calls?.map(tc => ({
id: tc.id || crypto.randomUUID(),
name: tc.name,
arguments: tc.arguments,
}));
// Fallback: Some models (like Qwen) output tool calls as JSON in text
// Try to parse tool call from response text if no structured tool_calls
if (!toolCalls && content && options.tools) {
const parsed = parseToolCallFromText(content, options.tools);
if (parsed) {
console.log('[Workers AI] Parsed tool call from text:', parsed);
toolCalls = [parsed];
content = ''; // Clear content since it was a tool call, not text
}
}
return {
content,
toolCalls,
finishReason: toolCalls ? 'tool_calls' : 'stop',
};
}
// All external providers via Compat endpoint
// For OpenRouter models, pass the full ID to route through OpenRouter
if (isOpenRouterModel) {
return callCompatEndpoint(env, provider, fullModelId, messages, options.tools, true);
}
return callCompatEndpoint(env, provider, model, messages, options.tools, false);
}
/**
* Chat with streaming response
*/
export async function chatStream(
env: Env,
messages: ChatMessage[],
options: ChatOptions
): Promise<ReadableStream<Uint8Array>> {
// For now, use non-streaming and wrap
const result = await chat(env, messages, options);
return new ReadableStream({
start(controller) {
const encoder = new TextEncoder();
if (result.content) {
controller.enqueue(encoder.encode(`data: ${JSON.stringify({ response: result.content })}\n\n`));
}
if (result.toolCalls) {
controller.enqueue(encoder.encode(`data: ${JSON.stringify({ tool_calls: result.toolCalls })}\n\n`));
}
controller.enqueue(encoder.encode('data: [DONE]\n\n'));
controller.close();
},
});
}
/**
* Check if a model supports tool calling
*/
export function supportsTools(model: string): boolean {
return TOOL_CAPABLE_MODELS.includes(model);
}
/**
* Get available models for a provider
*/
export function getModelsForProvider(provider: string): string[] {
const providerInfo = PROVIDERS[provider as keyof typeof PROVIDERS];
return providerInfo?.models || [];
}
