config.ts•4.01 kB
/**
* Configuration for Apify MCP Server evaluations.
*/
import { readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
// Read version from test-cases.json
function getTestCasesVersion(): string {
const currentFilename = fileURLToPath(import.meta.url);
const currentDirname = dirname(currentFilename);
const testCasesPath = join(currentDirname, 'test-cases.json');
const testCasesContent = readFileSync(testCasesPath, 'utf-8');
const testCases = JSON.parse(testCasesContent);
return testCases.version;
}
// Evaluator names
export const EVALUATOR_NAMES = {
TOOLS_EXACT_MATCH: 'tool-exact-match',
TOOL_SELECTION_LLM: 'tool-selection-llm',
} as const;
export type EvaluatorName = typeof EVALUATOR_NAMES[keyof typeof EVALUATOR_NAMES];
// Models to evaluate
export const MODELS_TO_EVALUATE = [
'openai/gpt-4o-mini',
'anthropic/claude-3.5-haiku',
'google/gemini-2.5-flash',
];
export const TOOL_SELECTION_EVAL_MODEL = 'openai/gpt-4o-mini';
export const PASS_THRESHOLD = 0.7;
export const DATASET_NAME = `mcp_server_dataset_v${getTestCasesVersion()}`;
// System prompt
export const SYSTEM_PROMPT = 'You are a helpful assistant';
export const TOOL_CALLING_BASE_TEMPLATE = `
You are an evaluation assistant evaluating user queries and tool calls to
determine whether a tool was chosen and if it was a right tool.
The tool calls have been generated by a separate agent, and chosen from the list of
tools provided below. It is your job to decide whether that agent chose
the right tool to call.
[BEGIN DATA]
************
[User's previous interaction with the assistant]: {{context}}
[User query]: {{query}}
************
[LLM decided to call these tools]: {{tool_calls}}
[LLM response]: {{llm_response}}
************
[END DATA]
DECISION: [correct or incorrect]
EXPLANATION: [Super short explanation of why the tool choice was correct or incorrect]
Your response must be single word, either "correct" or "incorrect",
and should not contain any text or characters aside from that word.
"correct" means the correct tool call was chosen, the correct parameters
were extracted from the query, the tool call generated is runnable and correct,
and that no outside information not present in the query was used
in the generated query.
"incorrect" means that the chosen tool was not correct
or that the tool signature includes parameter values that don't match
the formats specified in the tool signatures below.
You must not use any outside information or make assumptions.
Base your decision solely on the information provided in [BEGIN DATA] ... [END DATA],
the [Tool Definitions], and the [Reference instructions] (if provided).
Reference instructions are optional and are intended to help you understand the use case and make your decision.
[Reference instructions]: {{reference}}
[Tool definitions]: {{tool_definitions}}
`
export function getRequiredEnvVars(): Record<string, string | undefined> {
return {
PHOENIX_BASE_URL: process.env.PHOENIX_BASE_URL,
PHOENIX_API_KEY: process.env.PHOENIX_API_KEY,
OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY,
OPENROUTER_BASE_URL: process.env.OPENROUTER_BASE_URL,
};
}
// Removes newlines and trims whitespace. Useful for Authorization header values
// because CI secrets sometimes include trailing newlines or quotes.
export function sanitizeHeaderValue(value?: string): string | undefined {
if (value == null) return value;
return value.replace(/[\r\n]/g, '').trim().replace(/^"|"$/g, '');
}
export function validateEnvVars(): boolean {
const envVars = getRequiredEnvVars();
const missing = Object.entries(envVars)
.filter(([, value]) => !value)
.map(([key]) => key);
if (missing.length > 0) {
// eslint-disable-next-line no-console
console.error(`Missing required environment variables: ${missing.join(', ')}`);
return false;
}
return true;
}