coderswap_validate_search
Test search quality and coverage by running validation queries to verify knowledge base search performance and identify gaps in content retrieval.
Instructions
Run validation queries to test search quality and coverage (non-DSL quality check)
Input Schema
| Name | Required | Description | Default |
|---|---|---|---|
| project_id | Yes | ||
| test_queries | No | ||
| run_full_suite | No |
Input Schema (JSON Schema)
{
"properties": {
"project_id": {
"minLength": 1,
"type": "string"
},
"run_full_suite": {
"default": false,
"type": "boolean"
},
"test_queries": {
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"project_id"
],
"type": "object"
}
Implementation Reference
- src/index.ts:584-630 (handler)MCP tool handler for 'coderswap_validate_search'. Invokes CoderSwapClient.testSearchQuality and generates a formatted search quality report with aggregate metrics.async ({ project_id, test_queries, run_full_suite = false }) => { try { log('debug', 'Testing search quality', { project_id, run_full_suite }) const report = await client.testSearchQuality({ project_id, test_queries, run_full_suite }) const output = { queries_tested: report.aggregate.queries_tested, average_top_score: report.aggregate.average_top_score, zero_result_queries: report.aggregate.zero_result_queries } const avgScore = (report.aggregate.average_top_score * 100).toFixed(1) const zeroResults = report.aggregate.zero_result_queries.length let summary = `Search Quality Report\n${'='.repeat(40)}\n` summary += `Queries tested: ${report.aggregate.queries_tested}\n` summary += `Average top score: ${avgScore}%\n` summary += `Zero-result queries: ${zeroResults}\n\n` if (report.results.length > 0) { summary += 'Top Results:\n' report.results.slice(0, 3).forEach(r => { const score = (r.topScore * 100).toFixed(1) summary += ` • "${r.query}" → ${score}% (${r.count} results)\n` }) } log('info', `Search quality test completed: ${report.aggregate.queries_tested} queries`) return { content: [{ type: 'text', text: summary }], structuredContent: output } } catch (error) { log('error', 'Search quality test failed', { project_id, error: error instanceof Error ? error.message : error }) return { content: [{ type: 'text', text: `✗ Search quality test failed: ${error instanceof Error ? error.message : 'Unknown error'}` }], isError: true } } }
- src/coderswap-client.ts:149-185 (helper)Core implementation of search validation logic in CoderSwapClient. Uses predefined test queries for full suite or custom queries, performs hybrid searches, sorts by score, and computes aggregate statistics including average top score and zero-result queries.async testSearchQuality(input: TestSearchQualityInput) { const queries = input.run_full_suite ? [ 'what is hybrid search', 'how to implement rag', 'error troubleshooting vector search', 'bm25 algorithm', 'semantic vs keyword search' ] : input.test_queries || [] const uniqueQueries = Array.from(new Set(queries)) if (uniqueQueries.length === 0) { throw new Error('No queries provided for search quality test') } const results = [] as Array<{ query: string; topScore: number; count: number; items: SearchResult[] }> for (const query of uniqueQueries) { const response = await this.search({ project_id: input.project_id, query }) const sorted = response.results.sort((a, b) => (b.score ?? 0) - (a.score ?? 0)) results.push({ query, topScore: sorted[0]?.score ?? 0, count: sorted.length, items: sorted }) } const aggregate = { queries_tested: results.length, average_top_score: results.reduce((sum, item) => sum + (item.topScore || 0), 0) / Math.max(results.length, 1), zero_result_queries: results.filter((item) => item.count === 0).map((item) => item.query) } return { aggregate, results } }
- src/index.ts:570-582 (schema)Input and output schema definitions for the 'coderswap_validate_search' MCP tool using Zod.{ title: 'Validate CoderSwap Search Quality', description: 'Run validation queries to test search quality and coverage (non-DSL quality check)', inputSchema: { project_id: z.string().min(1, 'project_id is required'), test_queries: z.array(z.string()).optional(), run_full_suite: z.boolean().default(false) }, outputSchema: { queries_tested: z.number(), average_top_score: z.number(), zero_result_queries: z.array(z.string()) }
- src/index.ts:568-631 (registration)Registration of the 'coderswap_validate_search' tool with the MCP server.server.registerTool( 'coderswap_validate_search', { title: 'Validate CoderSwap Search Quality', description: 'Run validation queries to test search quality and coverage (non-DSL quality check)', inputSchema: { project_id: z.string().min(1, 'project_id is required'), test_queries: z.array(z.string()).optional(), run_full_suite: z.boolean().default(false) }, outputSchema: { queries_tested: z.number(), average_top_score: z.number(), zero_result_queries: z.array(z.string()) } }, async ({ project_id, test_queries, run_full_suite = false }) => { try { log('debug', 'Testing search quality', { project_id, run_full_suite }) const report = await client.testSearchQuality({ project_id, test_queries, run_full_suite }) const output = { queries_tested: report.aggregate.queries_tested, average_top_score: report.aggregate.average_top_score, zero_result_queries: report.aggregate.zero_result_queries } const avgScore = (report.aggregate.average_top_score * 100).toFixed(1) const zeroResults = report.aggregate.zero_result_queries.length let summary = `Search Quality Report\n${'='.repeat(40)}\n` summary += `Queries tested: ${report.aggregate.queries_tested}\n` summary += `Average top score: ${avgScore}%\n` summary += `Zero-result queries: ${zeroResults}\n\n` if (report.results.length > 0) { summary += 'Top Results:\n' report.results.slice(0, 3).forEach(r => { const score = (r.topScore * 100).toFixed(1) summary += ` • "${r.query}" → ${score}% (${r.count} results)\n` }) } log('info', `Search quality test completed: ${report.aggregate.queries_tested} queries`) return { content: [{ type: 'text', text: summary }], structuredContent: output } } catch (error) { log('error', 'Search quality test failed', { project_id, error: error instanceof Error ? error.message : error }) return { content: [{ type: 'text', text: `✗ Search quality test failed: ${error instanceof Error ? error.message : 'Unknown error'}` }], isError: true } } } )