advanced-search-qa.e2e.test.ts•12.6 kB
/**
* Advanced Search QA Test Suite for Issue #579
*
* This test reproduces all the issues identified in GitHub issue #579:
* 1. Personal name formatting bug (all people show as "Unnamed")
* 2. Poor error handling with unnecessary retries on 400 errors
* 3. Missing client-side validation for invalid attributes
* 4. Operator validation missing for attribute types
* 5. Performance issues from retry logic
*
* Each test is designed to validate the specific issue and verify fixes.
*/
import { describe, it, beforeAll, afterAll, expect } from 'vitest';
import { MCPTestClient } from 'mcp-test-client';
import type { ToolResult } from '@modelcontextprotocol/sdk/types.js';
describe('Issue #579: Advanced Search Filter QA Tests', () => {
let client: MCPTestClient;
beforeAll(async () => {
client = new MCPTestClient({
serverCommand: 'node',
serverArgs: ['./dist/index.js'],
});
await client.init();
});
afterAll(async () => {
await client.cleanup();
});
describe('🔍 Issue 4: Personal Name Formatting Bug', () => {
it('should display actual person names instead of "Unnamed" in search results', async () => {
const startTime = performance.now();
await client.assertToolCall(
'advanced-search',
{
resource_type: 'people',
filters: {
filters: [
{
attribute: { slug: 'name' },
condition: 'contains',
value: 'John',
},
],
},
limit: 5,
},
(result: ToolResult) => {
const endTime = performance.now();
const duration = endTime - startTime;
console.log(
'🔍 Personal Name Search Test Result:',
JSON.stringify(result, null, 2)
);
console.log(`⏱️ Execution time: ${duration.toFixed(2)}ms`);
expect(result.isError).toBeFalsy();
if (result.content && result.content.length > 0) {
const content = result.content[0];
if ('text' in content) {
console.log('📄 Result text:', content.text);
// ISSUE: This should NOT contain "Unnamed" for actual people with names
// Before fix: All results show as "1. Unnamed", "2. Unnamed", etc.
// After fix: Should show actual names like "1. John Santarpi", "2. John Smith", etc.
const hasUnnamed = content.text.includes('Unnamed');
const hasActualNames = content.text.match(
/\d+\.\s+[A-Z][a-z]+\s+[A-Z][a-z]+/
);
if (hasUnnamed && !hasActualNames) {
console.warn(
'❌ BUG CONFIRMED: All people showing as "Unnamed" instead of actual names'
);
console.warn(
'💡 This indicates the personal name formatting bug from Issue #579'
);
} else if (!hasUnnamed && hasActualNames) {
console.log('✅ FIXED: People showing with actual names');
}
// For now, we expect this test to show the bug until we fix it
expect(typeof content.text).toBe('string');
}
}
// Performance check - should be under 3000ms for valid requests
expect(duration).toBeLessThan(3000);
}
);
});
it('should handle personal name attribute structure correctly', async () => {
// Test to verify we can get detailed info that shows the personal name structure
await client.assertToolCall(
'search-records',
{
resource_type: 'people',
limit: 1,
},
(result: ToolResult) => {
console.log(
'🔍 Personal Name Structure Test:',
JSON.stringify(result, null, 2)
);
expect(result.isError).toBeFalsy();
if (result.content && result.content.length > 0) {
const content = result.content[0];
if ('text' in content) {
// This test helps us understand the structure of person records
console.log('📄 Person record structure:', content.text);
}
}
}
);
});
});
describe('⚠️ Issue 2: Poor Error Handling and Performance', () => {
it('should NOT retry 400 validation errors and should complete quickly', async () => {
const startTime = performance.now();
await client.assertToolCall(
'advanced-search',
{
resource_type: 'people',
filters: {
filters: [
{
attribute: { slug: 'first_name' }, // Invalid attribute - should be 'name'
condition: 'contains',
value: 'John',
},
],
},
},
(result: ToolResult) => {
const endTime = performance.now();
const duration = endTime - startTime;
console.log(
'⚠️ Invalid Attribute Test Result:',
JSON.stringify(result, null, 2)
);
console.log(`⏱️ Execution time: ${duration.toFixed(2)}ms`);
// ISSUE: Before fix, this would take 6000-8000ms due to retries
// After fix: Should complete quickly (under 1000ms) with immediate error
if (duration > 5000) {
console.warn(
'❌ PERFORMANCE BUG CONFIRMED: Taking too long due to unnecessary retries'
);
console.warn(
'💡 This indicates retry logic is retrying 400 errors from Issue #579'
);
} else if (duration < 1000) {
console.log('✅ PERFORMANCE FIXED: Quick response without retries');
}
// For debugging - check what error we get
if (result.isError) {
console.log('📄 Error details:', result);
}
// The request should complete within reasonable time for API calls
expect(duration).toBeLessThan(10000); // Allow for network latency and API response time
}
);
});
it('should handle invalid operators gracefully and quickly', async () => {
const startTime = performance.now();
await client.assertToolCall(
'advanced-search',
{
resource_type: 'people',
filters: {
filters: [
{
attribute: { slug: 'name' },
condition: 'invalid_operator', // Invalid condition
value: 'John',
},
],
},
},
(result: ToolResult) => {
const endTime = performance.now();
const duration = endTime - startTime;
console.log(
'⚠️ Invalid Operator Test Result:',
JSON.stringify(result, null, 2)
);
console.log(`⏱️ Execution time: ${duration.toFixed(2)}ms`);
// Should complete quickly with a clear error message
if (duration > 5000) {
console.warn(
'❌ PERFORMANCE BUG: Invalid operator taking too long'
);
}
expect(duration).toBeLessThan(10000); // Allow for network latency and API response time
}
);
});
});
describe('🔒 Issue 5: Attribute-Type-Specific Operator Validation', () => {
it('should validate operators against select field types before API call', async () => {
const startTime = performance.now();
await client.assertToolCall(
'advanced-search',
{
resource_type: 'companies',
filters: {
filters: [
{
attribute: { slug: 'employee_range' }, // Select field - only supports $eq
condition: 'contains', // Invalid for select fields
value: '50-100',
},
],
},
},
(result: ToolResult) => {
const endTime = performance.now();
const duration = endTime - startTime;
console.log(
'🔒 Operator Validation Test Result:',
JSON.stringify(result, null, 2)
);
console.log(`⏱️ Execution time: ${duration.toFixed(2)}ms`);
// ISSUE: Before fix, this would take 8920ms (197.3% over budget) due to retries
// After fix: Should validate client-side and error immediately
if (duration > 7000) {
console.warn(
'❌ OPERATOR VALIDATION BUG: Select field operator validation missing'
);
console.warn(
'💡 This indicates lack of client-side operator validation from Issue #579'
);
}
expect(duration).toBeLessThan(10000);
}
);
});
});
describe('✅ Issue 1: Valid Requests Should Work Efficiently', () => {
it('should handle valid personal name search with good performance', async () => {
const startTime = performance.now();
await client.assertToolCall(
'advanced-search',
{
resource_type: 'people',
filters: {
filters: [
{
attribute: { slug: 'name' },
condition: 'contains',
value: 'John',
},
],
},
limit: 5,
},
(result: ToolResult) => {
const endTime = performance.now();
const duration = endTime - startTime;
console.log(
'✅ Valid Search Test Result:',
JSON.stringify(result, null, 2)
);
console.log(`⏱️ Execution time: ${duration.toFixed(2)}ms`);
// This should work well (as mentioned in the issue, valid requests perform fine)
expect(result.isError).toBeFalsy();
expect(duration).toBeLessThan(5000);
if (result.content && result.content.length > 0) {
const content = result.content[0];
if ('text' in content) {
console.log('📄 Valid search results:', content.text);
}
}
}
);
});
it('should handle multi-filter AND logic efficiently', async () => {
const startTime = performance.now();
await client.assertToolCall(
'advanced-search',
{
resource_type: 'companies',
filters: {
filters: [
{
attribute: { slug: 'name' },
condition: 'contains',
value: 'Tech',
},
{
attribute: { slug: 'industry' },
condition: 'equals',
value: 'Technology',
},
],
},
limit: 5,
},
(result: ToolResult) => {
const endTime = performance.now();
const duration = endTime - startTime;
console.log(
'✅ Multi-Filter Test Result:',
JSON.stringify(result, null, 2)
);
console.log(`⏱️ Execution time: ${duration.toFixed(2)}ms`);
// This should work well as mentioned in Issue #579 Test 3
expect(duration).toBeLessThan(10000); // Allow for network latency and API response time
if (result.content && result.content.length > 0) {
const content = result.content[0];
if ('text' in content) {
console.log('📄 Multi-filter results:', content.text);
}
}
}
);
});
});
describe('📊 Performance Summary', () => {
it('should provide performance analysis summary', () => {
console.log(`
📊 ISSUE #579 PERFORMANCE ANALYSIS SUMMARY
Expected Performance Patterns:
- ✅ Valid requests: 830ms - 2955ms (acceptable)
- ❌ Invalid requests: 6000ms - 8920ms (terrible due to retries)
Key Issues to Fix:
1. 🏷️ Personal names showing as "Unnamed" instead of actual names
2. ⚠️ 400 errors being retried unnecessarily (causing 2-3x performance penalty)
3. 🔍 Missing client-side attribute validation
4. 🔒 Missing operator/attribute type validation
5. ⏱️ Performance budget: Should stay under 3000ms for all requests
Fix Priority:
- P1: Stop retrying 400 errors (immediate performance fix)
- P1: Fix personal name formatting (user-visible bug)
- P2: Add client-side validation (prevent unnecessary API calls)
- P2: Add operator validation (prevent type mismatches)
`);
expect(true).toBe(true); // Just a summary test
});
});
});