regression.test.ts•22.9 kB
/**
* Performance Regression Test Suite
*
* Automated performance tests with budgets to prevent regressions.
* Integrated into CI/CD pipeline to catch performance degradations early.
*/
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
import { config } from 'dotenv';
import { performance } from 'perf_hooks';
// Load environment variables
config();
// Set performance test flag to use mock data when API key is not available
if (!process.env.ATTIO_API_KEY || process.env.E2E_MODE !== 'true') {
process.env.PERFORMANCE_TEST = 'true';
}
// Mock the API client for tests (when not using real API)
if (!process.env.ATTIO_API_KEY || process.env.E2E_MODE !== 'true') {
vi.mock('../../src/api/attio-client', () => ({
getAttioClient: vi.fn(() => ({
post: vi.fn().mockResolvedValue({
data: {
data: {
id: { record_id: 'a1b2c3d4-e5f6-7890-abcd-ef1234567890' },
values: {
name: [{ value: 'Mock Company' }],
},
},
},
}),
get: vi.fn().mockResolvedValue({ data: { data: [] } }),
put: vi.fn().mockResolvedValue({ data: { data: {} } }),
delete: vi.fn().mockResolvedValue({ data: { data: { success: true } } }),
})),
initializeAttioClient: vi.fn(),
isAttioClientInitialized: vi.fn(() => true),
}));
// Mock UniversalSearchService to avoid import path issues
vi.mock('../../src/services/UniversalSearchService.js', () => ({
UniversalSearchService: {
searchRecords: vi.fn().mockResolvedValue([
{
id: { record_id: 'search_001' },
values: {
name: [{ value: 'Mock Search Result 1' }],
domain: [{ value: 'search1.com' }],
},
},
{
id: { record_id: 'search_002' },
values: {
name: [{ value: 'Mock Search Result 2' }],
domain: [{ value: 'search2.com' }],
},
},
]),
},
}));
// Mock resource-specific search functions to prevent real API calls
vi.mock('../../src/objects/companies/index.js', async (importOriginal) => {
const actual = (await importOriginal()) as any;
return {
...actual,
advancedSearchCompanies: vi.fn().mockResolvedValue([
{
id: { record_id: 'comp_001' },
values: {
name: [{ value: 'Mock Company 1' }],
domain: [{ value: 'mock1.com' }],
},
},
{
id: { record_id: 'comp_002' },
values: {
name: [{ value: 'Mock Company 2' }],
domain: [{ value: 'mock2.com' }],
},
},
]),
getCompanyDetails: vi.fn().mockResolvedValue({
id: { record_id: 'comp_001' },
values: {
name: [{ value: 'Mock Company Details' }],
domain: [{ value: 'mock.com' }],
},
}),
updateCompany: vi.fn().mockResolvedValue({
id: { record_id: 'comp_001' },
values: {
name: [{ value: 'Updated Mock Company' }],
domain: [{ value: 'updated-mock.com' }],
},
}),
deleteCompany: vi.fn().mockResolvedValue({ success: true }),
createCompany: vi.fn().mockResolvedValue({
id: { record_id: 'comp_new' },
values: {
name: [{ value: 'New Mock Company' }],
domain: [{ value: 'new-mock.com' }],
},
}),
};
});
vi.mock('../../src/objects/people/index.js', async (importOriginal) => {
const actual = (await importOriginal()) as any;
return {
...actual,
advancedSearchPeople: vi.fn().mockResolvedValue([
{
id: { record_id: 'person_001' },
values: {
name: [{ full_name: 'John Doe' }],
email: [{ value: 'john@mock1.com' }],
},
},
]),
getPersonDetails: vi.fn().mockResolvedValue({
id: { record_id: 'person_001' },
values: {
name: [{ full_name: 'Mock Person Details' }],
email: [{ value: 'mock@person.com' }],
},
}),
updatePerson: vi.fn().mockResolvedValue({
id: { record_id: 'person_001' },
values: {
name: [{ full_name: 'Updated Mock Person' }],
email: [{ value: 'updated@person.com' }],
},
}),
deletePerson: vi.fn().mockResolvedValue({ success: true }),
createPerson: vi.fn().mockResolvedValue({
id: { record_id: 'person_new' },
values: {
name: [{ full_name: 'New Mock Person' }],
email: [{ value: 'new@person.com' }],
},
}),
};
});
vi.mock('../../src/objects/lists.js', async (importOriginal) => {
const actual = (await importOriginal()) as any;
return {
...actual,
searchLists: vi.fn().mockResolvedValue([
{
id: { record_id: 'list_001' },
values: {
name: [{ value: 'Mock List' }],
},
},
]),
};
});
vi.mock('../../src/objects/records/index.js', async (importOriginal) => {
const actual = (await importOriginal()) as any;
return {
...actual,
listObjectRecords: vi.fn().mockResolvedValue([
{
id: { record_id: 'record_001' },
values: {
name: [{ value: 'Mock Record' }],
},
},
]),
};
});
vi.mock('../../src/objects/tasks.js', async (importOriginal) => {
const actual = (await importOriginal()) as any;
return {
...actual,
listTasks: vi.fn().mockResolvedValue([
{
id: { task_id: 'task_001' },
content: 'Mock Task',
status: 'pending',
},
]),
};
});
}
import {
coreOperationsToolConfigs,
advancedOperationsToolConfigs,
} from '../../src/handlers/tool-configs/universal/index.js';
import { UniversalResourceType } from '../../src/handlers/tool-configs/universal/types.js';
import { initializeAttioClient } from '../../src/api/attio-client.js';
import { enhancedPerformanceTracker } from '../../src/middleware/performance-enhanced.js';
// Environment-aware performance budgets (following universal test pattern)
const isCI = process.env.CI === 'true' || process.env.GITHUB_ACTIONS === 'true';
const CI_MULTIPLIER = isCI ? 2.5 : 1; // 2.5x longer timeouts for CI environments
console.log(
`Performance regression testing with ${
isCI ? 'CI' : 'LOCAL'
} budgets (multiplier: ${CI_MULTIPLIER}x)`
);
// Performance test configuration - use environment variables or defaults with CI adjustments
const PERFORMANCE_BUDGETS = {
notFound: Math.round(
parseInt(process.env.PERF_BUDGET_NOT_FOUND || '2000', 10) * CI_MULTIPLIER
),
search: Math.round(
parseInt(process.env.PERF_BUDGET_SEARCH || '3000', 10) * CI_MULTIPLIER
),
create: Math.round(
parseInt(process.env.PERF_BUDGET_CREATE || '3000', 10) * CI_MULTIPLIER
),
update: Math.round(
parseInt(process.env.PERF_BUDGET_UPDATE || '3000', 10) * CI_MULTIPLIER
),
delete: Math.round(
parseInt(process.env.PERF_BUDGET_DELETE || '2000', 10) * CI_MULTIPLIER
),
getDetails: Math.round(
parseInt(process.env.PERF_BUDGET_GET_DETAILS || '2000', 10) * CI_MULTIPLIER
),
batchSmall: Math.round(
parseInt(process.env.PERF_BUDGET_BATCH_SMALL || '5000', 10) * CI_MULTIPLIER
),
batchLarge: Math.round(
parseInt(process.env.PERF_BUDGET_BATCH_LARGE || '10000', 10) * CI_MULTIPLIER
),
};
// Test timeout with buffer - environment-aware like universal test
vi.setConfig({
testTimeout: Math.max(30000, Math.round(30000 * CI_MULTIPLIER)), // At least 30s, more in CI
hookTimeout: Math.round(20000 * CI_MULTIPLIER), // Scaled hook timeout for cleanup
});
// Skip tests if no API key available
const SKIP_TESTS =
!process.env.ATTIO_API_KEY || process.env.SKIP_PERFORMANCE_TESTS === 'true';
describe('Performance Regression Tests', () => {
if (SKIP_TESTS) {
it.skip('Skipping performance tests - no API key or explicitly skipped', () => {});
return;
}
let testRecordId: string | null = null;
const timestamp = Date.now();
beforeAll(async () => {
// Initialize API client
const apiKey = process.env.ATTIO_API_KEY!;
await initializeAttioClient(apiKey);
// Clear performance tracker
enhancedPerformanceTracker.clear();
// Create a test record for performance testing
try {
const createResult = await coreOperationsToolConfigs[
'create-record'
].handler({
resource_type: UniversalResourceType.COMPANIES,
record_data: {
name: `Perf Test Company ${timestamp}`,
website: `https://perftest-${timestamp}.com`,
description: 'Performance regression test record',
},
});
testRecordId = (createResult as any)?.id?.record_id || null;
console.log('Created test record:', testRecordId);
} catch (error: unknown) {
console.error('Failed to create test record:', error);
}
});
afterAll(async () => {
// Clean up test record
if (testRecordId) {
try {
await coreOperationsToolConfigs['delete-record'].handler({
resource_type: UniversalResourceType.COMPANIES,
record_id: testRecordId,
});
console.log('Cleaned up test record:', testRecordId);
} catch (error: unknown) {
console.error('Failed to clean up test record:', error);
}
}
// Generate performance report
const report = enhancedPerformanceTracker.generateReport();
console.log('\n' + report);
});
describe('404 Response Performance', () => {
it('should return 404 for invalid ID format within budget', async () => {
const invalidId = 'invalid-id-format';
const startTime = performance.now();
try {
await coreOperationsToolConfigs['get-record-details'].handler({
resource_type: UniversalResourceType.COMPANIES,
record_id: invalidId,
});
// Should not reach here
expect.fail('Expected error for invalid ID');
} catch (error: any) {
const duration = performance.now() - startTime;
// Verify it's a validation error (enhanced error message format)
expect(error).toMatchObject({
body: expect.objectContaining({
message: expect.stringContaining(
'Invalid record identifier format'
),
}),
});
// Check performance budget
expect(duration).toBeLessThan(PERFORMANCE_BUDGETS.notFound);
// Log for visibility
console.log(
`404 (invalid format) response time: ${duration.toFixed(0)}ms`
);
}
});
it('should return 404 for non-existent valid ID within budget', async () => {
// Valid MongoDB ObjectId format but doesn't exist
const nonExistentId = '507f1f77bcf86cd799439011';
const startTime = performance.now();
try {
await coreOperationsToolConfigs['get-record-details'].handler({
resource_type: UniversalResourceType.COMPANIES,
record_id: nonExistentId,
});
// Should not reach here
expect.fail('Expected error for non-existent ID');
} catch (error: any) {
const duration = performance.now() - startTime;
// Check performance budget
expect(duration).toBeLessThan(PERFORMANCE_BUDGETS.notFound);
// Log for visibility
console.log(
`404 (non-existent) response time: ${duration.toFixed(0)}ms`
);
}
});
it.skip('should cache 404 responses for faster subsequent requests', async () => {
const nonExistentId = '507f1f77bcf86cd799439012';
// First request - should hit API
const firstStart = performance.now();
try {
await coreOperationsToolConfigs['get-record-details'].handler({
resource_type: UniversalResourceType.COMPANIES,
record_id: nonExistentId,
});
} catch (error: unknown) {
// Expected
}
const firstDuration = performance.now() - firstStart;
// Second request - should hit cache
const secondStart = performance.now();
try {
await coreOperationsToolConfigs['get-record-details'].handler({
resource_type: UniversalResourceType.COMPANIES,
record_id: nonExistentId,
});
} catch (error: unknown) {
// Expected
}
const secondDuration = performance.now() - secondStart;
// Second request should be significantly faster or both should be very fast (< 5ms)
// If both are already sub-5ms, the cache is working effectively
const bothVeryFast = firstDuration < 5 && secondDuration < 5;
const secondFaster = secondDuration < firstDuration * 0.8; // More lenient timing
expect(bothVeryFast || secondFaster).toBe(true);
console.log(
`404 cache performance: First: ${firstDuration.toFixed(
0
)}ms, Second: ${secondDuration.toFixed(0)}ms`
);
});
});
describe('Search Operation Performance', () => {
it('should complete search within budget', async () => {
const startTime = performance.now();
const results = await coreOperationsToolConfigs['search-records'].handler(
{
resource_type: UniversalResourceType.COMPANIES,
query: 'test',
limit: 10,
}
);
const duration = performance.now() - startTime;
// Check performance budget
expect(duration).toBeLessThan(PERFORMANCE_BUDGETS.search);
// Verify results
expect(Array.isArray(results)).toBe(true);
console.log(
`Search operation time: ${duration.toFixed(0)}ms (${
(results as any[]).length
} results)`
);
});
it('should handle pagination efficiently', async () => {
const startTime = performance.now();
const results = await coreOperationsToolConfigs['search-records'].handler(
{
resource_type: UniversalResourceType.COMPANIES,
limit: 20,
offset: 0,
}
);
const duration = performance.now() - startTime;
// Check performance budget
expect(duration).toBeLessThan(PERFORMANCE_BUDGETS.search);
console.log(`Paginated search time: ${duration.toFixed(0)}ms`);
});
it('should validate parameters quickly', async () => {
const startTime = performance.now();
try {
await coreOperationsToolConfigs['search-records'].handler({
resource_type: UniversalResourceType.COMPANIES,
limit: -5, // Invalid parameter
});
} catch (error: any) {
const duration = performance.now() - startTime;
// Validation should be very fast (under 100ms)
expect(duration).toBeLessThan(100);
// Schema validation returns specific error message
expect(error).toMatchObject({
body: expect.objectContaining({
message: expect.stringMatching(
/must be at least 1|positive integer/i
),
}),
});
console.log(`Parameter validation time: ${duration.toFixed(0)}ms`);
}
});
});
describe('CRUD Operation Performance', () => {
it('should get record details within budget', async () => {
if (!testRecordId) {
console.warn('Skipping test - no test record available');
return;
}
const startTime = performance.now();
const record = await coreOperationsToolConfigs[
'get-record-details'
].handler({
resource_type: UniversalResourceType.COMPANIES,
record_id: testRecordId,
});
const duration = performance.now() - startTime;
// Check performance budget
expect(duration).toBeLessThan(PERFORMANCE_BUDGETS.getDetails);
expect(record).toBeDefined();
console.log(`Get details time: ${duration.toFixed(0)}ms`);
});
it('should update record within budget', async () => {
if (!testRecordId) {
console.warn('Skipping test - no test record available');
return;
}
const startTime = performance.now();
const updated = await coreOperationsToolConfigs['update-record'].handler({
resource_type: UniversalResourceType.COMPANIES,
record_id: testRecordId,
record_data: {
description: `Updated at ${new Date().toISOString()}`,
},
});
const duration = performance.now() - startTime;
// Check performance budget
expect(duration).toBeLessThan(PERFORMANCE_BUDGETS.update);
expect(updated).toBeDefined();
console.log(`Update operation time: ${duration.toFixed(0)}ms`);
});
it('should create record within budget', async () => {
const startTime = performance.now();
const created = await coreOperationsToolConfigs['create-record'].handler({
resource_type: UniversalResourceType.COMPANIES,
record_data: {
name: `Perf Test Create ${timestamp}`,
website: `https://create-${timestamp}.com`,
},
});
const duration = performance.now() - startTime;
// Check performance budget
expect(duration).toBeLessThan(PERFORMANCE_BUDGETS.create);
// Log the response for debugging
console.log('Create response:', created);
// Only check for record ID if creation succeeded and has proper structure
// When using mocks, the response might be empty or different
if (created && Object.keys(created).length > 0) {
expect(created).toBeDefined();
// Check for either new or legacy response structure
const recordId =
(created as any)?.id?.record_id ||
(created as any)?.record_id ||
(created as any)?.data?.id?.record_id ||
(created as any)?.data?.data?.id?.record_id;
// Only assert on record ID if we're using real API
if (process.env.ATTIO_API_KEY && process.env.E2E_MODE === 'true') {
expect(recordId).toBeDefined();
}
} else {
// Skip test assertions when using mocks or API issues
console.warn(
'Skipping create test assertions - mock or API response issue'
);
}
console.log(`Create operation time: ${duration.toFixed(0)}ms`);
// Clean up (only if we have a real record ID)
if (created && Object.keys(created).length > 0) {
const recordId =
(created as any)?.id?.record_id ||
(created as any)?.record_id ||
(created as any)?.data?.id?.record_id ||
(created as any)?.data?.data?.id?.record_id;
if (
recordId &&
process.env.ATTIO_API_KEY &&
process.env.E2E_MODE === 'true'
) {
try {
await coreOperationsToolConfigs['delete-record'].handler({
resource_type: UniversalResourceType.COMPANIES,
record_id: recordId,
});
} catch (deleteError) {
console.warn('Failed to clean up test record:', deleteError);
}
}
}
});
it('should delete record within budget', async () => {
// Create a record to delete
const toDelete = await coreOperationsToolConfigs['create-record'].handler(
{
resource_type: UniversalResourceType.COMPANIES,
record_data: {
name: `Perf Test Delete ${timestamp}`,
website: `https://delete-${timestamp}.com`,
},
}
);
// Check for either new or legacy response structure
const deleteId =
(toDelete as any)?.id?.record_id ||
(toDelete as any)?.record_id ||
(toDelete as any)?.data?.id?.record_id;
if (!deleteId) {
console.warn('Skipping delete test - failed to create record');
return;
}
const startTime = performance.now();
const result = await coreOperationsToolConfigs['delete-record'].handler({
resource_type: UniversalResourceType.COMPANIES,
record_id: deleteId,
});
const duration = performance.now() - startTime;
// Check performance budget
expect(duration).toBeLessThan(PERFORMANCE_BUDGETS.delete);
expect((result as any).success).toBe(true);
console.log(`Delete operation time: ${duration.toFixed(0)}ms`);
});
});
describe('Performance Statistics', () => {
it('should track timing splits correctly', async () => {
if (!testRecordId) {
console.warn('Skipping test - no test record available');
return;
}
// Perform an operation
await coreOperationsToolConfigs['get-record-details'].handler({
resource_type: UniversalResourceType.COMPANIES,
record_id: testRecordId,
});
// Get statistics
const stats =
enhancedPerformanceTracker.getStatistics('get-record-details');
expect(stats).toBeDefined();
expect(stats.count).toBeGreaterThan(0);
expect((stats as any).timing.p95).toBeDefined();
expect((stats as any).apiTiming.average).toBeDefined();
expect((stats as any).overhead.average).toBeDefined();
console.log('Performance Statistics:', {
operations: stats.count,
avgTotal: (stats as any).timing.average.toFixed(0) + 'ms',
p95Total: (stats as any).timing.p95.toFixed(0) + 'ms',
avgAPI: (stats as any).apiTiming.average.toFixed(0) + 'ms',
avgOverhead: (stats as any).overhead.average.toFixed(0) + 'ms',
});
});
it('should have acceptable p95 and p99 latencies', async () => {
const stats = enhancedPerformanceTracker.getStatistics();
if (stats && (stats as any).count > 0) {
// P95 should be under 5 seconds
expect((stats as any).timing.p95).toBeLessThan(5000);
// P99 should be under 10 seconds
expect((stats as any).timing.p99).toBeLessThan(10000);
console.log(
`Latency percentiles - P50: ${(stats as any).timing.p50.toFixed(
0
)}ms, P95: ${(stats as any).timing.p95.toFixed(0)}ms, P99: ${(
stats as any
).timing.p99.toFixed(0)}ms`
);
}
});
});
describe('Performance Alerts', () => {
it('should generate alerts for operations exceeding budget', async () => {
// Intentionally trigger a slow operation (search with large limit)
try {
await coreOperationsToolConfigs['search-records'].handler({
resource_type: UniversalResourceType.COMPANIES,
limit: 100,
});
} catch (error: unknown) {
// Might fail due to limit validation
}
// Check if any alerts were generated
const report = enhancedPerformanceTracker.generateReport();
console.log(
'Performance alerts check:',
report.includes('Budget Violations')
);
});
});
});