NCP - Natural Context Provider

/** * Regression snapshot tests for CLI commands * These tests capture expected outputs to detect unintended changes */ import { execSync } from 'child_process'; import * as path from 'path'; import * as fs from 'fs'; import * as os from 'os'; import { fileURLToPath } from 'url'; import { describe, it, expect, beforeAll, afterAll, jest } from '@jest/globals'; // Keep a high timeout for the real operations we need jest.setTimeout(120000); // Increase global timeout jest.retryTimes(3); // Allow test retries for flaky tests // Create __dirname equivalent for ESM const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); describe.skip('CLI Command Regression Tests', () => { // NOTE: Skipped - Complex integration tests that require careful environment setup // and may be flaky in CI/CD environments. These tests are better handled through // E2E test suite or manual testing. const CLI_PATH = path.join(__dirname, '..', 'dist', 'index.js'); let testConfigDir: string; beforeAll(async () => { console.error('Setting up regression test suite...'); try { // Create isolated test config directory testConfigDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ncp-test-')); const profilesDir = path.join(testConfigDir, 'profiles'); fs.mkdirSync(profilesDir, { recursive: true }); // Create a minimal test profile with NO MCPs for faster, more reliable tests const testProfile = { name: 'test-regression', description: 'Isolated test profile for regression tests', mcpServers: {} }; fs.writeFileSync( path.join(profilesDir, 'all.json'), JSON.stringify(testProfile, null, 2) ); console.error(`Created test config at: ${testConfigDir}`); console.error('Test profile configured with git mock server'); } catch (err) { console.error('Failed to setup test config:', err); throw err; } }); afterAll(async () => { console.error('Cleaning up regression test suite...'); try { // Clean up test config directory if (testConfigDir && fs.existsSync(testConfigDir)) { fs.rmSync(testConfigDir, { recursive: true, force: true }); console.error(`Removed test config: ${testConfigDir}`); } // Give a moment for cleanup to complete await new Promise(resolve => setTimeout(resolve, 500)); } catch (err) { console.error('Error during cleanup:', err); // Don't throw in afterAll } }); // Helper to run CLI commands with isolated test config function runCommand(args: string): string { try { const result = execSync(`node ${CLI_PATH} ${args}`, { env: { ...process.env, FORCE_COLOR: '0', // Disable colors for consistent snapshots NCP_CONFIG_PATH: testConfigDir // Use isolated test config }, encoding: 'utf-8' }); return result.toString(); } catch (error: any) { return error.stdout?.toString() || error.message; } } // Helper to normalize output for comparison function normalizeOutput(output: string): string { return output .replace(/\d+\.\d+\.\d+/g, 'X.X.X') // Normalize version numbers .replace(/\d+ tools?/g, 'N tools') // Normalize tool counts .replace(/\d+ MCPs?/g, 'N MCPs') // Normalize MCP counts .replace(/\(\d+%\s+match\)/g, '(N% match)') // Normalize match percentages .replace(/\/Users\/[^\s]+/g, '/path/to/file') // Normalize file paths .trim(); } describe('find command', () => { // Skip find tests for now - they're slow with empty profiles and can timeout // These should be run manually or in CI with proper test MCPs configured test.skip('should execute find command without errors', async () => { const output = runCommand('find git-commit --depth 0'); const normalized = normalizeOutput(output); // Command should execute successfully (may or may not find tools depending on user's config) expect(normalized).toBeDefined(); expect(normalized).not.toContain('[NCP ERROR]'); expect(normalized).not.toContain('undefined'); expect(normalized).not.toContain('TypeError'); // Should either find tools or show "No tools found" message const hasResults = normalized.includes('Found tools for'); const hasNoResultsMessage = /No tools found/i.test(normalized); expect(hasResults || hasNoResultsMessage).toBe(true); // If tools were found, check they don't have double-prefixed descriptions if (hasResults) { expect(normalized).not.toMatch(/(\w+):\s*\1:/); // No repeated prefixes } }); test.skip('should find filesystem tools', () => { const output = runCommand('find "list files" --depth 0'); const normalized = normalizeOutput(output); // Should find relevant tools expect(normalized).toMatch(/Found tools|No tools found/); }); }); describe('list command', () => { test('should list all profiles', () => { const output = runCommand('list'); const normalized = normalizeOutput(output); // Should show profile structure expect(normalized).toContain('📦'); // Summary line was removed from the output expect(normalized).toContain('Profiles ▶ MCPs'); }); test('should filter non-empty profiles', () => { const output = runCommand('list --non-empty'); const normalized = normalizeOutput(output); // Should not show empty profiles expect(normalized).not.toContain('(empty)'); }); }); describe('help command', () => { test('should show proper help structure', () => { const output = runCommand('help'); const normalized = normalizeOutput(output); // Should have main sections expect(normalized).toContain('Natural Context Provider'); expect(normalized).toContain('Commands:'); expect(normalized).toContain('Quick Start:'); expect(normalized).toContain('Examples:'); // Should have core commands expect(normalized).toContain('find'); expect(normalized).toContain('add'); expect(normalized).toContain('list'); expect(normalized).toContain('run'); }); }); describe('Critical functionality checks', () => { // Skip find-based tests - they require fully configured MCPs and can timeout test.skip('single-word queries should work', () => { const output = runCommand('find git-commit --depth 0'); // The command should execute without errors // It may or may not find tools depending on environment expect(output).toBeDefined(); expect(output).not.toContain('[NCP ERROR]'); expect(output).not.toContain('undefined'); expect(output).not.toContain('TypeError'); }); test.skip('probe failures should not leak to CLI', () => { const output = runCommand('find test-query'); expect(output).not.toContain('[NCP ERROR]'); expect(output).not.toContain('Probe timeout'); }); }); }); // Snapshot comparison test describe('Output Snapshot Comparison', () => { const SNAPSHOT_DIR = path.join(__dirname, 'snapshots'); beforeAll(() => { if (!fs.existsSync(SNAPSHOT_DIR)) { fs.mkdirSync(SNAPSHOT_DIR); } }); function compareSnapshot(command: string, name: string, testConfigPath?: string) { const env: any = { ...process.env, FORCE_COLOR: '0' }; if (testConfigPath) { env.NCP_CONFIG_PATH = testConfigPath; } const output = execSync(`node ${path.join(__dirname, '..', 'dist', 'index.js')} ${command}`, { env, encoding: 'utf-8' }).toString(); const normalized = output .replace(/\d+\.\d+\.\d+/g, 'X.X.X') .replace(/\d+ tools?/g, 'N tools') .replace(/\(\d+%\s+match\)/g, '(N% match)') .trim(); const snapshotFile = path.join(SNAPSHOT_DIR, `${name}.snap`); if (process.env.UPDATE_SNAPSHOTS === 'true') { fs.writeFileSync(snapshotFile, normalized); console.log(`Updated snapshot: ${name}`); } else if (fs.existsSync(snapshotFile)) { const expected = fs.readFileSync(snapshotFile, 'utf-8'); if (expected !== normalized) { console.log('Expected:', expected.substring(0, 200)); console.log('Received:', normalized.substring(0, 200)); throw new Error(`Snapshot mismatch for ${name}. Run with UPDATE_SNAPSHOTS=true to update.`); } } else { fs.writeFileSync(snapshotFile, normalized); console.log(`Created new snapshot: ${name}`); } } test.skip('find command snapshot', () => { compareSnapshot('find git-commit --depth 0 --limit 3', 'find-git-commit'); }); test.skip('list command snapshot', () => { compareSnapshot('list --non-empty', 'list-non-empty'); }); test.skip('help command snapshot', () => { compareSnapshot('help', 'help'); }); });