/**
* CLI Integration Tests
*
* Tests the actual CLI commands end-to-end without user interaction.
* Can be run in CI/CD pipelines.
*/
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
describe('CLI Integration Tests', () => {
let testConfigPath: string;
const CLI_PATH = path.join(__dirname, '../../dist/index.js');
beforeEach(() => {
// Create isolated test environment
testConfigPath = fs.mkdtempSync(path.join(os.tmpdir(), 'ncp-cli-test-'));
});
afterEach(() => {
// Cleanup
if (fs.existsSync(testConfigPath)) {
fs.rmSync(testConfigPath, { recursive: true, force: true });
}
});
const parseArgs = (input: string): string[] => {
const args: string[] = [];
let current = '';
let inSingle = false;
let inDouble = false;
for (let i = 0; i < input.length; i++) {
const char = input[i];
if (char === '\\') {
const next = input[i + 1];
if (next !== undefined) {
current += next;
i++;
} else {
current += char;
}
continue;
}
if (char === "'" && !inDouble) {
inSingle = !inSingle;
continue;
}
if (char === '"' && !inSingle) {
inDouble = !inDouble;
continue;
}
if (char === ' ' && !inSingle && !inDouble) {
if (current.length > 0) {
args.push(current);
current = '';
}
continue;
}
current += char;
}
if (current.length > 0) {
args.push(current);
}
return args;
};
const runCLI = (args: string, opts: { env?: Record<string, string> } = {}) => {
const argArray = parseArgs(args);
const childEnv = { ...process.env };
// Keep isolated-vm disabled during tests to avoid platform-specific segfaults.
// The subprocess/worker-thread fallbacks are exercised instead.
childEnv.NCP_DISABLE_ISOLATED_VM = 'true';
const result = spawnSync('node', [CLI_PATH, ...argArray], {
encoding: 'utf-8',
env: {
...childEnv,
NCP_CONFIG_PATH: testConfigPath,
NCP_CONFIRM_BEFORE_RUN: 'false', // Disable confirmation dialogs for automated tests
...opts.env
},
timeout: 30000
});
const exitCode =
result.status !== null && result.status !== undefined
? result.status
: result.signal
? 1
: result.error
? 1
: 0;
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode,
signal: result.signal || null
};
};
describe('Discovery (find command)', () => {
test('should find scheduler tools', () => {
const result = runCLI('find scheduler --depth 0 --limit 5');
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain('scheduler');
expect(result.stdout).toMatch(/(Found tools|No tools found)/);
}, 60000);
test('should find with custom confidence threshold', () => {
const result = runCLI('find scheduler --confidence-threshold 0.5 --depth 0');
expect(result.exitCode).toBe(0);
expect(result.stdout).toMatch(/(Found tools|No tools found)/);
}, 60000);
// TODO: Fix pagination test - search needs to return results for pagination to trigger
test.skip('should handle pagination', () => {
const result = runCLI('find scheduler --page 1 --limit 3 --depth 0');
expect(result.exitCode).toBe(0);
expect(result.stdout).toMatch(/Page \d+ of \d+/);
}, 60000);
});
describe('Tool Validation', () => {
// TODO: Fix scheduler:validate tool - currently returns exit code 1
test.skip('should validate scheduler tool', () => {
const result = runCLI('run scheduler:validate --params \'{"tool":"schedule","arguments":{"name":"test","schedule":"every 5 minutes","tool":"example:tool","parameters":{}}}\'');
expect(result.exitCode).toBe(0);
// Should complete without errors
}, 60000);
});
describe('Error Handling', () => {
test('should handle unknown tool gracefully', () => {
const result = runCLI('run unknown:tool');
expect(result.exitCode).not.toBe(0);
expect(result.stdout + result.stderr).toMatch(/not found|error/i);
}, 30000);
test('should handle invalid parameters gracefully', () => {
const result = runCLI('run scheduler:validate --params \'invalid json\'');
expect(result.exitCode).not.toBe(0);
}, 30000);
});
describe('Help and Version', () => {
test('should display version', () => {
const result = runCLI('--version');
expect(result.exitCode).toBe(0);
expect(result.stdout).toMatch(/\d+\.\d+\.\d+/); // Version format
}, 10000);
test('should display help', () => {
const result = runCLI('--help');
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain('Usage');
}, 10000);
test('should display find command help', () => {
const result = runCLI('find --help');
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain('find');
}, 10000);
});
describe('Profile Management', () => {
test('should work with default profile', () => {
const result = runCLI('find scheduler --depth 0 --limit 3');
expect(result.exitCode).toBe(0);
expect(result.stdout).toMatch(/(Found tools|No tools found)/);
}, 60000);
});
describe('Environment Variable Handling', () => {
test('should respect NCP_DEBUG environment variable', () => {
const result = runCLI('find scheduler --depth 0 --limit 1', {
env: { NCP_DEBUG: 'true' }
});
// Debug mode should still work
expect(result.exitCode).toBe(0);
}, 60000);
// TODO: Fix CLI exit code handling for environment variables
test.skip('should disable confirmation prompts in CI', () => {
const result = runCLI('run scheduler:list_schedules', {
env: { NCP_CONFIRM_BEFORE_RUN: 'false' }
});
// Should execute without waiting for user input
expect(result.exitCode).toBe(0);
}, 60000);
});
describe('Smart Parameter Parsing (NEW)', () => {
test('should accept named parameters with --flag value format', () => {
// Test with a tool that accepts parameters
// Using scheduler:list with optional filters
const result = runCLI('run scheduler list_schedules --params \'{"limit": 5}\' --no-prompt');
// Should not error on unknown flag (graceful handling)
// The tool will either accept or reject based on its schema
expect(result.exitCode >= 0).toBe(true);
}, 30000);
test('should accept --key=value format for parameters', () => {
// Test with key=value format
const result = runCLI('run scheduler list_schedules --params \'{"limit": 5}\' --no-prompt');
// Should handle key=value format
expect(result.exitCode >= 0).toBe(true);
}, 30000);
test('should handle mixed positional and named parameters', () => {
// For tools that support positional args
const result = runCLI('run scheduler list_schedules --no-prompt');
// Should complete without errors
expect(result.exitCode >= 0).toBe(true);
}, 30000);
test('should report error for missing required parameters', () => {
// Test a tool that requires parameters but none are provided
const result = runCLI('run scheduler:validate --no-prompt');
// Should fail or report missing params
// The exact behavior depends on the tool's requirements
expect(result.stdout + result.stderr).toMatch(/required|missing|parameter|error/i);
}, 30000);
});
describe('Tool Naming Convention (NEW)', () => {
test('should support new "mcp tool" format', () => {
// New format: ncp run scheduler list_schedules
const result = runCLI('run scheduler list_schedules --no-prompt');
// Should recognize and execute
expect(result.exitCode >= 0).toBe(true);
}, 30000);
test('should still support legacy "mcp:tool" format', () => {
// Legacy format: ncp run scheduler:list_schedules
const result = runCLI('run scheduler:list_schedules --no-prompt');
// Should recognize and execute
expect(result.exitCode >= 0).toBe(true);
}, 30000);
test('should handle ambiguous tool names with new syntax', () => {
// If tool name has spaces or special characters, both formats should work
const result1 = runCLI('run scheduler list_schedules --no-prompt');
const result2 = runCLI('run scheduler:list_schedules --no-prompt');
// Both formats should behave the same
expect(result1.exitCode).toBe(result2.exitCode);
}, 60000);
});
describe('Backward Compatibility', () => {
test('should still support --params JSON format (legacy)', () => {
// Legacy: using --params with JSON
const result = runCLI('run scheduler:list_schedules --params \'{}\'');
// Should still work
expect(result.exitCode >= 0).toBe(true);
}, 30000);
test('should still support interactive mode when no args provided', () => {
// Existing behavior: no arguments should trigger interactive mode
// We'll just check that the command at least attempts to run
const result = runCLI('run scheduler --no-prompt 2>&1 || true');
// Should attempt execution or report an error (not crash)
expect(result.stdout + result.stderr).toMatch(/tool|error|parameter|missing/i);
}, 30000);
});
describe('Fuzzy Matching (Phase 2)', () => {
test('should suggest command when unknown command is typed', () => {
const result = runCLI('fnd --help');
// Should suggest 'find' for typo 'fnd'
expect(result.stderr + result.stdout).toMatch(/Did you mean|find/i);
}, 10000);
test.skip('should handle multiple typo suggestions', () => {
const result = runCLI('lst');
// Should not crash, should suggest alternatives
expect((result.stdout || '') + (result.stderr || '')).toMatch(/unknown|Did you mean|command/i);
}, 10000);
test.skip('should show help prompt for unknown commands', () => {
const result = runCLI('invalidcmd');
// Should direct user to help
expect((result.stdout || '') + (result.stderr || '')).toMatch(/help|unknown/i);
}, 10000);
});
describe('Status Indicators (Phase 2)', () => {
test('should display MCPs health status in find results', () => {
const result = runCLI('find scheduler --depth 0 --limit 1');
// Should show health status (MCPs: X/X healthy)
expect(result.stdout).toMatch(/healthy|MCPs/i);
}, 60000);
test('should show status indicators in list output', () => {
const result = runCLI('list');
// Output should contain status information
expect(result.exitCode).toBe(0);
expect(result.stdout.length).toBeGreaterThan(0);
}, 30000);
});
describe('Doctor Command (Phase 3)', () => {
test('should run system diagnostics', () => {
const result = runCLI('doctor');
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain('Diagnostics');
}, 30000);
test('should display health summary', () => {
const result = runCLI('doctor');
expect(result.exitCode).toBe(0);
expect(result.stdout).toMatch(/Summary:/);
}, 30000);
test('should show status indicators for each check', () => {
const result = runCLI('doctor');
expect(result.exitCode).toBe(0);
expect(result.stdout).toMatch(/(healthy MCPs|No MCPs configured)/i);
}, 30000);
test('should verify Node.js version check', () => {
const result = runCLI('doctor');
expect(result.exitCode).toBe(0);
expect(result.stdout).toMatch(/System check complete/i);
}, 30000);
test('should verify npm availability check', () => {
const result = runCLI('doctor');
expect(result.exitCode).toBe(0);
expect(result.stdout).toMatch(/System check complete/i);
}, 30000);
test('should check working directory', () => {
const result = runCLI('doctor');
expect(result.exitCode).toBe(0);
expect(result.stdout).toMatch(/System check complete/i);
}, 30000);
test('should check profile directory status', () => {
const result = runCLI('doctor');
expect(result.exitCode).toBe(0);
expect(result.stdout).toMatch(/Profile:\s+\w+/i);
}, 30000);
test('should check cache system status', () => {
const result = runCLI('doctor');
expect(result.exitCode).toBe(0);
expect(result.stdout).toMatch(/(healthy MCPs|No MCPs configured)/i);
}, 30000);
test('should handle doctor with MCP name argument', () => {
const result = runCLI('doctor scheduler');
// Should handle gracefully, even if MCP doesn't exist
expect(result.exitCode >= 0).toBe(true);
}, 30000);
});
describe('Output Formatting (Phase 3)', () => {
test('should format find results with proper output', () => {
const result = runCLI('find scheduler --depth 0 --limit 1');
expect(result.exitCode).toBe(0);
// Results should be properly formatted
expect(result.stdout.length).toBeGreaterThan(0);
}, 60000);
test('should handle list output formatting', () => {
const result = runCLI('list');
expect(result.exitCode).toBe(0);
// List should be readable
expect(result.stdout).toBeTruthy();
}, 30000);
});
describe('Code Execution Command', () => {
test('should display code command help', () => {
const result = runCLI('code --help');
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain('Execute TypeScript');
expect(result.stdout).toContain('MCP');
expect(result.stdout).toContain('--file');
expect(result.stdout).toContain('--timeout');
}, 10000);
test('should execute simple expression directly', () => {
const result = runCLI('code "return 2 + 2"');
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain('4');
expect(result.stdout).toMatch(/Result|✅/);
}, 60000);
test('should execute code with string operations', () => {
const result = runCLI('code "return \\"hello\\".toUpperCase()"');
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain('HELLO');
}, 60000);
test('should execute code from file', () => {
// Create a temporary test file
const testFile = path.join(testConfigPath, 'test-code.ts');
fs.writeFileSync(testFile, 'return { sum: 1 + 2, product: 2 * 3 }');
// On Windows, convert backslashes to forward slashes for CLI argument
// (Node.js path operations work with both, and this avoids escaping issues)
const cliPath = testFile.replace(/\\/g, '/');
const result = runCLI(`code --file "${cliPath}"`);
// Log output for debugging if test fails
if (result.exitCode !== 0) {
console.log('File execution failed:');
console.log(' Path:', cliPath);
console.log(' stdout:', result.stdout);
console.log(' stderr:', result.stderr);
}
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain('sum');
expect(result.stdout).toContain('3');
expect(result.stdout).toContain('product');
expect(result.stdout).toContain('6');
}, 60000);
test('should handle code execution errors gracefully', () => {
const result = runCLI('code "throw new Error(\\"test error\\")"');
expect(result.exitCode).not.toBe(0);
expect(result.stdout + result.stderr).toMatch(/error|Error/i);
}, 60000);
test('should support --json output format', () => {
const result = runCLI('code "return { a: 1, b: 2 }" --json');
expect(result.exitCode).toBe(0);
// Should contain valid JSON output
expect(result.stdout).toContain('"a"');
expect(result.stdout).toContain('"b"');
}, 60000);
test('should handle file not found gracefully', () => {
const result = runCLI('code --file /nonexistent/file.ts');
expect(result.exitCode).not.toBe(0);
expect(result.stdout + result.stderr).toMatch(/error|not found|ENOENT/i);
}, 10000);
test('should respect timeout option', () => {
// Very short timeout should fail for long-running code
const result = runCLI('code "while(true){}" --timeout 100');
// Should timeout/fail
expect(result.exitCode).not.toBe(0);
}, 30000);
test('should have access to MCP namespaces', () => {
// Test that schedule namespace is available (internal MCP)
const result = runCLI('code "return typeof schedule"');
expect(result.exitCode).toBe(0);
// schedule should be an object (namespace)
expect(result.stdout).toMatch(/object|function/);
}, 60000);
});
describe('Code Execution via run command (code:run)', () => {
test('should execute code via run code:run', () => {
const result = runCLI('run code:run --params \'{"code": "return 1 + 1"}\'');
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain('2');
}, 60000);
test('should have CodeMCP properly initialized', () => {
// This tests the orchestrator injection fix
// Use single quotes inside code to avoid JSON escaping issues
const result = runCLI("run code:run --params '{\"code\": \"return 42\"}'");
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain('42');
// Should NOT contain "not yet initialized" error
expect(result.stdout).not.toContain('not yet initialized');
}, 60000);
});
});
