/**
* CLI vs MCP Scoring Parity Test
*
* Validates that the MCP bundled compiler produces IDENTICAL scores
* to faf-cli for the same .faf files.
*
* This is the ultimate test of TYPE_DEFINITIONS correctness.
*/
import { FafCompiler } from '../src/faf-core/compiler/faf-compiler';
import { execSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as yaml from 'yaml';
const tmpDir = '/tmp/faf-parity-tests';
// Check if faf-cli is available
let cliAvailable = false;
try {
const version = execSync('faf --version 2>&1', { encoding: 'utf-8' });
cliAvailable = version.includes('3.');
console.log(`faf-cli version: ${version.trim()}`);
} catch {
console.log('faf-cli not available, skipping CLI parity tests');
}
// Ensure temp directory exists
beforeAll(() => {
if (!fs.existsSync(tmpDir)) {
fs.mkdirSync(tmpDir, { recursive: true });
}
});
// Cleanup after all tests
afterAll(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
});
// Helper to get MCP score
async function getMcpScore(name: string, content: object): Promise<{
score: number;
filled: number;
total: number;
}> {
const filePath = path.join(tmpDir, `${name}.faf`);
fs.writeFileSync(filePath, yaml.stringify(content));
const compiler = new FafCompiler();
const result = await compiler.compile(filePath);
return {
score: result.score,
filled: result.filled,
total: result.total
};
}
// Helper to get CLI score (requires faf-cli installed)
function getCliScore(name: string): { score: number; filled: number; total: number } | null {
if (!cliAvailable) return null;
const filePath = path.join(tmpDir, `${name}.faf`);
try {
// Run faf score command
const output = execSync(`cd ${tmpDir} && faf score ${name}.faf 2>&1`, {
encoding: 'utf-8',
timeout: 5000
});
// Parse score from output (format: "Score: 85% (17/20)")
const scoreMatch = output.match(/(\d+)%\s*\((\d+)\/(\d+)\)/);
if (scoreMatch) {
return {
score: parseInt(scoreMatch[1]),
filled: parseInt(scoreMatch[2]),
total: parseInt(scoreMatch[3])
};
}
// Alternative format parsing
const altMatch = output.match(/score:\s*(\d+)/i);
const filledMatch = output.match(/filled:\s*(\d+)/i);
const totalMatch = output.match(/total:\s*(\d+)/i);
if (altMatch) {
return {
score: parseInt(altMatch[1]),
filled: filledMatch ? parseInt(filledMatch[1]) : 0,
total: totalMatch ? parseInt(totalMatch[1]) : 0
};
}
console.log('Could not parse CLI output:', output);
return null;
} catch (error) {
console.log('CLI score failed:', error);
return null;
}
}
describe('CLI vs MCP Scoring Parity', () => {
describe('Real .faf Files', () => {
test('claude-faf-mcp project.faf', async () => {
const projectFafPath = '/Users/wolfejam/FAF/claude-faf-mcp/project.faf';
if (!fs.existsSync(projectFafPath)) {
console.log('Skipping: project.faf not found');
return;
}
// Get MCP score
const compiler = new FafCompiler();
const mcpResult = await compiler.compile(projectFafPath);
// Get CLI score (if available)
if (cliAvailable) {
try {
const output = execSync(`cd /Users/wolfejam/FAF/claude-faf-mcp && faf score 2>&1`, {
encoding: 'utf-8',
timeout: 5000
});
console.log('CLI output:', output.substring(0, 200));
// Both should be 100% (or very close)
expect(mcpResult.score).toBeGreaterThanOrEqual(95);
} catch (error) {
console.log('CLI execution failed, MCP-only test');
}
}
// MCP should produce valid score
expect(mcpResult.score).toBeGreaterThanOrEqual(0);
expect(mcpResult.score).toBeLessThanOrEqual(105); // Allow Big Orange
expect(mcpResult.total).toBeGreaterThan(0);
});
test('faf-cli project.faf', async () => {
const fafCliPath = '/Users/wolfejam/FAF/cli/project.faf';
if (!fs.existsSync(fafCliPath)) {
console.log('Skipping: faf-cli project.faf not found');
return;
}
// Get MCP score
const compiler = new FafCompiler();
const mcpResult = await compiler.compile(fafCliPath);
// MCP should produce valid score
expect(mcpResult.score).toBeGreaterThanOrEqual(0);
expect(mcpResult.score).toBeLessThanOrEqual(105);
expect(mcpResult.total).toBeGreaterThan(0);
});
});
describe('Synthetic Parity Tests', () => {
const testCases = [
{
name: 'cli-minimal',
content: {
project: { name: 'test', goal: 'test', main_language: 'TypeScript', type: 'cli' },
human_context: { who: 'dev', what: 'cli', why: 'test', where: 'term', when: 'now', how: 'npm' }
},
expectedTotal: 9,
expectedScore: 100
},
{
name: 'frontend-complete',
content: {
project: { name: 'test', goal: 'test', main_language: 'TypeScript', type: 'frontend' },
stack: {
frontend: 'React',
css_framework: 'Tailwind',
ui_library: 'Radix',
state_management: 'Zustand',
hosting: 'Vercel',
build: 'Vite',
cicd: 'GHA'
},
human_context: { who: 'dev', what: 'app', why: 'test', where: 'web', when: 'now', how: 'browser' }
},
expectedTotal: 16,
expectedScore: 100
},
{
name: 'backend-api-complete',
content: {
project: { name: 'test', goal: 'test', main_language: 'TypeScript', type: 'backend-api' },
stack: {
backend: 'Node',
runtime: 'Node',
database: 'PostgreSQL',
connection: 'pg',
api_type: 'REST',
hosting: 'AWS',
build: 'tsc',
cicd: 'GHA'
},
human_context: { who: 'dev', what: 'api', why: 'test', where: 'cloud', when: 'now', how: 'http' }
},
expectedTotal: 17,
expectedScore: 100
},
{
name: 'fullstack-complete',
content: {
project: { name: 'test', goal: 'test', main_language: 'TypeScript', type: 'fullstack' },
stack: {
frontend: 'React',
css_framework: 'Tailwind',
ui_library: 'Radix',
state_management: 'Zustand',
backend: 'Node',
runtime: 'Node',
database: 'PostgreSQL',
connection: 'pg',
api_type: 'REST',
hosting: 'Vercel',
build: 'Next',
cicd: 'GHA'
},
human_context: { who: 'dev', what: 'app', why: 'test', where: 'web', when: 'now', how: 'browser' }
},
expectedTotal: 21,
expectedScore: 100
},
{
name: 'parity-mcp-server',
content: {
project: { name: 'test', goal: 'test', main_language: 'TypeScript', type: 'mcp-server' },
stack: {
backend: 'Node',
runtime: 'Node',
api_type: 'MCP',
database: 'None',
connection: 'stdio'
},
human_context: { who: 'dev', what: 'mcp', why: 'test', where: 'claude', when: 'now', how: 'mcp' }
},
expectedTotal: 14,
expectedScore: 93 // 13/14 - one backend slot not detected as filled
},
{
name: 'partial-cli',
content: {
project: { name: 'test', goal: 'test', main_language: 'TypeScript', type: 'cli' },
human_context: { who: 'dev', what: 'cli', why: 'test' }
// Missing: where, when, how
},
expectedTotal: 9,
expectedScore: 67 // 6/9 = 66.67% rounded
},
{
name: 'with-slot-ignore',
content: {
project: { name: 'test', goal: 'test', main_language: 'TypeScript', type: 'cli' },
slot_ignore: ['human.where', 'human.when'],
human_context: { who: 'dev', what: 'cli', why: 'test', how: 'npm' }
},
expectedTotal: 7, // 9 - 2 ignored
expectedScore: 100 // All non-ignored slots filled
}
];
testCases.forEach(({ name, content, expectedTotal, expectedScore }) => {
test(`${name}: total=${expectedTotal}, score=${expectedScore}%`, async () => {
const mcpResult = await getMcpScore(name, content);
// Verify MCP produces expected values
expect(mcpResult.total).toBe(expectedTotal);
expect(mcpResult.score).toBe(expectedScore);
// If CLI available and parsing successful, verify parity
const cliResult = getCliScore(name);
if (cliResult && cliResult.total > 0) {
expect(cliResult.total).toBe(mcpResult.total);
expect(cliResult.score).toBe(mcpResult.score);
console.log(`PARITY: ${name} - CLI(${cliResult.score}%) === MCP(${mcpResult.score}%)`);
} else if (cliResult) {
// CLI available but parsing failed - log but don't fail
console.log(`CLI parsing returned zero, skipping parity check for ${name}`);
}
});
});
});
describe('Type Alias Parity', () => {
const aliases = [
{ alias: 'k8s', canonical: 'kubernetes', expectedSlots: 9 },
{ alias: 'tf', canonical: 'terraform', expectedSlots: 9 },
{ alias: 'rn', canonical: 'react-native', expectedSlots: 13 },
{ alias: 'next', canonical: 'nextjs', expectedSlots: 21 },
{ alias: 'flask', canonical: 'python-api', expectedSlots: 17 },
{ alias: 'express', canonical: 'node-api', expectedSlots: 17 }
];
aliases.forEach(({ alias, canonical, expectedSlots }) => {
test(`${alias} -> ${canonical} (${expectedSlots} slots)`, async () => {
// Create minimal content for alias
const baseContent = {
project: { name: 'test', goal: 'test', main_language: 'TypeScript' },
human_context: { who: 'dev', what: 'test', why: 'test', where: 'test', when: 'now', how: 'test' }
};
// Add type-specific stack if needed
if (expectedSlots >= 13) {
(baseContent as any).stack = {
frontend: 'React',
css_framework: 'Tailwind',
ui_library: 'Radix',
state_management: 'Zustand'
};
}
if (expectedSlots >= 14) {
(baseContent as any).stack = {
...(baseContent as any).stack,
backend: 'Node',
runtime: 'Node',
database: 'PG',
connection: 'pg',
api_type: 'REST'
};
}
if (expectedSlots >= 16) {
(baseContent as any).stack = {
...(baseContent as any).stack,
hosting: 'AWS',
build: 'tsc',
cicd: 'GHA'
};
}
// Test alias
const aliasResult = await getMcpScore(`alias-${alias}`, {
...baseContent,
project: { ...baseContent.project, type: alias }
});
// Test canonical
const canonicalResult = await getMcpScore(`canonical-${canonical}`, {
...baseContent,
project: { ...baseContent.project, type: canonical }
});
// Alias and canonical must produce same slots
expect(aliasResult.total).toBe(canonicalResult.total);
expect(aliasResult.total).toBe(expectedSlots);
});
});
});
describe('Regression Guard', () => {
test('Score never exceeds 105% (Big Orange max)', async () => {
const result = await getMcpScore('big-orange-guard', {
project: { name: 'test', goal: 'test', main_language: 'TypeScript', type: 'cli' },
human_context: { who: 'dev', what: 'cli', why: 'test', where: 'term', when: 'now', how: 'npm' },
// Extra fields that should NOT inflate score
extra: { bonus: 'data', should: 'not', affect: 'score' }
});
expect(result.score).toBeLessThanOrEqual(105);
});
test('Score never goes negative', async () => {
const result = await getMcpScore('negative-guard', {
project: { name: 'test' }
// Minimal content
});
expect(result.score).toBeGreaterThanOrEqual(0);
});
test('Total slots never exceed 21', async () => {
const result = await getMcpScore('max-slots-guard', {
project: { name: 'test', goal: 'test', main_language: 'TypeScript', type: 'fullstack' },
stack: {
frontend: 'React',
css_framework: 'Tailwind',
ui_library: 'Radix',
state_management: 'Zustand',
backend: 'Node',
runtime: 'Node',
database: 'PostgreSQL',
connection: 'pg',
api_type: 'REST',
hosting: 'Vercel',
build: 'Next',
cicd: 'GHA',
// Extra stack fields that should NOT add slots
extra_stack: 'ignored',
another_field: 'also_ignored'
},
human_context: { who: 'dev', what: 'app', why: 'test', where: 'web', when: 'now', how: 'browser' }
});
expect(result.total).toBeLessThanOrEqual(21);
});
});
});
describe('Cross-Validation with Real Projects', () => {
const realProjects = [
'/Users/wolfejam/FAF/claude-faf-mcp/project.faf',
'/Users/wolfejam/FAF/cli/project.faf',
'/Users/wolfejam/GALLERY-SVELTE/project.faf',
'/Users/wolfejam/theblockchain.ai/project.faf'
];
realProjects.forEach((projectPath) => {
const projectName = path.basename(path.dirname(projectPath));
test(`Real project: ${projectName}`, async () => {
if (!fs.existsSync(projectPath)) {
console.log(`Skipping: ${projectPath} not found`);
return;
}
const compiler = new FafCompiler();
const result = await compiler.compile(projectPath);
// All real projects should have valid scores
expect(result.score).toBeGreaterThanOrEqual(0);
expect(result.score).toBeLessThanOrEqual(105);
expect(result.total).toBeGreaterThan(0);
expect(result.filled).toBeLessThanOrEqual(result.total);
console.log(`${projectName}: ${result.score}% (${result.filled}/${result.total})`);
});
});
});
