#!/usr/bin/env node
/**
* End-to-End Test Script for NIST CSF MCP Server
*
* This script tests the complete workflow:
* 1. Organization creation
* 2. Quick assessment
* 3. Gap analysis generation
* 4. Implementation plan creation
* 5. Progress tracking
* 6. Executive report generation
*/
import { getDatabase, closeDatabase } from './db/database.js';
import { getMonitoredDatabase, closeMonitoredDatabase } from './db/monitored-database.js';
// import { initializeFramework } from './services/framework-loader.js';
// import { logger } from './utils/enhanced-logger.js';
import { toolAnalytics } from './utils/analytics.js';
// Tool imports
import { createProfile } from './tools/create_profile.js';
import { quickAssessment } from './tools/quick_assessment.js';
import { assessMaturity } from './tools/assess_maturity.js';
import { calculateRiskScore } from './tools/calculate_risk_score.js';
import { generateGapAnalysis } from './tools/generate_gap_analysis.js';
import { generatePriorityMatrix } from './tools/generate_priority_matrix.js';
import { createImplementationPlan } from './tools/create_implementation_plan.js';
import { estimateImplementationCost } from './tools/estimate_implementation_cost.js';
import { trackProgressTool } from './tools/track_progress.js';
import { generateReportTool } from './tools/generate_report.js';
// import { compareProfilesTool } from './tools/compare_profiles.js';
import { exportDataTool } from './tools/export_data.js';
// Test data and configuration
const TEST_CONFIG = {
organization: {
name: 'Test Corp E2E',
industry: 'technology',
size: 'medium',
description: 'End-to-end test organization for NIST CSF MCP validation'
},
profile: {
name: 'E2E Test Profile',
description: 'Complete workflow test profile',
target_implementation_tier: 'Adaptive'
},
assessments: [
// Identity and Access Management
{ subcategory_id: 'ID.AM-1', current_tier: 'Partial', target_tier: 'Adaptive', notes: 'Asset inventory needs enhancement' },
{ subcategory_id: 'ID.AM-2', current_tier: 'Partial', target_tier: 'Adaptive', notes: 'Software platforms documented' },
{ subcategory_id: 'PR.AC-1', current_tier: 'Partial', target_tier: 'Repeatable', notes: 'Identity management system in place' },
{ subcategory_id: 'PR.AC-4', current_tier: 'Initial', target_tier: 'Adaptive', notes: 'Access permissions need review' },
// Detection and Response
{ subcategory_id: 'DE.AE-1', current_tier: 'Partial', target_tier: 'Repeatable', notes: 'Baseline network activity established' },
{ subcategory_id: 'DE.CM-1', current_tier: 'Initial', target_tier: 'Adaptive', notes: 'Network monitoring needs improvement' },
{ subcategory_id: 'RS.RP-1', current_tier: 'Initial', target_tier: 'Repeatable', notes: 'Response plan exists but needs testing' },
{ subcategory_id: 'RS.CO-2', current_tier: 'Partial', target_tier: 'Adaptive', notes: 'Incident reporting process defined' },
// Recovery
{ subcategory_id: 'RC.RP-1', current_tier: 'Initial', target_tier: 'Repeatable', notes: 'Recovery plan needs validation' },
{ subcategory_id: 'RC.CO-3', current_tier: 'Partial', target_tier: 'Adaptive', notes: 'Recovery communication plan in place' }
]
};
// Test report interface
interface TestResult {
step: string;
description: string;
status: 'PASS' | 'FAIL' | 'SKIP';
duration: number;
data?: any;
error?: string;
details?: string;
}
interface TestReport {
testSuite: string;
startTime: Date;
endTime?: Date;
totalDuration?: number;
results: TestResult[];
summary: {
total: number;
passed: number;
failed: number;
skipped: number;
successRate: number;
};
artifacts: {
organizationId?: string;
profileId?: string;
gapAnalysisId?: string;
implementationPlanId?: string;
reportData?: any;
};
}
class E2ETestRunner {
private db: any;
// private framework: any;
private report: TestReport;
private useMonitoring: boolean;
constructor(useMonitoring: boolean = true) {
this.useMonitoring = useMonitoring;
this.report = {
testSuite: 'NIST CSF MCP Server End-to-End Workflow Test',
startTime: new Date(),
results: [],
summary: { total: 0, passed: 0, failed: 0, skipped: 0, successRate: 0 },
artifacts: {}
};
}
/**
* Initialize test environment
*/
async initialize(): Promise<void> {
console.log('🚀 Initializing E2E Test Environment');
try {
// Initialize database and framework
this.db = this.useMonitoring ? getMonitoredDatabase() : getDatabase();
// this.framework = await initializeFramework();
console.log('✅ Test environment initialized successfully');
} catch (error) {
console.error('❌ Failed to initialize test environment:', error);
throw error;
}
}
/**
* Execute a test step with error handling and timing
*/
async executeStep(
stepName: string,
description: string,
testFunction: () => Promise<any>
): Promise<TestResult> {
const startTime = Date.now();
console.log(`\n🔍 ${stepName}: ${description}`);
try {
const result = await testFunction();
const duration = Date.now() - startTime;
const testResult: TestResult = {
step: stepName,
description,
status: 'PASS',
duration,
data: result,
details: `Completed successfully in ${duration}ms`
};
console.log(`✅ ${stepName} - PASSED (${duration}ms)`);
return testResult;
} catch (error: any) {
const duration = Date.now() - startTime;
const testResult: TestResult = {
step: stepName,
description,
status: 'FAIL',
duration,
error: error.message,
details: `Failed after ${duration}ms: ${error.message}`
};
console.log(`❌ ${stepName} - FAILED (${duration}ms): ${error.message}`);
return testResult;
}
}
/**
* Step 1: Create test organization and profile
*/
async testOrganizationCreation(): Promise<TestResult> {
return this.executeStep(
'STEP_1_ORG_CREATION',
'Create test organization and profile',
async () => {
// Create profile (which includes organization creation)
const profileData = {
organization: TEST_CONFIG.organization,
profile: TEST_CONFIG.profile
};
const result = await createProfile({
org_name: profileData.organization.name,
sector: profileData.organization.industry,
size: profileData.organization.size as 'small' | 'medium' | 'large' | 'enterprise',
profile_type: 'current',
profile_name: profileData.profile.name,
description: profileData.profile.description
});
if (!result.profile_id) {
throw new Error('Profile creation failed - no profile_id returned');
}
this.report.artifacts.profileId = result.profile_id;
this.report.artifacts.organizationId = result.org_id;
return {
profileId: result.profile_id,
organizationId: result.org_id,
message: result.message
};
}
);
}
/**
* Step 2: Perform quick assessment
*/
async testQuickAssessment(): Promise<TestResult> {
return this.executeStep(
'STEP_2_QUICK_ASSESSMENT',
'Perform quick assessment with test data',
async () => {
if (!this.report.artifacts.profileId) {
throw new Error('No profile ID available from previous step');
}
// const assessmentData = {
// profile_id: this.report.artifacts.profileId,
// assessments: TEST_CONFIG.assessments
// };
const result = await quickAssessment({
profile_id: this.report.artifacts.profileId!,
simplified_answers: {
govern: 'partial',
identify: 'partial',
protect: 'partial',
detect: 'yes',
respond: 'no',
recover: 'partial'
},
assessed_by: 'e2e-test',
confidence_level: 'medium'
});
if (!result.success) {
throw new Error('Quick assessment failed - operation not successful');
}
return {
assessmentsCreated: result.details?.assessmentsCreated || 0,
functionSummaries: result.details?.functionSummaries?.length || 0,
overallAverage: result.initial_maturity_scores.overall_average,
message: result.message
};
}
);
}
/**
* Step 3: Assess maturity and calculate risk
*/
async testMaturityAndRiskAssessment(): Promise<TestResult> {
return this.executeStep(
'STEP_3_MATURITY_RISK',
'Assess maturity levels and calculate risk scores',
async () => {
if (!this.report.artifacts.profileId) {
throw new Error('No profile ID available');
}
// Get maturity assessment
const maturityResult = await assessMaturity({
profile_id: this.report.artifacts.profileId!,
include_recommendations: true,
include_subcategory_details: true
});
// Calculate risk score
const riskResult = await calculateRiskScore({
profile_id: this.report.artifacts.profileId!,
include_recommendations: true,
include_heat_map: false
});
return {
maturity: {
overallMaturityTier: maturityResult.overall_maturity_tier,
overallMaturityScore: maturityResult.overall_maturity_score,
functionBreakdown: maturityResult.function_breakdown?.length || 0
},
risk: {
overallRiskScore: riskResult.overall_risk_score,
riskLevel: riskResult.risk_level,
criticalRisks: riskResult.risk_summary?.critical_risks || 0
}
};
}
);
}
/**
* Step 4: Generate gap analysis
*/
async testGapAnalysis(): Promise<TestResult> {
return this.executeStep(
'STEP_4_GAP_ANALYSIS',
'Generate comprehensive gap analysis',
async () => {
if (!this.report.artifacts.profileId) {
throw new Error('No profile ID available');
}
// const gapData = {
// current_profile_id: this.report.artifacts.profileId,
// target_profile_id: this.report.artifacts.profileId,
// analysis_name: 'E2E Test Gap Analysis'
// };
const result = await generateGapAnalysis({
current_profile_id: this.report.artifacts.profileId!,
target_profile_id: this.report.artifacts.profileId!,
include_priority_matrix: true,
include_visualizations: false,
minimum_gap_score: 0
});
if (!result.analysis_id) {
throw new Error('Gap analysis generation failed - no analysis ID returned');
}
this.report.artifacts.gapAnalysisId = result.analysis_id;
return {
gapAnalysisId: result.analysis_id,
totalGaps: result.gap_summary?.total_gaps || 0,
criticalGaps: result.gap_summary?.critical_gaps || 0,
recommendations: result.recommendations?.immediate_actions?.length || 0
};
}
);
}
/**
* Step 5: Generate priority matrix
*/
async testPriorityMatrix(): Promise<TestResult> {
return this.executeStep(
'STEP_5_PRIORITY_MATRIX',
'Generate implementation priority matrix',
async () => {
if (!this.report.artifacts.gapAnalysisId) {
throw new Error('No gap analysis ID available');
}
const result = await generatePriorityMatrix({
profile_id: this.report.artifacts.profileId!,
include_recommendations: true,
matrix_type: 'effort_impact' as const,
include_resource_estimates: true,
max_items_per_quadrant: 10
});
return {
highPriorityItems: result.quadrants?.high_value_low_effort?.items?.length || 0,
mediumPriorityItems: result.quadrants?.high_value_high_effort?.items?.length || 0,
lowPriorityItems: result.quadrants?.low_value_low_effort?.items?.length || 0,
totalItems: result.summary?.total_items || 0
};
}
);
}
/**
* Step 6: Create implementation plan
*/
async testImplementationPlan(): Promise<TestResult> {
return this.executeStep(
'STEP_6_IMPLEMENTATION_PLAN',
'Create comprehensive implementation plan',
async () => {
if (!this.report.artifacts.gapAnalysisId) {
throw new Error('No gap analysis ID available');
}
// const planData = {
// gap_analysis_id: this.report.artifacts.gapAnalysisId,
// plan_name: 'E2E Test Implementation Plan',
// target_completion_date: new Date(Date.now() + 365 * 24 * 60 * 60 * 1000).toISOString().split('T')[0],
// budget_limit: 500000,
// resource_constraints: {
// available_hours_per_month: 160,
// team_size: 5,
// security_expertise_level: 'intermediate'
// }
// };
const result = await createImplementationPlan({
analysis_id: this.report.artifacts.gapAnalysisId!,
timeline_months: 12,
available_resources: 5,
prioritization_strategy: 'risk_based' as const,
phase_duration: 3,
include_dependencies: true,
include_milestones: true,
plan_name: 'E2E Test Implementation Plan'
});
if (!result.plan_id) {
throw new Error('Implementation plan creation failed - no plan ID returned');
}
this.report.artifacts.implementationPlanId = result.plan_id;
return {
planId: result.plan_id,
totalPhases: result.total_phases,
totalItems: result.phases?.reduce((sum, phase) => sum + (phase.items?.length || 0), 0) || 0,
estimatedDuration: result.timeline_months,
estimatedCost: result.estimated_cost
};
}
);
}
/**
* Step 7: Estimate implementation cost
*/
async testCostEstimation(): Promise<TestResult> {
return this.executeStep(
'STEP_7_COST_ESTIMATION',
'Generate detailed cost estimates',
async () => {
if (!this.report.artifacts.implementationPlanId) {
throw new Error('No implementation plan ID available');
}
const result = await estimateImplementationCost({
subcategory_ids: ['ID.AM-1', 'PR.AC-1', 'DE.AE-1'],
organization_size: TEST_CONFIG.organization.size as 'small' | 'medium' | 'large' | 'enterprise',
include_ongoing_costs: true,
include_risk_adjusted: false,
currency: 'USD' as const,
include_contingency: true
});
return {
totalCost: result.cost_summary?.total_one_time || 0,
breakdown: result.cost_breakdown,
timeframe: 12 // Default timeframe since timeline_months doesn't exist in cost_summary
};
}
);
}
/**
* Step 8: Track progress updates
*/
async testProgressTracking(): Promise<TestResult> {
return this.executeStep(
'STEP_8_PROGRESS_TRACKING',
'Track implementation progress updates',
async () => {
if (!this.report.artifacts.profileId) {
throw new Error('No profile ID available');
}
// Simulate some progress updates
const progressUpdates = [
{
profile_id: this.report.artifacts.profileId,
subcategory_id: 'ID.AM-1',
status: 'in_progress',
completion_percentage: 45,
notes: 'Asset inventory project started',
milestone: 'Phase 1 - Discovery'
},
{
profile_id: this.report.artifacts.profileId,
subcategory_id: 'PR.AC-1',
status: 'completed',
completion_percentage: 100,
notes: 'Identity management system upgraded',
milestone: 'Phase 1 - Complete'
},
{
profile_id: this.report.artifacts.profileId,
subcategory_id: 'DE.AE-1',
status: 'in_progress',
completion_percentage: 25,
notes: 'Network monitoring tools being evaluated',
milestone: 'Phase 1 - Planning'
}
];
const result = await trackProgressTool.execute({
profile_id: this.report.artifacts.profileId!,
updates: progressUpdates.map(u => ({
subcategory_id: u.subcategory_id,
current_implementation: u.status === 'completed' ? 'fully_implemented' : 'partially_implemented',
current_maturity: Math.floor(u.completion_percentage / 20),
status: u.status as any,
notes: u.notes
}))
}, this.db);
return {
updatesProcessed: progressUpdates.length,
totalSubcategories: result.total_subcategories,
overallCompletion: result.overall_completion_percentage || 0,
completedCount: result.completed || 0
};
}
);
}
/**
* Step 9: Generate executive report
*/
async testExecutiveReport(): Promise<TestResult> {
return this.executeStep(
'STEP_9_EXECUTIVE_REPORT',
'Generate comprehensive executive report',
async () => {
if (!this.report.artifacts.profileId) {
throw new Error('No profile ID available');
}
const result = await generateReportTool.execute({
profile_id: this.report.artifacts.profileId!,
report_type: 'executive',
format: 'json',
include_recommendations: true
}, this.db);
this.report.artifacts.reportData = result;
return {
reportType: result.report_type,
format: result.format,
success: result.success,
fileSize: result.metadata?.report_size || 0
};
}
);
}
/**
* Step 10: Test data export and validation
*/
async testDataExport(): Promise<TestResult> {
return this.executeStep(
'STEP_10_DATA_EXPORT',
'Export and validate all generated data',
async () => {
if (!this.report.artifacts.profileId) {
throw new Error('No profile ID available');
}
const result = await exportDataTool.execute({
profile_id: this.report.artifacts.profileId!,
format: 'json',
include_assessments: true,
include_progress: true,
include_compliance: false,
include_milestones: false
}, this.db);
return {
exportFormat: result.format,
success: result.success,
recordCount: result.metadata?.record_count || 0,
fileSize: result.metadata?.file_size || 0,
includedSections: result.metadata?.included_sections?.length || 0
};
}
);
}
/**
* Run the complete test suite
*/
async runTests(): Promise<TestReport> {
console.log(`\n🎯 Starting ${this.report.testSuite}`);
console.log('=' .repeat(80));
try {
// Initialize test environment
await this.initialize();
// Execute all test steps
const testSteps = [
() => this.testOrganizationCreation(),
() => this.testQuickAssessment(),
() => this.testMaturityAndRiskAssessment(),
() => this.testGapAnalysis(),
() => this.testPriorityMatrix(),
() => this.testImplementationPlan(),
() => this.testCostEstimation(),
() => this.testProgressTracking(),
() => this.testExecutiveReport(),
() => this.testDataExport()
];
// Execute each test step
for (const testStep of testSteps) {
const result = await testStep();
this.report.results.push(result);
}
} catch (error: any) {
console.error('💥 Test suite failed during setup:', error.message);
this.report.results.push({
step: 'SETUP_FAILURE',
description: 'Test suite initialization failed',
status: 'FAIL',
duration: 0,
error: error.message
});
}
// Calculate final report
this.finalizeReport();
return this.report;
}
/**
* Generate final test report with summary
*/
private finalizeReport(): void {
this.report.endTime = new Date();
this.report.totalDuration = this.report.endTime.getTime() - this.report.startTime.getTime();
// Calculate summary statistics
this.report.summary.total = this.report.results.length;
this.report.summary.passed = this.report.results.filter(r => r.status === 'PASS').length;
this.report.summary.failed = this.report.results.filter(r => r.status === 'FAIL').length;
this.report.summary.skipped = this.report.results.filter(r => r.status === 'SKIP').length;
this.report.summary.successRate = this.report.summary.total > 0
? (this.report.summary.passed / this.report.summary.total) * 100
: 0;
}
/**
* Display detailed test report
*/
displayReport(): void {
console.log('\n' + '='.repeat(80));
console.log('📊 FINAL TEST REPORT');
console.log('='.repeat(80));
console.log(`\n📋 Test Suite: ${this.report.testSuite}`);
console.log(`⏱️ Start Time: ${this.report.startTime.toISOString()}`);
console.log(`⏱️ End Time: ${this.report.endTime?.toISOString()}`);
console.log(`⌚ Total Duration: ${this.report.totalDuration}ms (${(this.report.totalDuration! / 1000).toFixed(2)}s)`);
console.log(`\n📈 Summary Statistics:`);
console.log(` Total Tests: ${this.report.summary.total}`);
console.log(` ✅ Passed: ${this.report.summary.passed}`);
console.log(` ❌ Failed: ${this.report.summary.failed}`);
console.log(` ⏭️ Skipped: ${this.report.summary.skipped}`);
console.log(` 📊 Success Rate: ${this.report.summary.successRate.toFixed(2)}%`);
console.log(`\n🗂️ Generated Artifacts:`);
console.log(` Organization ID: ${this.report.artifacts.organizationId || 'N/A'}`);
console.log(` Profile ID: ${this.report.artifacts.profileId || 'N/A'}`);
console.log(` Gap Analysis ID: ${this.report.artifacts.gapAnalysisId || 'N/A'}`);
console.log(` Implementation Plan ID: ${this.report.artifacts.implementationPlanId || 'N/A'}`);
console.log(` Executive Report: ${this.report.artifacts.reportData ? 'Generated' : 'N/A'}`);
console.log(`\n📝 Detailed Results:`);
this.report.results.forEach((result, index) => {
const statusIcon = result.status === 'PASS' ? '✅' : result.status === 'FAIL' ? '❌' : '⏭️';
console.log(` ${statusIcon} ${index + 1}. ${result.step}`);
console.log(` Description: ${result.description}`);
console.log(` Duration: ${result.duration}ms`);
if (result.status === 'PASS' && result.data) {
console.log(` Result: ${JSON.stringify(result.data, null, 8).substring(0, 200)}...`);
}
if (result.status === 'FAIL' && result.error) {
console.log(` Error: ${result.error}`);
}
console.log('');
});
// Final status
const overallStatus = this.report.summary.failed === 0 ? 'SUCCESS' : 'FAILURE';
const statusIcon = overallStatus === 'SUCCESS' ? '🎉' : '💥';
console.log('='.repeat(80));
console.log(`${statusIcon} OVERALL TEST STATUS: ${overallStatus}`);
console.log('='.repeat(80));
if (overallStatus === 'SUCCESS') {
console.log('🎯 All major NIST CSF MCP Server workflows validated successfully!');
console.log('✅ Data persistence, tool interactions, and reporting verified');
console.log('🚀 System is ready for production deployment');
} else {
console.log('⚠️ Some tests failed - review errors above');
console.log('🔧 Fix failing components before deployment');
}
}
/**
* Cleanup test environment
*/
async cleanup(): Promise<void> {
console.log('\n🧹 Cleaning up test environment...');
try {
// Close database connections
if (this.useMonitoring) {
closeMonitoredDatabase();
} else {
closeDatabase();
}
// Stop analytics if running
if (toolAnalytics) {
toolAnalytics.stop();
}
console.log('✅ Test environment cleaned up successfully');
} catch (error) {
console.warn('⚠️ Warning during cleanup:', error);
}
}
}
/**
* Main test execution
*/
async function main() {
const useMonitoring = process.env.ENABLE_MONITORING !== 'false';
const testRunner = new E2ETestRunner(useMonitoring);
try {
// Run the complete test suite
const report = await testRunner.runTests();
// Display results
testRunner.displayReport();
// Write report to file
const fs = await import('fs');
const reportPath = `./test-reports/e2e-test-report-${Date.now()}.json`;
fs.mkdirSync('./test-reports', { recursive: true });
fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
console.log(`\n📄 Detailed report saved to: ${reportPath}`);
// Exit with appropriate code
const exitCode = report.summary.failed === 0 ? 0 : 1;
process.exit(exitCode);
} catch (error) {
console.error('💥 Test execution failed:', error);
process.exit(1);
} finally {
await testRunner.cleanup();
}
}
// Run tests if this file is executed directly
if (require.main === module) {
main().catch(console.error);
}
export type { TestReport, TestResult };
export { E2ETestRunner };
