#!/usr/bin/env node
/**
* Real YouTube Test - Test LearnMCP with actual YouTube content
* Tests the complete workflow with the provided YouTube URL
*/
import { LearnService } from './modules/learn-service.js';
import { createLearnLogger } from './modules/utils/custom-logger.js';
import { ContentExtractorFactory } from './modules/content-extractors/index.js';
// Test configuration with the provided YouTube URL
const TEST_CONFIG = {
projectId: 'youtube_test_' + Date.now(),
dataDir: process.env.FOREST_DATA_DIR || './.forest-data',
youtubeUrl: 'https://www.youtube.com/watch?v=scGlmUuq4WM'
};
class YouTubeTestRunner {
constructor() {
this.logger = createLearnLogger('YouTubeTest');
this.learnService = null;
this.extractorFactory = new ContentExtractorFactory();
this.startTime = Date.now();
}
/**
* Run the complete YouTube test
*/
async run() {
try {
this.logger.info('🎬 Starting Real YouTube Test', {
projectId: TEST_CONFIG.projectId,
youtubeUrl: TEST_CONFIG.youtubeUrl
});
await this.initializeServices();
await this.testYouTubeExtraction();
await this.testFullWorkflow();
await this.examineResults();
await this.cleanup();
const duration = Date.now() - this.startTime;
this.logger.info('✅ YouTube test completed successfully', {
duration: `${duration}ms`,
totalTime: `${Math.round(duration / 1000)}s`
});
} catch (error) {
this.logger.error('❌ YouTube test failed', {
error: error.message,
stack: error.stack
});
throw error;
}
}
/**
* Initialize services
*/
async initializeServices() {
this.logger.info('🔧 Initializing services...');
this.learnService = new LearnService(TEST_CONFIG.dataDir, this.logger);
await this.learnService.initialize();
this.logger.info('✅ Services initialized');
}
/**
* Test YouTube extraction directly
*/
async testYouTubeExtraction() {
this.logger.info('🎯 Testing Direct YouTube Extraction...');
try {
// Test URL validation
const canHandle = this.extractorFactory.canProcess(TEST_CONFIG.youtubeUrl);
this.logger.info('URL validation result', { canHandle });
if (!canHandle) {
throw new Error('YouTube URL cannot be processed by extractor factory');
}
// Get extractor info
const extractorInfo = this.extractorFactory.getExtractor(TEST_CONFIG.youtubeUrl);
this.logger.info('Extractor info', {
type: extractorInfo.type,
hasExtractor: !!extractorInfo.extractor
});
// Test direct extraction
this.logger.info('🔍 Starting direct content extraction...');
const extractedContent = await this.extractorFactory.extractContent(TEST_CONFIG.youtubeUrl);
this.logger.info('✅ Direct extraction completed', {
type: extractedContent.type,
title: extractedContent.metadata?.title || 'Unknown',
duration: extractedContent.metadata?.duration || 0,
hasTranscript: extractedContent.content?.transcriptAvailable || false,
contentLength: extractedContent.content?.transcript?.length || 0,
descriptionLength: extractedContent.content?.description?.length || 0
});
// Show detailed metadata
console.log('\n📋 EXTRACTED METADATA:');
console.log('Title:', extractedContent.metadata?.title || 'Unknown');
console.log('Author:', extractedContent.metadata?.author || 'Unknown');
console.log('Duration:', extractedContent.metadata?.duration || 0, 'seconds');
console.log('View Count:', extractedContent.metadata?.viewCount || 0);
console.log('Publish Date:', extractedContent.metadata?.publishDate || 'Unknown');
console.log('Language:', extractedContent.metadata?.language || 'Unknown');
console.log('Keywords:', extractedContent.metadata?.keywords?.slice(0, 5) || []);
// Show content preview
console.log('\n📝 CONTENT PREVIEW:');
if (extractedContent.content?.transcript) {
console.log('Transcript (first 500 chars):');
console.log(extractedContent.content.transcript.substring(0, 500) + '...');
} else {
console.log('No transcript available');
if (extractedContent.content?.description) {
console.log('Description (first 300 chars):');
console.log(extractedContent.content.description.substring(0, 300) + '...');
}
}
return extractedContent;
} catch (error) {
this.logger.error('Direct extraction failed', { error: error.message });
throw error;
}
}
/**
* Test the full LearnMCP workflow
*/
async testFullWorkflow() {
this.logger.info('🔄 Testing Full LearnMCP Workflow...');
try {
// Step 1: Add the YouTube URL as a learning source
this.logger.info('Step 1: Adding YouTube URL as learning source...');
const addResult = await this.learnService.addLearningSources(
TEST_CONFIG.projectId,
[TEST_CONFIG.youtubeUrl]
);
this.logger.info('Learning source added', {
addedCount: addResult.addedSources?.length || 0,
sourceId: addResult.addedSources?.[0]?.id || 'unknown'
});
// Step 2: Start processing
this.logger.info('Step 2: Starting background processing...');
const processResult = await this.learnService.processLearningSources(TEST_CONFIG.projectId);
this.logger.info('Processing started', {
queuedCount: processResult.queuedCount || 0
});
// Step 3: Wait for processing to complete
this.logger.info('Step 3: Waiting for processing to complete...');
let attempts = 0;
const maxAttempts = 20; // Increased for YouTube processing
while (attempts < maxAttempts) {
await this.sleep(3000); // Wait 3 seconds
const status = await this.learnService.getProcessingStatus(TEST_CONFIG.projectId);
this.logger.info(`Processing status check ${attempts + 1}/${maxAttempts}`, {
totalSources: status.sources?.totalSources || 0,
isProcessing: status.sources?.isProcessing || false,
statusCounts: status.sources?.statusCounts || {}
});
if (!status.sources?.isProcessing) {
this.logger.info('🎉 Processing completed!');
break;
}
attempts++;
}
if (attempts >= maxAttempts) {
this.logger.warn('Processing timeout - may still be running in background');
}
} catch (error) {
this.logger.error('Full workflow test failed', { error: error.message });
throw error;
}
}
/**
* Examine the final results and summarization
*/
async examineResults() {
this.logger.info('🔍 Examining Final Results...');
try {
// Get the processed sources
const sources = await this.learnService.listLearningSources(TEST_CONFIG.projectId);
this.logger.info('Sources status', {
totalSources: sources.length,
statuses: sources.map(s => ({ id: s.id, status: s.status, title: s.metadata?.title }))
});
// Try to get individual summaries
for (const source of sources) {
if (source.status === 'completed') {
this.logger.info(`Getting summary for source: ${source.id}`);
try {
const summary = await this.learnService.getLearningSummary(
TEST_CONFIG.projectId,
source.id
);
if (summary) {
console.log('\n🎯 INDIVIDUAL SUMMARY:');
console.log('Source ID:', source.id);
console.log('Title:', summary.metadata?.title || 'Unknown');
console.log('Summary Method:', summary.summaryMethod || 'Unknown');
console.log('Relevance Score:', summary.metadata?.relevanceScore || 'N/A');
console.log('Compression Ratio:', summary.metadata?.compressionRatio || 'N/A');
console.log('Original Length:', summary.metadata?.originalLength || 0);
console.log('Summary Length:', summary.metadata?.summaryLength || 0);
console.log('\n📄 SUMMARY CONTENT:');
console.log(summary.summary || 'No summary available');
if (summary.keyPoints && summary.keyPoints.length > 0) {
console.log('\n🔑 KEY POINTS:');
summary.keyPoints.forEach((point, index) => {
console.log(`${index + 1}. ${point}`);
});
}
if (summary.tags && summary.tags.length > 0) {
console.log('\n🏷️ TAGS:', summary.tags.join(', '));
}
}
} catch (error) {
this.logger.warn(`Failed to get summary for source ${source.id}`, {
error: error.message
});
}
}
}
// Try to get aggregated summary (what would be sent to Forest HTA)
this.logger.info('Getting aggregated summary for Forest integration...');
try {
const aggregatedSummary = await this.learnService.getLearningSummary(
TEST_CONFIG.projectId,
null, // No specific source ID = aggregated
2000 // Token limit for HTA integration
);
if (aggregatedSummary) {
console.log('\n🌲 FOREST HTA INTEGRATION PREVIEW:');
console.log('Sources Included:', aggregatedSummary.sourcesIncluded?.length || 0);
console.log('Total Sources:', aggregatedSummary.totalSources || 0);
console.log('Token Count:', aggregatedSummary.tokenCount || 0);
console.log('Truncated:', aggregatedSummary.metadata?.truncated || false);
console.log('\n📋 CONTENT FOR HTA BUILDER:');
console.log('```');
console.log(aggregatedSummary.aggregatedSummary || 'No aggregated summary available');
console.log('```');
console.log('\n🎯 This is what would be injected into Forest\'s HTA generation prompt!');
} else {
console.log('\n❌ No aggregated summary available for Forest integration');
}
} catch (error) {
this.logger.warn('Failed to get aggregated summary', { error: error.message });
}
} catch (error) {
this.logger.error('Results examination failed', { error: error.message });
throw error;
}
}
/**
* Cleanup test data
*/
async cleanup() {
this.logger.info('🧹 Cleaning up test data...');
try {
const sources = await this.learnService.listLearningSources(TEST_CONFIG.projectId);
if (sources.length > 0) {
const sourceIds = sources.map(s => s.id);
await this.learnService.deleteLearningSources(TEST_CONFIG.projectId, sourceIds);
this.logger.info('Test sources cleaned up', { deletedCount: sourceIds.length });
}
await this.learnService.shutdown();
this.logger.info('Services shut down');
} catch (error) {
this.logger.warn('Cleanup encountered issues', { error: error.message });
}
}
/**
* Sleep utility
*/
sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
}
// Run the test if this file is executed directly
if (import.meta.url === `file://${process.argv[1]}`) {
console.log(`
🎬 Real YouTube Test - LearnMCP
Testing with: ${TEST_CONFIG.youtubeUrl}
This will:
1. Extract video metadata and transcript
2. Process through LearnMCP workflow
3. Generate summaries and key points
4. Show what gets sent to Forest HTA builder
Press Ctrl+C to stop at any time.
`);
const test = new YouTubeTestRunner();
// Handle graceful shutdown
process.on('SIGINT', async () => {
console.log('\n🛑 Test interrupted by user');
try {
await test.cleanup();
} catch (error) {
console.error('Cleanup error:', error.message);
}
process.exit(0);
});
// Run the test
test.run().catch(error => {
console.error('❌ YouTube test failed:', error.message);
process.exit(1);
});
}
