MCP Puppet Pipeline

// Complete Automated Puppet Production Studio // Handles the full pipeline from character image to final video production // Removed deprecated character-pipeline.js import - using corrected AffogatoClient from guidance notes import { ClaudeCharacterAnalyzer } from './claude-analyzer.js'; import { ElevenLabsClient } from './elevenlabs-client.js'; import { VideoAssembler } from './video-assembler.js'; import OpenAI from 'openai'; import axios from 'axios'; import fs from 'fs/promises'; import path from 'path'; export class ProductionOrchestrator { constructor() { // Legacy pipeline removed - using corrected Affogato integration following user guidance notes // this.characterPipeline = new CharacterProductionPipeline(process.env.AFFOGATO_API_KEY); this.claudeAnalyzer = new ClaudeCharacterAnalyzer(); // Initialize AI services this.openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); // Initialize voice and video services this.elevenLabs = process.env.ELEVENLABS_API_KEY ? new ElevenLabsClient(process.env.ELEVENLABS_API_KEY) : null; this.videoAssembler = new VideoAssembler('./production_output/videos'); // Production settings this.outputPath = './production_output'; this.ensureOutputDirectory(); } async ensureOutputDirectory() { try { await fs.mkdir(this.outputPath, { recursive: true }); await fs.mkdir(path.join(this.outputPath, 'characters'), { recursive: true }); await fs.mkdir(path.join(this.outputPath, 'scripts'), { recursive: true }); await fs.mkdir(path.join(this.outputPath, 'scenes'), { recursive: true }); await fs.mkdir(path.join(this.outputPath, 'videos'), { recursive: true }); } catch (error) { console.log('Output directories already exist'); } } // STEP 1: Process uploaded character image (e.g. giraffe) into puppet form async processCharacterImage(imagePath, characterName = null) { console.log('🦒 STARTING CHARACTER IMAGE PROCESSING'); console.log('=====================================\n'); try { // Step 1: Analyze the uploaded image with Claude AI console.log('🔍 Step 1: Analyzing character image...'); // Read image and convert to base64 const fs = await import('fs'); const imageBuffer = fs.readFileSync(imagePath); const imageBase64 = imageBuffer.toString('base64'); const imageAnalysis = await this.claudeAnalyzer.analyzeCharacterImages([imageBase64], characterName || 'Character'); if (!imageAnalysis.success) { throw new Error(`Image analysis failed: ${imageAnalysis.error}`); } // Extract character name and create description from analysis const finalCharacterName = characterName || 'Character'; const description = `A character with ${JSON.stringify(imageAnalysis.traits.physical_consistency || {}).slice(0, 200)}...`; console.log(`✅ Character analyzed: ${finalCharacterName}`); // Step 2: Convert traits to puppet specifications console.log('🎭 Step 2: Converting to puppet specifications...'); const puppetTraits = this.convertToPuppetSpecs(imageAnalysis.traits); // Step 3: Create character in production pipeline console.log('⚙️ Step 3: Creating character in production system...'); const characterData = { name: finalCharacterName, description: description, characterType: 'Main Character', referenceImagePath: imagePath, personalityTraits: imageAnalysis.traits.emotional_expressions?.observed_emotions || [], voiceDescription: this.generateVoiceDescription(imageAnalysis.traits), characterTraits: puppetTraits }; const productionResult = await this.characterPipeline.createCharacterFromDescription(characterData); // Step 4: Store in Notion with blueprint images console.log('📊 Step 4: Storing character blueprints...'); const notionResult = await this.storeCharacterBlueprints( finalCharacterName, puppetTraits, productionResult?.images || [], imageAnalysis ); return { success: true, character_name: finalCharacterName, puppet_traits: puppetTraits, blueprint_images: productionResult.images, notion_page_id: notionResult.page_id, voice_profile: this.generateVoiceDescription(imageAnalysis.traits), ready_for_voice_generation: true, ready_for_scripting: true }; } catch (error) { console.error('❌ Character processing failed:', error.message); return { success: false, error: error.message }; } } // Convert image analysis traits to puppet specifications convertToPuppetSpecs(traits) { // Handle both old format (traits.physical) and new format (traits.physical_consistency) const physicalTraits = traits.physical_consistency || traits.physical || {}; const clothingTraits = traits.costume_details || traits.clothing || {}; const personalityTraits = traits.emotional_expressions || traits.personality || {}; const puppetSpecs = { physical: { size: this.determinePuppetSize(physicalTraits), material: 'felt and foam construction', skin_tone: physicalTraits.skin_tone || 'natural felt color', hair: physicalTraits.hair_details || physicalTraits.hair || 'fabric hair', eyes: 'mechanical puppet eyes with controlled movement', mouth: 'mechanical opening/closing mouth with inner color' }, clothing: { style: clothingTraits.signature_outfit || clothingTraits.style || 'casual puppet attire', colors: clothingTraits.color_palette || clothingTraits.colors || ['blue', 'red'], construction: 'removable puppet clothing with authentic fabric', accessories: clothingTraits.accessory_details || clothingTraits.accessories || [] }, personality: { traits: personalityTraits.observed_emotions || personalityTraits.core_traits || ['friendly'], expressions: personalityTraits.facial_mechanics || 'animated puppet expressions', puppet_behavior: this.generatePuppetBehavior(personalityTraits.observed_emotions || personalityTraits.core_traits || ['friendly']) }, puppet_mechanics: { mouth_operation: 'mechanical open/close with no lip movement', hand_controls: 'visible control rods or internal hand manipulation', eye_movement: 'controlled positioning without human-like animation', posture: 'bouncy puppet-like movement and positioning' }, cultural_background: traits.cultural_background || 'Universal', voice_profile: this.generateVoiceDescription(traits) }; return puppetSpecs; } // STEP 2: Generate voice profile for ElevenLabs generateVoiceDescription(traits) { const voiceTraits = []; if (traits.personality) { if (traits.personality.includes('wise') || traits.personality.includes('elderly')) { voiceTraits.push('mature, wise tone'); } if (traits.personality.includes('energetic') || traits.personality.includes('excited')) { voiceTraits.push('upbeat, animated delivery'); } if (traits.personality.includes('gentle') || traits.personality.includes('kind')) { voiceTraits.push('warm, gentle tone'); } } if (traits.physical) { if (traits.physical.size === 'large' || traits.physical.size === 'giant') { voiceTraits.push('deep, resonant voice'); } if (traits.physical.size === 'small' || traits.physical.size === 'tiny') { voiceTraits.push('higher pitched, lighter voice'); } } return voiceTraits.length > 0 ? voiceTraits.join(', ') : 'neutral, friendly puppet voice'; } // STEP 3: Generate script using OpenAI agents conversation async generateScript(characterName, scenario, conversationContext = null) { console.log('📝 GENERATING SCRIPT WITH AI AGENTS'); console.log('===================================\n'); try { // Create conversation between AI agents to develop script ideas const ideaGeneration = await this.openai.chat.completions.create({ // the newest OpenAI model is "gpt-5" which was released August 7, 2025. do not change this unless explicitly requested by the user model: "gpt-5", messages: [ { role: "system", content: `You are a creative writing team of AI agents developing a puppet show script. The main character is ${characterName}. Generate natural conversation between agents discussing creative ideas for the scenario: ${scenario}` }, { role: "user", content: `Start a creative discussion about script ideas for ${characterName} in this scenario: ${scenario}. Include dialogue, character emotions, and scene descriptions.` } ], temperature: 0.8, max_tokens: 2000 }); // Extract script from the agent conversation const scriptGeneration = await this.openai.chat.completions.create({ // the newest OpenAI model is "gpt-5" which was released August 7, 2025. do not change this unless explicitly requested by the user model: "gpt-5", messages: [ { role: "system", content: `Convert the following creative discussion into a structured puppet show script with clear scenes, dialogue, character emotions, and camera directions. Format as: SCENE X - [Description] - CHARACTER: dialogue (emotion) [camera angle]` }, { role: "user", content: ideaGeneration.choices[0].message.content } ], temperature: 0.3, max_tokens: 3000 }); const script = scriptGeneration.choices[0].message.content; // Save script to file const scriptPath = path.join(this.outputPath, 'scripts', `${characterName}_script.txt`); await fs.writeFile(scriptPath, script); console.log(`✅ Script generated and saved to: ${scriptPath}`); return { success: true, script: script, script_path: scriptPath, scenes_count: this.countScenes(script), ready_for_breakdown: true }; } catch (error) { console.error('❌ Script generation failed:', error.message); return { success: false, error: error.message }; } } // STEP 4: Break down script into scene prompts with camera angles async breakdownScript(scriptPath, characterName) { console.log('🎬 BREAKING DOWN SCRIPT INTO SCENES'); console.log('==================================\n'); try { const script = await fs.readFile(scriptPath, 'utf-8'); const sceneBreakdown = await this.openai.chat.completions.create({ // the newest OpenAI model is "gpt-5" which was released August 7, 2025. do not change this unless explicitly requested by the user model: "gpt-5", messages: [ { role: "system", content: `You are a director breaking down a puppet show script into individual visual shots. For each scene, create detailed visual prompts that capture: 1. Character position and emotion 2. Camera angle and framing 3. Background and lighting 4. Specific puppet mechanics (mouth position for dialogue, hand gestures) Format each shot as: SHOT X: [detailed visual prompt for image generation] EMOTION: [character emotion] CAMERA: [camera angle/framing] ACTION: [character action/gesture]` }, { role: "user", content: `Break down this puppet script into individual visual shots:\n\n${script}` } ], temperature: 0.2, max_tokens: 4000 }); const breakdown = sceneBreakdown.choices[0].message.content; // Parse shots into structured data const shots = this.parseSceneBreakdown(breakdown, characterName); // Save breakdown const breakdownPath = path.join(this.outputPath, 'scenes', `${characterName}_breakdown.json`); await fs.writeFile(breakdownPath, JSON.stringify(shots, null, 2)); console.log(`✅ Script broken down into ${shots.length} shots`); console.log(`💾 Breakdown saved to: ${breakdownPath}`); return { success: true, shots: shots, breakdown_path: breakdownPath, total_shots: shots.length, ready_for_image_generation: true }; } catch (error) { console.error('❌ Script breakdown failed:', error.message); return { success: false, error: error.message }; } } // STEP 5: Generate image stills for each scene async generateSceneImages(breakdownPath, characterName) { console.log('🎨 GENERATING SCENE IMAGES'); console.log('=========================\n'); try { const shots = JSON.parse(await fs.readFile(breakdownPath, 'utf-8')); const generatedImages = []; for (let i = 0; i < shots.length; i++) { const shot = shots[i]; console.log(`🎯 Generating image for shot ${i + 1}/${shots.length}...`); // Use existing character pipeline to generate scene image const sceneResult = await this.characterPipeline.generateCharacterScene( characterName, shot.visual_prompt, shot.emotion, shot.action ); if (sceneResult.success) { generatedImages.push({ shot_number: i + 1, image_url: sceneResult.image_url, emotion: shot.emotion, camera_angle: shot.camera, dialogue: shot.dialogue || null }); console.log(`✅ Shot ${i + 1} completed`); } else { console.log(`⚠️ Shot ${i + 1} failed: ${sceneResult.error}`); } // Brief pause to avoid rate limiting await new Promise(resolve => setTimeout(resolve, 2000)); } // Save image metadata const imagesPath = path.join(this.outputPath, 'scenes', `${characterName}_images.json`); await fs.writeFile(imagesPath, JSON.stringify(generatedImages, null, 2)); console.log(`✅ Generated ${generatedImages.length} scene images`); return { success: true, images: generatedImages, images_path: imagesPath, ready_for_video_generation: true }; } catch (error) { console.error('❌ Scene image generation failed:', error.message); return { success: false, error: error.message }; } } // Helper methods determinePuppetSize(physicalTraits) { // Logic to determine puppet size based on character analysis if (!physicalTraits || !physicalTraits.description) return 'medium'; const desc = physicalTraits.description.toLowerCase(); if (desc.includes('large') || desc.includes('tall') || desc.includes('big')) return 'large'; if (desc.includes('small') || desc.includes('tiny') || desc.includes('little')) return 'small'; return 'medium'; } generatePuppetBehavior(personalityTraits) { if (!personalityTraits) return 'neutral puppet behavior'; const behaviors = []; personalityTraits.forEach(trait => { if (trait.includes('energetic')) behaviors.push('bouncy movements'); if (trait.includes('wise')) behaviors.push('thoughtful gestures'); if (trait.includes('friendly')) behaviors.push('open arm movements'); if (trait.includes('shy')) behaviors.push('reserved posture'); }); return behaviors.length > 0 ? behaviors.join(', ') : 'natural puppet behavior'; } countScenes(script) { return (script.match(/SCENE \d+/g) || []).length; } parseSceneBreakdown(breakdown, characterName) { const shots = []; const shotRegex = /SHOT (\d+):\s*(.*?)\nEMOTION:\s*(.*?)\nCAMERA:\s*(.*?)\nACTION:\s*(.*?)(?=\n\nSHOT|\n$|$)/gs; let match; while ((match = shotRegex.exec(breakdown)) !== null) { shots.push({ shot_number: parseInt(match[1]), visual_prompt: match[2].trim(), emotion: match[3].trim(), camera: match[4].trim(), action: match[5].trim(), character_name: characterName }); } return shots; } async storeCharacterBlueprints(characterName, puppetTraits, blueprintImages, imageAnalysis) { // Store character blueprints in Notion database try { const notion = await this.characterPipeline.getNotionClient(); const response = await notion.pages.create({ parent: { database_id: process.env.CHARACTERS_MASTER_DB || '5b96c172-96b2-4b53-a52c-60d4874779f4' }, properties: { 'Character Name': { title: [{ text: { content: characterName } }] }, 'Character Type': { select: { name: 'Production Ready' } }, 'Description': { rich_text: [{ text: { content: imageAnalysis.description || 'Character processed from image' } }] }, 'Physical Traits': { rich_text: [{ text: { content: JSON.stringify(puppetTraits.physical, null, 2) } }] }, 'Production Status': { select: { name: 'Blueprint Complete' } } } }); return { success: true, page_id: response.id }; } catch (error) { console.log('⚠️ Notion storage failed, character still created locally'); return { success: false, error: error.message }; } } // STEP 6: Create ElevenLabs voice integration async generateCharacterVoice(characterName, voiceDescription, sampleText = null) { console.log('🎤 GENERATING CHARACTER VOICE'); console.log('============================\n'); if (!this.elevenLabs) { return { success: false, error: 'ElevenLabs API key not configured', integration_needed: 'Set ELEVENLABS_API_KEY environment variable' }; } try { const voiceResult = await this.elevenLabs.createCharacterVoice( characterName, voiceDescription ); if (voiceResult.success && sampleText) { // Generate sample audio const audioDir = path.join(this.outputPath, 'voices'); await fs.mkdir(audioDir, { recursive: true }); const sampleResult = await this.elevenLabs.generateSpeech( sampleText, voiceResult.voice_id, audioDir, characterName ); return { success: true, voice_id: voiceResult.voice_id, voice_name: voiceResult.voice_name, voice_description: voiceDescription, sample_text: sampleText, sample_audio: sampleResult.audio_path, ready_for_audio_generation: true }; } return voiceResult; } catch (error) { console.error('❌ Voice generation failed:', error.message); return { success: false, error: error.message }; } } // Master orchestrator method async createCompleteProduction(imagePath, scenario, characterName = null) { console.log('🎬 STARTING COMPLETE PRODUCTION PIPELINE'); console.log('=======================================\n'); const results = { character_processing: null, script_generation: null, scene_breakdown: null, image_generation: null, voice_generation: null, ready_for_video_assembly: false }; try { // Step 1: Process character image results.character_processing = await this.processCharacterImage(imagePath, characterName); if (!results.character_processing.success) { throw new Error('Character processing failed'); } const finalCharacterName = results.character_processing.character_name; // Step 2: Generate script results.script_generation = await this.generateScript(finalCharacterName, scenario); if (!results.script_generation.success) { throw new Error('Script generation failed'); } // Step 3: Break down script results.scene_breakdown = await this.breakdownScript( results.script_generation.script_path, finalCharacterName ); if (!results.scene_breakdown.success) { throw new Error('Scene breakdown failed'); } // Step 4: Generate scene images results.image_generation = await this.generateSceneImages( results.scene_breakdown.breakdown_path, finalCharacterName ); if (!results.image_generation.success) { throw new Error('Image generation failed'); } // Step 5: Generate voice profile results.voice_generation = await this.generateCharacterVoice( finalCharacterName, results.character_processing.voice_profile, `Hello, I am ${finalCharacterName}! Let me tell you a story.` ); // Step 6: Generate audio for all scenes if (results.voice_generation.success && results.scene_breakdown.success) { console.log('\n🎵 Step 6: Generating audio for scenes...'); results.audio_generation = await this.generateScriptAudio( results.scene_breakdown.breakdown_path, results.voice_generation.voice_id, finalCharacterName ); } // Step 7: Create video production manifest if (results.image_generation.success) { console.log('\n🎬 Step 7: Creating video production manifest...'); results.video_assembly = await this.createVideoManifest( finalCharacterName, results.image_generation.images, results.audio_generation?.audio_files || [] ); } results.ready_for_video_assembly = results.video_assembly?.success || false; console.log('\n🎉 COMPLETE PRODUCTION PIPELINE FINISHED'); console.log('======================================='); console.log(`✅ Character: ${finalCharacterName}`); console.log(`✅ Script: ${results.script_generation.scenes_count} scenes`); console.log(`✅ Images: ${results.image_generation.images.length} generated`); console.log(`✅ Voice: Profile created`); console.log('🎯 Ready for video assembly and lip sync!'); return { success: true, character_name: finalCharacterName, production_results: results, next_steps: [ 'Connect ElevenLabs for voice generation', 'Implement lip sync video generation', 'Build video assembly system' ] }; } catch (error) { console.error('❌ Production pipeline failed:', error.message); return { success: false, error: error.message, partial_results: results }; } } // STEP 7: Generate audio for script scenes async generateScriptAudio(breakdownPath, voiceId, characterName) { console.log('🎵 GENERATING SCENE AUDIO'); console.log('=========================\n'); if (!this.elevenLabs) { return { success: false, error: 'ElevenLabs not configured', integration_needed: 'Set ELEVENLABS_API_KEY environment variable' }; } try { const breakdown = JSON.parse(await fs.readFile(breakdownPath, 'utf-8')); const audioDir = path.join(this.outputPath, 'audio', characterName); await fs.mkdir(audioDir, { recursive: true }); const audioResult = await this.elevenLabs.generateScriptAudio( breakdown, voiceId, characterName, audioDir ); if (audioResult.success) { // Save audio metadata const audioMetadataPath = path.join(this.outputPath, 'audio', `${characterName}_audio.json`); await fs.writeFile(audioMetadataPath, JSON.stringify(audioResult.audio_files, null, 2)); console.log(`✅ Generated audio for ${audioResult.total_scenes} scenes`); return { success: true, audio_files: audioResult.audio_files, audio_metadata_path: audioMetadataPath, total_audio_files: audioResult.total_scenes }; } return audioResult; } catch (error) { console.error('❌ Scene audio generation failed:', error.message); return { success: false, error: error.message }; } } // STEP 8: Create video production manifest async createVideoManifest(characterName, sceneImages, audioFiles) { console.log('🎬 CREATING VIDEO MANIFEST'); console.log('==========================\n'); try { const manifestResult = await this.videoAssembler.createVideoManifest( characterName, sceneImages, audioFiles, { format: 'mp4', resolution: '1920x1080', fps: 30, quality: 'high', transition: 'fade' } ); if (manifestResult.success) { console.log(`✅ Video manifest created: ${manifestResult.total_duration}s duration`); console.log(`📊 ${manifestResult.total_scenes} scenes ready for rendering`); } return manifestResult; } catch (error) { console.error('❌ Video manifest creation failed:', error.message); return { success: false, error: error.message }; } } }