Shortcut MCP Server

# Loom Integration Technical Specification ## Phase 1 Implementation Details ### 1. Loom URL Detection ```typescript // src/loomDetector.ts export interface LoomVideo { url: string videoId: string source: 'description' | 'comment' | 'linked_file' sourceId?: number timestamp?: string } export function extractLoomVideos(story: Story): LoomVideo[] { const videos: LoomVideo[] = [] // Regex for Loom URLs const loomRegex = /https?:\/\/(?:www\.)?loom\.com\/share\/([a-zA-Z0-9]+)(?:\?[^\s]*)?/g // Check description const descriptionVideos = extractFromText(story.description, 'description') videos.push(...descriptionVideos) // Check comments story.comments.forEach(comment => { const commentVideos = extractFromText(comment.text, 'comment', comment.id) videos.push(...commentVideos) }) // Check linked files story.linked_files?.forEach(file => { if (file.url?.includes('loom.com')) { videos.push({ url: file.url, videoId: extractVideoId(file.url), source: 'linked_file', sourceId: file.id }) } }) return videos } ``` ### 2. Loom API Client ```typescript // src/loomClient.ts export interface LoomConfig { apiKey?: string personalAccessToken?: string baseUrl: string } export interface LoomTranscript { text: string segments: Array<{ text: string start_time: number end_time: number confidence?: number }> } export interface LoomVideo { id: string name: string description?: string duration: number created_at: string updated_at: string owner: { id: string name: string email?: string } privacy: 'public' | 'private' | 'password' | 'organization' thumbnail_url: string gif_thumbnail_url?: string video_url?: string embed_url: string transcript_available: boolean } export class LoomClient { constructor(private config: LoomConfig) {} async getVideo(videoId: string): Promise<LoomVideo> { // GET /v1/videos/{video_id} } async getTranscript(videoId: string): Promise<LoomTranscript | null> { // GET /v1/videos/{video_id}/transcript } async getVideoChapters(videoId: string): Promise<VideoChapter[]> { // GET /v1/videos/{video_id}/chapters } async downloadVideo(videoId: string): Promise<Buffer> { // Requires special permissions } } ``` ### 3. Video Analysis Service ```typescript // src/videoAnalysis.ts export interface VideoAnalysisOptions { includeTranscript: boolean extractKeyFrames: boolean detectAccounts: boolean analyzeFeatures: boolean maxFrames?: number } export interface VideoAnalysisResult { videoId: string metadata: LoomVideo transcript?: ProcessedTranscript keyFrames?: KeyFrame[] accountInfo?: AccountInfo features?: FeatureUsage[] summary: VideoSummary } export interface ProcessedTranscript { fullText: string segments: TranscriptSegment[] keyStatements: string[] questions: string[] errors: string[] speakers?: Speaker[] } export interface KeyFrame { timestamp: number frameUrl: string ocrText?: string detectedElements?: UIElement[] isErrorState?: boolean description?: string } export class VideoAnalysisService { constructor( private loomClient: LoomClient, private ocrService: OCRService, private aiService: AIService ) {} async analyzeVideo( videoId: string, options: VideoAnalysisOptions ): Promise<VideoAnalysisResult> { // Implementation } } ``` ### 4. OCR and Visual Analysis ```typescript // src/ocrService.ts export interface OCRService { extractText(imageUrl: string): Promise<string> detectUIElements(imageUrl: string): Promise<UIElement[]> findAccounts(text: string): AccountMatches } export interface UIElement { type: 'button' | 'input' | 'link' | 'error' | 'modal' text: string boundingBox: BoundingBox confidence: number } export interface AccountMatches { emails: Array<{ value: string; confidence: number }> userIds: Array<{ value: string; confidence: number }> organizations: Array<{ value: string; confidence: number }> } // Implementation using Google Cloud Vision or AWS Textract export class CloudVisionOCRService implements OCRService { async extractText(imageUrl: string): Promise<string> { // Use Google Cloud Vision API } async detectUIElements(imageUrl: string): Promise<UIElement[]> { // Custom logic or ML model } findAccounts(text: string): AccountMatches { // Regex patterns for common formats const patterns = { email: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, userId: /\b(user[_-]?id|uid|id)[\s:]+([a-zA-Z0-9-]+)\b/gi, uuid: /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, orgName: /\b(org|organization|company|team)[\s:]+([A-Za-z0-9\s]+)\b/gi } // Extract matches with confidence scoring } } ``` ### 5. Enhanced MCP Tools ```typescript // Add to src/server.ts const GetStoryLoomVideosSchema = ToolSchema.extend({ name: z.literal('shortcut_get_story_loom_videos'), description: z.literal('Extract all Loom video links from a story'), inputSchema: z.object({ story_id: z.number().describe('The numeric ID of the story'), }), }) const AnalyzeLoomVideoSchema = ToolSchema.extend({ name: z.literal('shortcut_analyze_loom_video'), description: z.literal('Analyze a Loom video for debugging information'), inputSchema: z.object({ video_url: z.string().describe('The Loom video URL'), include_transcript: z.boolean().optional().default(true), extract_key_frames: z.boolean().optional().default(true), detect_accounts: z.boolean().optional().default(true), max_frames: z.number().optional().default(10), }), }) const BatchAnalyzeVideosSchema = ToolSchema.extend({ name: z.literal('shortcut_batch_analyze_videos'), description: z.literal('Analyze all Loom videos in a story'), inputSchema: z.object({ story_id: z.number().describe('The numeric ID of the story'), analysis_options: z.object({ include_transcript: z.boolean().optional(), extract_key_frames: z.boolean().optional(), detect_accounts: z.boolean().optional(), }).optional(), }), }) ``` ### 6. Caching Strategy ```typescript // src/videoCache.ts export interface CacheEntry<T> { data: T timestamp: number ttl: number etag?: string } export class VideoAnalysisCache { private cache = new Map<string, CacheEntry<any>>() async get<T>(key: string): Promise<T | null> { const entry = this.cache.get(key) if (!entry) return null if (Date.now() - entry.timestamp > entry.ttl) { this.cache.delete(key) return null } return entry.data } async set<T>(key: string, data: T, ttl: number = 3600000): Promise<void> { this.cache.set(key, { data, timestamp: Date.now(), ttl }) } getCacheKey(videoId: string, options: any): string { return `loom:${videoId}:${JSON.stringify(options)}` } } ``` ### 7. Example Output Format ```json { "videoId": "abc123def456", "metadata": { "name": "Bug Report: Checkout Flow Error", "duration": 185, "created_at": "2024-01-15T10:30:00Z", "owner": { "name": "John Doe", "email": "john@example.com" } }, "transcript": { "fullText": "Hi, I'm trying to complete a purchase but I'm getting an error...", "keyStatements": [ "Error occurs when clicking 'Complete Purchase'", "Only happens for orders over $500", "Started happening yesterday around 3 PM" ], "questions": [ "Is this related to the recent payment gateway update?" ], "errors": [ "Payment processing failed: Transaction limit exceeded" ] }, "accountInfo": { "emails": ["customer@example.com"], "userIds": ["usr_123456"], "organizations": ["Acme Corp"], "subscriptionType": "Enterprise" }, "keyFrames": [ { "timestamp": 45, "description": "Error modal appears", "ocrText": "Payment processing failed: Transaction limit exceeded", "isErrorState": true }, { "timestamp": 120, "description": "Account settings page", "ocrText": "Account: customer@example.com\nPlan: Enterprise\nUsage: $1,234 / $500", "detectedElements": [ { "type": "error", "text": "Limit Exceeded", "confidence": 0.95 } ] } ], "features": [ { "featureName": "Checkout", "actions": [ { "timestamp": 30, "actionType": "click", "elementText": "Add to Cart", "outcome": "success" }, { "timestamp": 42, "actionType": "click", "elementText": "Complete Purchase", "outcome": "error" } ], "errorMessages": ["Transaction limit exceeded"] } ], "summary": { "description": "User encounters payment limit error during checkout for orders over $500", "reproducationSteps": [ "1. Add items totaling over $500 to cart", "2. Proceed to checkout", "3. Click 'Complete Purchase'", "4. Error appears: 'Transaction limit exceeded'" ], "suggestedActions": [ "Check account payment limits configuration", "Verify recent payment gateway changes", "Review transaction limit logic for Enterprise accounts" ], "severity": "high" } } ``` ## Development Roadmap ### Week 1: Foundation - [ ] Implement Loom URL detection - [ ] Create Loom API client - [ ] Basic transcript extraction - [ ] Simple MCP tool integration ### Week 2: Transcription Enhancement - [ ] Process transcript segments - [ ] Extract key statements and questions - [ ] Identify error messages in speech - [ ] Generate transcript summary ### Week 3: Visual Analysis MVP - [ ] Integrate Google Cloud Vision API - [ ] Basic OCR implementation - [ ] Email and ID detection - [ ] Key frame extraction at intervals ### Week 4: Advanced Analysis - [ ] Scene change detection - [ ] UI element recognition - [ ] Error state detection - [ ] Account information aggregation ### Week 5: Intelligence Layer - [ ] AI-powered summarization - [ ] Automatic reproduction steps - [ ] Severity assessment - [ ] Feature usage tracking ### Week 6: Production Ready - [ ] Caching implementation - [ ] Error handling and retries - [ ] Performance optimization - [ ] Documentation and tests