videoAnalysis.ts•12.9 kB
import type { LoomClient, LoomVideoMetadata, LoomTranscript } from './loomClient'
import type { LoomVideo } from './loomDetector'
import type { VisualElements } from './visualAnalyzer'
import { VisualAnalyzer } from './visualAnalyzer'
export interface VideoAnalysisOptions {
includeTranscript: boolean
extractKeyFrames: boolean
detectAccounts: boolean
analyzeFeatures: boolean
analyzeVisuals?: boolean
maxFrames?: number
}
export interface ProcessedTranscript {
fullText: string
segments: TranscriptSegment[]
keyStatements: string[]
questions: string[]
errors: string[]
speakers?: Speaker[]
}
export interface TranscriptSegment {
text: string
startTime: number
endTime: number
confidence?: number
}
export interface Speaker {
id: string
name?: string
segments: number[]
}
export interface AccountInfo {
emails: string[]
userIds: string[]
organizations: string[]
subscriptionType?: string
}
export interface VideoSummary {
description: string
reproducationSteps: string[]
suggestedActions: string[]
severity: 'critical' | 'high' | 'medium' | 'low'
}
export interface VideoAnalysisResult {
videoId: string
metadata: LoomVideoMetadata | null
publicInfo: any
transcript?: ProcessedTranscript
visualElements?: VisualElements
accountInfo?: AccountInfo
summary: VideoSummary
raw?: {
transcript?: LoomTranscript | null
}
}
export class VideoAnalysisService {
private visualAnalyzer: VisualAnalyzer
constructor(private loomClient: LoomClient) {
this.visualAnalyzer = new VisualAnalyzer(loomClient)
}
async analyzeVideo(
video: LoomVideo,
options: VideoAnalysisOptions = {
includeTranscript: true,
extractKeyFrames: false,
detectAccounts: true,
analyzeFeatures: true,
}
): Promise<VideoAnalysisResult> {
try {
console.log(`Analyzing video ${video.videoId} from ${video.source}`)
// Get public video info (doesn't require auth)
const publicInfo = await this.loomClient.getPublicVideoInfo(video.videoId)
console.log('Public info fetch result:', publicInfo ? 'SUCCESS' : 'NULL')
// Try to get authenticated metadata (may fail without API key)
const metadata = await this.loomClient.getVideo(video.videoId)
console.log('Metadata fetch result:', metadata ? 'SUCCESS' : 'NULL')
// If both are null, we can't analyze the video
if (!publicInfo && !metadata) {
console.error(`Failed to fetch video info for ${video.videoId}`)
console.error('Video URL:', video.url)
console.error('Video source:', video.source)
throw new Error(`Unable to fetch video information for video ID: ${video.videoId}`)
}
let processedTranscript: ProcessedTranscript | undefined
let rawTranscript: LoomTranscript | null = null
if (options.includeTranscript && metadata) {
rawTranscript = await this.loomClient.getTranscript(video.videoId)
if (rawTranscript) {
processedTranscript = this.processTranscript(rawTranscript)
}
}
let accountInfo: AccountInfo | undefined
if (options.detectAccounts && (processedTranscript || metadata)) {
accountInfo = this.detectAccountInfo(processedTranscript, publicInfo, metadata)
}
let visualElements: VisualElements | undefined
if (options.analyzeVisuals || options.analyzeVisuals === undefined) {
visualElements = await this.visualAnalyzer.extractVisualElements(
video.videoId,
rawTranscript
)
}
const summary = this.generateSummary(
video,
metadata,
publicInfo,
processedTranscript,
accountInfo,
visualElements
)
return {
videoId: video.videoId,
metadata,
publicInfo,
transcript: processedTranscript,
visualElements,
accountInfo,
summary,
raw: {
transcript: rawTranscript,
},
}
} catch (error) {
console.error('Error in analyzeVideo:', error)
throw error
}
}
private processTranscript(transcript: LoomTranscript): ProcessedTranscript {
const segments: TranscriptSegment[] = transcript.segments.map((seg) => ({
text: seg.text,
startTime: seg.start_time,
endTime: seg.end_time,
confidence: seg.confidence,
}))
const fullText = segments.map((s) => s.text).join(' ')
// Extract key statements (simple heuristic: sentences with certain keywords)
const keyStatements = this.extractKeyStatements(fullText)
// Extract questions
const questions = this.extractQuestions(fullText)
// Extract error messages
const errors = this.extractErrors(fullText)
return {
fullText,
segments,
keyStatements,
questions,
errors,
}
}
private extractKeyStatements(text: string): string[] {
const keywords = [
'error',
'bug',
'issue',
'problem',
'broken',
'fail',
'crash',
'stuck',
'freeze',
'slow',
'wrong',
'incorrect',
'missing',
'disappear',
'blank',
'empty',
'null',
'undefined',
"can't",
'cannot',
"won't",
"doesn't",
"isn't",
"aren't",
'notice',
'seeing',
'getting',
'happens',
'occurs',
'when i',
'if i',
'after i',
'before i',
'expected',
'should be',
'supposed to',
]
const sentences = text.match(/[^.!?]+[.!?]+/g) || []
const keyStatements: string[] = []
for (const sentence of sentences) {
const lowerSentence = sentence.toLowerCase().trim()
if (keywords.some((keyword) => lowerSentence.includes(keyword))) {
keyStatements.push(sentence.trim())
}
}
return keyStatements.slice(0, 10) // Limit to 10 most relevant
}
private extractQuestions(text: string): string[] {
const questions = text.match(/[^.!?]*\?/g) || []
return questions.map((q) => q.trim()).filter((q) => q.length > 10)
}
private extractErrors(text: string): string[] {
const errorPatterns = [
/error[:\s]+([^.!?\n]+)/gi,
/exception[:\s]+([^.!?\n]+)/gi,
/failed[:\s]+([^.!?\n]+)/gi,
/unable to[:\s]+([^.!?\n]+)/gi,
/could not[:\s]+([^.!?\n]+)/gi,
/message[:\s]+["']([^"']+)["']/gi,
]
const errors: string[] = []
for (const pattern of errorPatterns) {
const matches = [...text.matchAll(pattern)]
for (const match of matches) {
if (match[1]) {
errors.push(match[1].trim())
}
}
}
return [...new Set(errors)] // Remove duplicates
}
private detectAccountInfo(
transcript?: ProcessedTranscript,
publicInfo?: any,
metadata?: LoomVideoMetadata | null
): AccountInfo {
const accountInfo: AccountInfo = {
emails: [],
userIds: [],
organizations: [],
}
// Extract from metadata
if (metadata?.owner?.email) {
accountInfo.emails.push(metadata.owner.email)
}
// Extract from public info
// Note: OEmbed API doesn't provide author_name, only provider info
// Extract from transcript
if (transcript) {
const emailRegex = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g
const emails = transcript.fullText.match(emailRegex) || []
accountInfo.emails.push(...emails)
// Look for user IDs
const userIdPatterns = [
/user[_\s-]?id[:\s]+([a-zA-Z0-9-]+)/gi,
/uid[:\s]+([a-zA-Z0-9-]+)/gi,
/account[:\s]+([a-zA-Z0-9-]+)/gi,
/customer[_\s-]?id[:\s]+([a-zA-Z0-9-]+)/gi,
]
for (const pattern of userIdPatterns) {
const matches = [...transcript.fullText.matchAll(pattern)]
for (const match of matches) {
if (match[1]) {
accountInfo.userIds.push(match[1])
}
}
}
}
// Remove duplicates
accountInfo.emails = [...new Set(accountInfo.emails)]
accountInfo.userIds = [...new Set(accountInfo.userIds)]
accountInfo.organizations = [...new Set(accountInfo.organizations)]
return accountInfo
}
private generateSummary(
video: LoomVideo,
metadata: LoomVideoMetadata | null,
publicInfo: any,
transcript?: ProcessedTranscript,
accountInfo?: AccountInfo,
visualElements?: VisualElements
): VideoSummary {
let description = ''
const reproducationSteps: string[] = []
const suggestedActions: string[] = []
// Build description
if (metadata && typeof metadata === 'object' && metadata !== null) {
const videoName = (metadata as any).name || 'Untitled'
const duration = (metadata as any).duration
? `${Math.round((metadata as any).duration)}s`
: 'unknown duration'
const author = (metadata as any).owner?.name || 'Unknown'
description = `Loom video "${videoName}" (${duration}) by ${author}`
if ((metadata as any).description) {
description += `: ${(metadata as any).description}`
}
} else if (publicInfo) {
description = `Loom video "${publicInfo.title || 'Untitled'}" (${publicInfo.duration ? Math.round(publicInfo.duration) + 's' : 'unknown duration'})`
} else {
description = `Loom video ${video.videoId} from ${video.source}`
}
// Extract reproduction steps from transcript
if (transcript && transcript.keyStatements.length > 0) {
// Look for sequential actions
const actionKeywords = [
'click',
'tap',
'press',
'select',
'enter',
'type',
'navigate',
'go to',
'open',
]
for (const statement of transcript.keyStatements) {
if (actionKeywords.some((kw) => statement.toLowerCase().includes(kw))) {
reproducationSteps.push(statement)
}
}
}
// Generate suggested actions based on errors and issues
if (transcript?.errors.length) {
suggestedActions.push('Review error messages found in video')
suggestedActions.push(`Investigate: ${transcript.errors[0]}`)
}
if (accountInfo?.emails.length) {
suggestedActions.push(`Check account for user: ${accountInfo.emails[0]}`)
}
if (transcript?.questions.length) {
suggestedActions.push('Address user questions from video')
}
// Determine severity
let severity: VideoSummary['severity'] = 'medium'
if (transcript) {
const criticalKeywords = [
'crash',
'data loss',
'security',
'payment',
'cannot access',
'completely broken',
]
const highKeywords = ['error', 'fail', 'broken', 'bug', 'stuck']
const lowerText = transcript.fullText.toLowerCase()
if (criticalKeywords.some((kw) => lowerText.includes(kw))) {
severity = 'critical'
} else if (highKeywords.some((kw) => lowerText.includes(kw))) {
severity = 'high'
}
}
return {
description,
reproducationSteps,
suggestedActions,
severity,
}
}
}