enhancedVideoAnalyzer.ts•34.1 kB
import type { VideoCodeAnalysis } from './videoCodeAnalyzer'
import { VideoCodeAnalyzer } from './videoCodeAnalyzer'
import type { VisualElements, TimestampedElement } from './visualAnalyzer'
import { LoomClient } from './loomClient'
import type { VideoAnalysisService } from './videoAnalysis'
import type { AudioAnalysisResult, VerbalIssue } from './audioAnalyzer'
import { AudioAnalyzer } from './audioAnalyzer'
import * as fs from 'fs'
export interface ScreenState {
timestamp: number
route: string
componentStack: string[]
visibleElements: string[]
dataPresent: Record<string, any>
}
export interface UserAction {
timestamp: number
type: 'click' | 'type' | 'navigate' | 'scroll' | 'hover' | 'submit'
target: string
value?: string
result?: 'success' | 'error' | 'loading'
}
export interface ErrorOccurrence {
timestamp: number
errorMessage: string
errorType: 'ui' | 'console' | 'network' | 'validation'
possibleCauses: string[]
relatedCode: CodeReference[]
}
export interface CodeReference {
file: string
line: number
type: 'error_handler' | 'validation' | 'api_call' | 'component' | 'state_update'
snippet: string
}
export interface DataFlow {
timestamp: number
dataType: string
source: string
destination: string
transformations: string[]
value?: any
}
export interface EnhancedVideoAnalysis extends VideoCodeAnalysis {
screenStates: ScreenState[]
userActions: UserAction[]
errors: ErrorOccurrence[]
dataFlows: DataFlow[]
debuggingSuggestions: DebuggingSuggestion[]
reproducibleSteps: ReproducibleStep[]
audioAnalysis?: AudioAnalysisResult
verbalVisualCorrelations?: VerbalVisualCorrelation[]
}
export interface DebuggingSuggestion {
priority: 'high' | 'medium' | 'low'
type: 'breakpoint' | 'log' | 'inspect' | 'trace'
location: CodeReference
reason: string
additionalContext?: string
}
export interface ReproducibleStep {
order: number
action: string
expectedResult: string
actualResult?: string
timestamp?: number
}
export interface VerbalVisualCorrelation {
verbalIssue: VerbalIssue
visualEvent: TimestampedElement | null
correlationScore: number
combinedInsight?: string
}
export class EnhancedVideoAnalyzer {
private videoCodeAnalyzer: VideoCodeAnalyzer
private audioAnalyzer: AudioAnalyzer
constructor(
private loomClient: LoomClient,
private videoAnalysisService: VideoAnalysisService,
private codebasePath: string
) {
this.videoCodeAnalyzer = new VideoCodeAnalyzer(
loomClient,
videoAnalysisService,
codebasePath
)
this.audioAnalyzer = new AudioAnalyzer()
}
async analyzeVideoInDepth(videoUrl: string): Promise<EnhancedVideoAnalysis> {
// Get base analysis
const baseAnalysis = await this.videoCodeAnalyzer.analyzeVideoWithCodebase(videoUrl)
// Get detailed video analysis with visuals
const videoAnalysis = await this.videoAnalysisService.analyzeVideo(
{ url: videoUrl, videoId: this.extractVideoId(videoUrl), source: 'direct' },
{
includeTranscript: true,
analyzeVisuals: true,
detectAccounts: true,
extractKeyFrames: false,
analyzeFeatures: true,
}
)
// Extract screen states
const screenStates = await this.extractScreenStates(
videoAnalysis.visualElements,
videoAnalysis.transcript
)
// Track user actions
const userActions = await this.trackUserActions(
videoAnalysis.visualElements,
videoAnalysis.transcript
)
// Correlate errors with code
const errors = await this.correlateErrors(
baseAnalysis,
videoAnalysis.visualElements,
screenStates
)
// Analyze data flow
const dataFlows = await this.analyzeDataFlow(
userActions,
screenStates,
videoAnalysis.transcript
)
// Generate debugging suggestions
const debuggingSuggestions = await this.generateDebuggingSuggestions(
errors,
userActions,
baseAnalysis.relevantFiles
)
// Create reproducible steps
const reproducibleSteps = this.createReproducibleSteps(userActions, screenStates, errors)
// Analyze audio if transcript is available
let audioAnalysis: AudioAnalysisResult | undefined
let verbalVisualCorrelations: VerbalVisualCorrelation[] | undefined
if (videoAnalysis.transcript) {
audioAnalysis = await this.audioAnalyzer.analyzeTranscript(videoAnalysis.transcript)
// Correlate verbal issues with visual events
if (videoAnalysis.visualElements?.timestamps) {
const correlations = this.audioAnalyzer.correlateWithVisualEvents(
audioAnalysis,
videoAnalysis.visualElements.timestamps
)
verbalVisualCorrelations = correlations.map((c) => ({
verbalIssue: c.verbal,
visualEvent: c.visual,
correlationScore: c.correlation,
combinedInsight: this.generateCombinedInsight(c.verbal, c.visual),
}))
}
// Enhance error detection with verbal issues
this.enhanceErrorsWithVerbalIssues(errors, audioAnalysis, verbalVisualCorrelations)
// Update debugging suggestions based on audio analysis
this.enhanceDebuggingSuggestionsWithAudio(debuggingSuggestions, audioAnalysis)
}
return {
...baseAnalysis,
screenStates,
userActions,
errors,
dataFlows,
debuggingSuggestions,
reproducibleSteps,
audioAnalysis,
verbalVisualCorrelations,
}
}
private async extractScreenStates(
visualElements?: VisualElements,
transcript?: any
): Promise<ScreenState[]> {
const states: ScreenState[] = []
const stateMap = new Map<number, ScreenState>()
// Extract from visual elements
if (visualElements?.timestamps) {
for (const element of visualElements.timestamps) {
const timestamp = Math.floor(element.timestamp)
if (!stateMap.has(timestamp)) {
stateMap.set(timestamp, {
timestamp,
route: '',
componentStack: [],
visibleElements: [],
dataPresent: {},
})
}
const state = stateMap.get(timestamp)!
if (element.type === 'navigation' && element.content.startsWith('/')) {
state.route = element.content
}
if (element.type === 'ui_text') {
state.visibleElements.push(element.content)
// Try to identify components
const componentMatch = element.content.match(
/\b([A-Z][a-zA-Z]+(?:Component|Page|Modal|Form))\b/
)
if (componentMatch) {
state.componentStack.push(componentMatch[1])
}
}
}
}
// Extract from URLs mentioned
if (visualElements?.urlsAndRoutes) {
for (const url of visualElements.urlsAndRoutes) {
if (url.startsWith('/')) {
// Find the closest timestamp
const closestState = this.findOrCreateClosestState(stateMap, 0)
closestState.route = url
}
}
}
// Extract data from form fields
if (visualElements?.formFields) {
for (const field of visualElements.formFields) {
const closestState = this.findOrCreateClosestState(stateMap, 0)
closestState.dataPresent[field] = '<user input>'
}
}
// Convert map to sorted array
states.push(...Array.from(stateMap.values()).sort((a, b) => a.timestamp - b.timestamp))
// Infer routes from component names if no explicit routes
for (const state of states) {
if (!state.route && state.componentStack.length > 0) {
// Try to infer route from component names
const mainComponent =
state.componentStack.find((c) => c.includes('Page')) || state.componentStack[0]
if (mainComponent) {
state.route = '/' + mainComponent.replace(/Page|Component/, '').toLowerCase()
}
}
}
return states
}
private async trackUserActions(
visualElements?: VisualElements,
transcript?: any
): Promise<UserAction[]> {
const actions: UserAction[] = []
if (!visualElements) return actions
// Pattern matching for actions in transcript
const actionPatterns = [
{
pattern: /click(?:ing|ed)?\s+(?:on\s+)?(?:the\s+)?["']?([^"']+)["']?/gi,
type: 'click' as const,
},
{
pattern: /press(?:ing|ed)?\s+(?:the\s+)?["']?([^"']+)["']?/gi,
type: 'click' as const,
},
{
pattern: /tap(?:ping|ped)?\s+(?:on\s+)?["']?([^"']+)["']?/gi,
type: 'click' as const,
},
{
pattern:
/typ(?:e|ing|ed)\s+["']?([^"']+)["']?\s+(?:in|into)\s+(?:the\s+)?["']?([^"']+)["']?/gi,
type: 'type' as const,
},
{
pattern:
/enter(?:ing|ed)?\s+["']?([^"']+)["']?\s+(?:in|into)\s+(?:the\s+)?["']?([^"']+)["']?/gi,
type: 'type' as const,
},
{
pattern: /navigat(?:e|ing|ed)\s+(?:to\s+)?["']?([^"']+)["']?/gi,
type: 'navigate' as const,
},
{ pattern: /go(?:ing)?\s+(?:to\s+)?["']?([^"']+)["']?/gi, type: 'navigate' as const },
{
pattern: /submit(?:ting|ted)?\s+(?:the\s+)?["']?([^"']+)["']?/gi,
type: 'submit' as const,
},
{
pattern: /scroll(?:ing|ed)?\s+(?:to\s+)?["']?([^"']+)["']?/gi,
type: 'scroll' as const,
},
]
// Extract from timestamps
if (visualElements.timestamps) {
for (const element of visualElements.timestamps) {
if (element.type === 'interaction') {
// Parse interaction content
for (const { pattern, type } of actionPatterns) {
const matches = [...element.content.matchAll(pattern)]
for (const match of matches) {
actions.push({
timestamp: element.timestamp,
type,
target: match[1],
value: match[2] || undefined,
result: 'success', // Default, will be updated if error follows
})
}
}
}
}
}
// Extract from button labels
for (const button of visualElements.buttonLabels) {
actions.push({
timestamp: 0, // No specific timestamp
type: 'click',
target: button,
result: 'success',
})
}
// Look for errors that follow actions to mark them as failed
if (visualElements.errorMessages.length > 0) {
for (let i = actions.length - 1; i >= 0; i--) {
const action = actions[i]
// Check if any error occurred within 5 seconds of the action
const errorNearby = visualElements.timestamps.some(
(e) => e.type === 'error' && Math.abs(e.timestamp - action.timestamp) < 5
)
if (errorNearby) {
action.result = 'error'
}
}
}
// Sort by timestamp
return actions.sort((a, b) => a.timestamp - b.timestamp)
}
private async correlateErrors(
baseAnalysis: VideoCodeAnalysis,
visualElements?: VisualElements,
screenStates?: ScreenState[]
): Promise<ErrorOccurrence[]> {
const errors: ErrorOccurrence[] = []
if (!visualElements) return errors
// Process each error message
for (const errorMsg of visualElements.errorMessages) {
const error: ErrorOccurrence = {
timestamp: 0, // Will be set if we find timestamp
errorMessage: errorMsg,
errorType: this.classifyError(errorMsg),
possibleCauses: this.identifyPossibleCauses(errorMsg),
relatedCode: [],
}
// Find timestamp for this error
const timestampedError = visualElements.timestamps.find(
(t) => t.type === 'error' && t.content.includes(errorMsg)
)
if (timestampedError) {
error.timestamp = timestampedError.timestamp
}
// Find related code
error.relatedCode = await this.findErrorRelatedCode(
errorMsg,
baseAnalysis.relevantFiles
)
errors.push(error)
}
return errors
}
private classifyError(errorMessage: string): ErrorOccurrence['errorType'] {
const lowerError = errorMessage.toLowerCase()
if (
lowerError.includes('network') ||
lowerError.includes('fetch') ||
lowerError.includes('api')
) {
return 'network'
} else if (
lowerError.includes('validation') ||
lowerError.includes('invalid') ||
lowerError.includes('required')
) {
return 'validation'
} else if (
lowerError.includes('console') ||
lowerError.includes('uncaught') ||
lowerError.includes('exception')
) {
return 'console'
} else {
return 'ui'
}
}
private identifyPossibleCauses(errorMessage: string): string[] {
const causes: string[] = []
const lowerError = errorMessage.toLowerCase()
if (lowerError.includes('network') || lowerError.includes('failed to fetch')) {
causes.push('Network connectivity issue')
causes.push('API endpoint not available')
causes.push('CORS configuration problem')
}
if (lowerError.includes('undefined') || lowerError.includes('null')) {
causes.push('Missing data or uninitialized state')
causes.push('Race condition in data loading')
causes.push('Incorrect property access')
}
if (lowerError.includes('validation') || lowerError.includes('invalid')) {
causes.push('Input validation rules not met')
causes.push('Data type mismatch')
causes.push('Missing required fields')
}
if (lowerError.includes('permission') || lowerError.includes('unauthorized')) {
causes.push('Authentication issue')
causes.push('Insufficient permissions')
causes.push('Session expired')
}
return causes
}
private async findErrorRelatedCode(
errorMessage: string,
relevantFiles: any[]
): Promise<CodeReference[]> {
const references: CodeReference[] = []
// Extract key words from error
const errorKeywords = errorMessage
.split(/\s+/)
.filter((w) => w.length > 3 && !/^(the|and|or|in|on|at|to|for)$/i.test(w))
.slice(0, 5)
for (const file of relevantFiles.slice(0, 10)) {
try {
const content = fs.readFileSync(file.path, 'utf8')
const lines = content.split('\n')
for (let i = 0; i < lines.length; i++) {
const line = lines[i]
const lineLower = line.toLowerCase()
// Check for error handling
if (
lineLower.includes('catch') ||
lineLower.includes('error') ||
lineLower.includes('throw')
) {
// Check if any error keywords match
if (errorKeywords.some((kw) => lineLower.includes(kw.toLowerCase()))) {
references.push({
file: file.path,
line: i + 1,
type: 'error_handler',
snippet: line.trim(),
})
}
}
// Check for validation
if (
lineLower.includes('validate') ||
lineLower.includes('required') ||
lineLower.includes('invalid')
) {
references.push({
file: file.path,
line: i + 1,
type: 'validation',
snippet: line.trim(),
})
}
}
} catch (error) {
// Skip files that can't be read
}
}
return references.slice(0, 5) // Limit to top 5 references
}
private async analyzeDataFlow(
userActions: UserAction[],
screenStates: ScreenState[],
transcript?: any
): Promise<DataFlow[]> {
const dataFlows: DataFlow[] = []
// Track data from user inputs
const typeActions = userActions.filter((a) => a.type === 'type')
for (const action of typeActions) {
if (action.value) {
dataFlows.push({
timestamp: action.timestamp,
dataType: 'user_input',
source: `User (${action.target})`,
destination: action.target,
transformations: [],
value: action.value,
})
}
}
// Track data appearing in UI
for (const state of screenStates) {
for (const [field, value] of Object.entries(state.dataPresent)) {
dataFlows.push({
timestamp: state.timestamp,
dataType: 'display_data',
source: state.route || 'unknown',
destination: field,
transformations: [],
value,
})
}
}
// Analyze submit actions for data flow
const submitActions = userActions.filter((a) => a.type === 'submit')
for (const submit of submitActions) {
// Find preceding type actions
const precedingInputs = userActions
.filter((a) => a.type === 'type' && a.timestamp < submit.timestamp)
.slice(-5) // Last 5 inputs
if (precedingInputs.length > 0) {
dataFlows.push({
timestamp: submit.timestamp,
dataType: 'form_submission',
source: precedingInputs.map((i) => i.target).join(', '),
destination: submit.target,
transformations: ['validation', 'serialization'],
value: {
inputs: precedingInputs.map((i) => ({ field: i.target, value: i.value })),
},
})
}
}
return dataFlows
}
private async generateDebuggingSuggestions(
errors: ErrorOccurrence[],
userActions: UserAction[],
relevantFiles: any[]
): Promise<DebuggingSuggestion[]> {
const suggestions: DebuggingSuggestion[] = []
// Suggest breakpoints for errors
for (const error of errors) {
for (const codeRef of error.relatedCode) {
suggestions.push({
priority: 'high',
type: 'breakpoint',
location: codeRef,
reason: `Error occurred: "${error.errorMessage}". Set breakpoint to investigate.`,
additionalContext: error.possibleCauses.join('; '),
})
}
}
// Suggest logging for failed actions
const failedActions = userActions.filter((a) => a.result === 'error')
for (const action of failedActions) {
// Find relevant component file
const componentFile = relevantFiles.find(
(f) =>
f.reason.includes('Component') &&
(f.path.toLowerCase().includes(action.target.toLowerCase()) ||
f.snippets?.some((s) =>
s.content.toLowerCase().includes(action.target.toLowerCase())
))
)
if (componentFile) {
suggestions.push({
priority: 'medium',
type: 'log',
location: {
file: componentFile.path,
line: componentFile.snippets?.[0]?.line || 1,
type: 'component',
snippet: componentFile.snippets?.[0]?.content || '',
},
reason: `Action "${action.type}" on "${action.target}" failed. Add logging to trace the issue.`,
})
}
}
// Suggest state inspection for data flow issues
const complexDataFlows = userActions.filter((a) => a.type === 'type' || a.type === 'submit')
if (complexDataFlows.length > 3) {
const stateFile = relevantFiles.find(
(f) =>
f.path.includes('state') ||
f.path.includes('store') ||
f.path.includes('context')
)
if (stateFile) {
suggestions.push({
priority: 'medium',
type: 'inspect',
location: {
file: stateFile.path,
line: 1,
type: 'state_update',
snippet: '',
},
reason: 'Complex data flow detected. Inspect state changes during form interactions.',
})
}
}
return suggestions
}
private createReproducibleSteps(
userActions: UserAction[],
screenStates: ScreenState[],
errors: ErrorOccurrence[]
): ReproducibleStep[] {
const steps: ReproducibleStep[] = []
let stepOrder = 1
// Group actions by screen state
for (let i = 0; i < screenStates.length; i++) {
const state = screenStates[i]
const nextState = screenStates[i + 1]
// Add navigation step if route changed
if (i === 0 || (i > 0 && state.route !== screenStates[i - 1].route)) {
steps.push({
order: stepOrder++,
action: `Navigate to ${state.route || 'initial page'}`,
expectedResult: `Page loads with ${state.visibleElements.slice(0, 3).join(', ')}`,
timestamp: state.timestamp,
})
}
// Add actions that occurred in this state
const stateActions = userActions.filter((a) => {
if (nextState) {
return a.timestamp >= state.timestamp && a.timestamp < nextState.timestamp
} else {
return a.timestamp >= state.timestamp
}
})
for (const action of stateActions) {
let actionDesc = ''
let expectedResult = ''
switch (action.type) {
case 'click':
actionDesc = `Click on "${action.target}"`
expectedResult =
action.result === 'error'
? 'Error occurs'
: 'Action completes successfully'
break
case 'type':
actionDesc = `Type "${action.value}" into "${action.target}"`
expectedResult = 'Input is accepted'
break
case 'submit':
actionDesc = `Submit form "${action.target}"`
expectedResult =
action.result === 'error'
? 'Form submission fails'
: 'Form submits successfully'
break
case 'navigate':
actionDesc = `Navigate to ${action.target}`
expectedResult = 'Navigation completes'
break
default:
actionDesc = `${action.type} "${action.target}"`
expectedResult = 'Action completes'
}
steps.push({
order: stepOrder++,
action: actionDesc,
expectedResult,
actualResult: action.result === 'error' ? 'Error occurred' : undefined,
timestamp: action.timestamp,
})
}
// Add any errors that occurred
const stateErrors = errors.filter((e) => {
if (nextState) {
return e.timestamp >= state.timestamp && e.timestamp < nextState.timestamp
} else {
return e.timestamp >= state.timestamp
}
})
for (const error of stateErrors) {
steps.push({
order: stepOrder++,
action: 'Observe error',
expectedResult: 'No error',
actualResult: error.errorMessage,
timestamp: error.timestamp,
})
}
}
return steps
}
private findOrCreateClosestState(
stateMap: Map<number, ScreenState>,
timestamp: number
): ScreenState {
// Round to nearest second
const roundedTime = Math.floor(timestamp)
if (stateMap.has(roundedTime)) {
return stateMap.get(roundedTime)!
}
// Find closest existing state
let closestTime = roundedTime
let minDiff = Infinity
for (const time of stateMap.keys()) {
const diff = Math.abs(time - roundedTime)
if (diff < minDiff) {
minDiff = diff
closestTime = time
}
}
if (minDiff < 5) {
// Within 5 seconds, use existing state
return stateMap.get(closestTime)!
}
// Create new state
const newState: ScreenState = {
timestamp: roundedTime,
route: '',
componentStack: [],
visibleElements: [],
dataPresent: {},
}
stateMap.set(roundedTime, newState)
return newState
}
private extractVideoId(videoUrl: string): string {
return LoomClient.extractVideoId(videoUrl) || ''
}
private generateCombinedInsight(
verbal: VerbalIssue,
visual: TimestampedElement | null
): string {
if (!visual) {
return `User verbally reported: "${verbal.text}" (${verbal.type})`
}
// Generate insights based on correlation
const insights: string[] = []
if (verbal.type === 'bug' && visual.type === 'error') {
insights.push(
`Error confirmed: User reported bug matches visual error at ${visual.timestamp}s`
)
} else if (verbal.type === 'confusion' && visual.type === 'navigation') {
insights.push(`Navigation issue: User confusion during navigation to ${visual.content}`)
} else if (verbal.type === 'frustration' && visual.type === 'interaction') {
insights.push(
`Frustrating interaction: User expressed frustration during ${visual.content}`
)
} else {
insights.push(`${verbal.type} reported while ${visual.type} occurred`)
}
// Add severity context
if (verbal.severity === 'critical' || verbal.severity === 'high') {
insights.push(`High priority issue based on user emphasis`)
}
return insights.join('. ')
}
private enhanceErrorsWithVerbalIssues(
errors: ErrorOccurrence[],
audioAnalysis: AudioAnalysisResult,
correlations?: VerbalVisualCorrelation[]
): void {
// Add verbal issues that weren't caught as visual errors
const verbalBugs = audioAnalysis.verbalIssues.filter((v) => v.type === 'bug')
for (const verbalBug of verbalBugs) {
// Check if this verbal bug already has a corresponding error
const hasVisualError = errors.some(
(e) => Math.abs(e.timestamp - verbalBug.timestamp) < 5
)
if (!hasVisualError) {
// Add as a new error based on verbal report
errors.push({
timestamp: verbalBug.timestamp,
errorMessage: `User reported: ${verbalBug.text}`,
errorType: 'ui', // Default to UI since it's verbally reported
possibleCauses: [
'Issue may not have visual indicators',
'Error might be in console/network tab',
...verbalBug.keywords.map((k) => `Related to: ${k}`),
],
relatedCode: [],
})
}
}
// Enhance existing errors with verbal context
for (const error of errors) {
const verbalContext = audioAnalysis.verbalIssues.find(
(v) => Math.abs(v.timestamp - error.timestamp) < 5
)
if (verbalContext) {
// Add verbal description to possible causes
error.possibleCauses.unshift(`User description: "${verbalContext.text}"`)
// Upgrade severity if verbal issue is high severity
if (verbalContext.severity === 'critical' || verbalContext.severity === 'high') {
error.errorType = 'ui' // Ensure it's marked as user-facing
}
}
}
}
private enhanceDebuggingSuggestionsWithAudio(
suggestions: DebuggingSuggestion[],
audioAnalysis: AudioAnalysisResult
): void {
// Add suggestions based on problem statements
for (const problem of audioAnalysis.problemStatements) {
if (problem.expectedBehavior && problem.actualBehavior) {
suggestions.push({
priority: 'high',
type: 'log',
location: {
file: 'relevant component',
line: 0,
type: 'component',
snippet: '',
},
reason: `User expects: "${problem.expectedBehavior}" but gets: "${problem.actualBehavior}"`,
additionalContext: problem.userImpact,
})
}
}
// Add suggestions based on technical terms mentioned
const frequentTerms = audioAnalysis.technicalTerms
.filter((t) => t.occurrences.length >= 2)
.sort((a, b) => b.occurrences.length - a.occurrences.length)
for (const term of frequentTerms.slice(0, 3)) {
suggestions.push({
priority: 'medium',
type: 'inspect',
location: {
file: `files containing "${term.term}"`,
line: 0,
type: term.category === 'component' ? 'component' : 'state_update',
snippet: term.occurrences[0].context,
},
reason: `Term "${term.term}" mentioned ${term.occurrences.length} times - likely area of issue`,
additionalContext: `Category: ${term.category}`,
})
}
// Prioritize suggestions based on emotional tone
const frustrationPoints = audioAnalysis.emotionalTone.filter(
(t) => t.tone === 'frustrated' && t.intensity > 0.6
)
if (frustrationPoints.length > 0) {
// Boost priority of suggestions near frustration points
for (const suggestion of suggestions) {
const nearFrustration = frustrationPoints.some((f) =>
suggestion.additionalContext?.includes(f.indicators.join(' '))
)
if (nearFrustration && suggestion.priority === 'medium') {
suggestion.priority = 'high'
}
}
}
}
}