visualAnalyzer.ts•11.1 kB
import type { LoomClient } from './loomClient'
export interface VisualElements {
uiText: string[]
errorMessages: string[]
buttonLabels: string[]
formFields: string[]
urlsAndRoutes: string[]
codeSnippets: string[]
timestamps: TimestampedElement[]
}
export interface TimestampedElement {
timestamp: number
type: 'ui_text' | 'error' | 'navigation' | 'code' | 'interaction'
content: string
confidence: number
}
export interface FrameAnalysis {
timestamp: number
elements: {
text: string[]
errors: string[]
urls: string[]
interactions: string[]
}
}
export class VisualAnalyzer {
constructor(private loomClient: LoomClient) {}
async extractVisualElements(videoId: string, transcript?: any): Promise<VisualElements> {
const elements: VisualElements = {
uiText: [],
errorMessages: [],
buttonLabels: [],
formFields: [],
urlsAndRoutes: [],
codeSnippets: [],
timestamps: [],
}
// Extract from transcript with timestamps
if (transcript?.segments) {
for (const segment of transcript.segments) {
await this.analyzeTranscriptSegment(segment, elements)
}
}
// Analyze video metadata for additional context
await this.analyzeVideoMetadata(videoId, elements)
// Deduplicate while preserving order
elements.uiText = this.deduplicatePreserveOrder(elements.uiText)
elements.errorMessages = this.deduplicatePreserveOrder(elements.errorMessages)
elements.buttonLabels = this.deduplicatePreserveOrder(elements.buttonLabels)
elements.formFields = this.deduplicatePreserveOrder(elements.formFields)
elements.urlsAndRoutes = this.deduplicatePreserveOrder(elements.urlsAndRoutes)
elements.codeSnippets = this.deduplicatePreserveOrder(elements.codeSnippets)
return elements
}
private async analyzeTranscriptSegment(segment: any, elements: VisualElements) {
const text = segment.text
const timestamp = segment.start_time || 0
// Detect UI text patterns
const uiPatterns = [
/clicking (?:on |the )?["']?([^"']+)["']?/gi,
/button (?:labeled |called |named )?["']?([^"']+)["']?/gi,
/(?:see|shows?|displays?) (?:the )?["']?([^"']+)["']?/gi,
/(?:page|screen|view) (?:shows?|displays?|contains?) ["']?([^"']+)["']?/gi,
]
for (const pattern of uiPatterns) {
const matches = [...text.matchAll(pattern)]
for (const match of matches) {
if (match[1] && match[1].length > 2 && match[1].length < 50) {
elements.uiText.push(match[1])
elements.timestamps.push({
timestamp,
type: 'ui_text',
content: match[1],
confidence: 0.8,
})
}
}
}
// Detect error messages
const errorPatterns = [
/error:?\s*["']?([^"'\n]+)["']?/gi,
/(?:warning|alert|exception):?\s*["']?([^"'\n]+)["']?/gi,
/(?:failed|failure|cannot|unable to)\s+([^.!?\n]+)/gi,
/(?:invalid|incorrect|missing)\s+([^.!?\n]+)/gi,
]
for (const pattern of errorPatterns) {
const matches = [...text.matchAll(pattern)]
for (const match of matches) {
if (match[1] && match[1].length > 5) {
elements.errorMessages.push(match[1].trim())
elements.timestamps.push({
timestamp,
type: 'error',
content: match[1].trim(),
confidence: 0.9,
})
}
}
}
// Detect button labels
const buttonPatterns = [
/(?:click|press|tap) (?:the |on )?["']?([^"']+)["']? button/gi,
/button (?:labeled|called|named) ["']?([^"']+)["']?/gi,
/["']([^"']+)["'] button/gi,
]
for (const pattern of buttonPatterns) {
const matches = [...text.matchAll(pattern)]
for (const match of matches) {
if (match[1] && match[1].length < 30) {
elements.buttonLabels.push(match[1])
}
}
}
// Detect form fields
const formPatterns = [
/(?:fill|enter|type) (?:in |into )?(?:the )?["']?([^"']+)["']? (?:field|input|box)/gi,
/(?:field|input|textbox) (?:labeled|called|named) ["']?([^"']+)["']?/gi,
/["']([^"']+)["'] (?:field|input|textbox)/gi,
]
for (const pattern of formPatterns) {
const matches = [...text.matchAll(pattern)]
for (const match of matches) {
if (match[1] && match[1].length < 30) {
elements.formFields.push(match[1])
}
}
}
// Detect URLs and routes
const urlPatterns = [
/(?:navigate|go|redirect) (?:to )?["']?([/\w-]+)["']?/gi,
/(?:url|route|path):?\s*["']?([/\w-]+)["']?/gi,
/(?:at|on) ["']?([/\w-]+)["']? (?:page|route)/gi,
/https?:\/\/[^\s"']+/gi,
/\/[\w-]+(?:\/[\w-]+)*/g,
]
for (const pattern of urlPatterns) {
const matches = [...text.matchAll(pattern)]
for (const match of matches) {
const url = match[1] || match[0]
if (url && url.length > 1) {
elements.urlsAndRoutes.push(url)
if (url.startsWith('/') || url.includes('://')) {
elements.timestamps.push({
timestamp,
type: 'navigation',
content: url,
confidence: 0.9,
})
}
}
}
}
// Detect code snippets
const codePatterns = [
/`([^`]+)`/g,
/(?:function|class|const|let|var)\s+(\w+)/gi,
/(\w+)\s*\(\s*\)/g,
/(?:import|export)\s+(?:\{[^}]+\}|\w+)\s+from\s+["']([^"']+)["']/gi,
]
for (const pattern of codePatterns) {
const matches = [...text.matchAll(pattern)]
for (const match of matches) {
const code = match[1] || match[0]
if (code && code.length > 2 && code.length < 100) {
elements.codeSnippets.push(code)
elements.timestamps.push({
timestamp,
type: 'code',
content: code,
confidence: 0.7,
})
}
}
}
}
private async analyzeVideoMetadata(videoId: string, elements: VisualElements) {
try {
// Get video metadata which might contain additional context
const metadata = await this.loomClient.getVideoMetadata(videoId)
if (metadata.name) {
// Extract keywords from video title
const titleWords = metadata.name
.split(/\s+/)
.filter((w) => w.length > 3 && /^[A-Z]/.test(w))
elements.uiText.push(...titleWords)
}
if (metadata.description) {
// Look for URLs in description
const urls = metadata.description.match(/https?:\/\/[^\s]+/g) || []
elements.urlsAndRoutes.push(...urls)
}
} catch (error) {
// Metadata fetch failed, continue without it
}
}
async analyzeKeyFrames(videoId: string, intervals: number = 5): Promise<FrameAnalysis[]> {
// This would analyze key frames from the video
// For now, we'll return a placeholder implementation
// In a real implementation, this would:
// 1. Extract frames at regular intervals
// 2. Use OCR to extract text
// 3. Detect UI elements and patterns
// 4. Identify error states
const frames: FrameAnalysis[] = []
// Placeholder: Create mock frame analysis
// In production, this would use actual frame extraction
for (let i = 0; i < intervals; i++) {
frames.push({
timestamp: i * 30, // Every 30 seconds
elements: {
text: [],
errors: [],
urls: [],
interactions: [],
},
})
}
return frames
}
extractComponentNames(elements: VisualElements): string[] {
const components: Set<string> = new Set()
// Extract from UI text
for (const text of elements.uiText) {
// Look for React-style component names
const componentPattern =
/\b([A-Z][a-zA-Z]+(?:Component|Page|Modal|Form|Button|List|View|Panel|Card|Header|Footer|Nav|Menu|Dialog|Alert|Table|Grid))\b/g
const matches = text.match(componentPattern) || []
matches.forEach((m) => components.add(m))
}
// Extract from button labels (they often match component names)
for (const label of elements.buttonLabels) {
const words = label.split(/\s+/)
for (const word of words) {
if (/^[A-Z]/.test(word) && word.length > 3) {
components.add(word)
}
}
}
return Array.from(components)
}
inferUserFlow(timestamps: TimestampedElement[]): string[] {
const flow: string[] = []
// Sort by timestamp
const sorted = [...timestamps].sort((a, b) => a.timestamp - b.timestamp)
let lastNavigation: string | null = null
for (const element of sorted) {
if (element.type === 'navigation') {
if (element.content !== lastNavigation) {
flow.push(`Navigate to ${element.content}`)
lastNavigation = element.content
}
} else if (element.type === 'interaction') {
flow.push(`User action: ${element.content}`)
} else if (element.type === 'error' && element.confidence > 0.8) {
flow.push(`Error encountered: ${element.content}`)
}
}
return flow
}
private deduplicatePreserveOrder(items: string[]): string[] {
const seen = new Set<string>()
const result: string[] = []
for (const item of items) {
const normalized = item.trim().toLowerCase()
if (!seen.has(normalized) && item.trim().length > 0) {
seen.add(normalized)
result.push(item.trim())
}
}
return result
}
}