MCP Webcam Server

import { useRef, useState, useEffect, useCallback, useMemo } from "react"; import Webcam from "react-webcam"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardFooter, CardHeader, CardTitle, } from "@/components/ui/card"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from "@/components/ui/select"; import { Input } from "@/components/ui/input"; import { Checkbox } from "@/components/ui/checkbox"; import { captureScreen } from "@/utils/screenCapture"; import { Github, Info, Link2, Users, Copy, Check } from "lucide-react"; import { Badge } from "@/components/ui/badge"; interface Session { id: string; connectedAt: string; lastActivity: string; isStale: boolean; capabilities: { sampling: boolean; tools: boolean; resources: boolean; }; clientInfo?: { name: string; version: string; }; } export function WebcamCapture() { const [webcamInstance, setWebcamInstance] = useState<Webcam | null>(null); const clientIdRef = useRef<string | null>(null); const [_, setClientId] = useState<string | null>(null); // State for configuration const [config, setConfig] = useState<{ mcpHostConfigured: boolean; mcpHost: string } | null>(null); // State for copy functionality const [copied, setCopied] = useState(false); // Copy to clipboard function const copyToClipboard = async (text: string) => { try { await navigator.clipboard.writeText(text); setCopied(true); setTimeout(() => setCopied(false), 2000); } catch (err) { console.error('Failed to copy:', err); } }; // Generate random 5-character user ID if none provided and in multiuser mode const generateUserId = () => { return Math.random().toString(36).substring(2, 7).toLowerCase(); }; // Validate and sanitize user ID const validateUserId = (userId: string): string => { if (!userId) return 'default'; // Remove any non-alphanumeric characters and hyphens/underscores const sanitized = userId.replace(/[^a-zA-Z0-9_-]/g, ''); // Limit to 30 characters max const truncated = sanitized.substring(0, 30); // If empty after sanitization, return default return truncated || 'default'; }; // Extract user parameter from URL const urlUserParam = new URLSearchParams(window.location.search).get('user'); const userParam = useMemo(() => { if (urlUserParam) { return validateUserId(urlUserParam); } // Only generate random ID in multiuser mode (when MCP_HOST is configured) if (config?.mcpHostConfigured) { // Store in sessionStorage to persist across refreshes const storageKey = 'mcp-webcam-user-id'; let storedUserId = sessionStorage.getItem(storageKey); if (!storedUserId) { storedUserId = generateUserId(); sessionStorage.setItem(storageKey, storedUserId); } return validateUserId(storedUserId); } return 'default'; }, [urlUserParam, config?.mcpHostConfigured]); // Determine if we should show the banner (when MCP_HOST is explicitly set) const showBanner = config?.mcpHostConfigured || false; // Update URL when user param changes (for autogenerated IDs) useEffect(() => { // Only update URL if we don't already have a user param in URL and we have a non-default userParam if (!urlUserParam && userParam !== 'default' && config?.mcpHostConfigured) { const url = new URL(window.location.href); url.searchParams.set('user', userParam); // Use replaceState to avoid adding to browser history window.history.replaceState({}, '', url.toString()); } }, [userParam, urlUserParam, config?.mcpHostConfigured]); const [devices, setDevices] = useState<MediaDeviceInfo[]>([]); const [selectedDevice, setSelectedDevice] = useState<string>("default"); const [frozenFrame, setFrozenFrame] = useState<string | null>(null); // New state for sampling results const [samplingResult, setSamplingResult] = useState<string | null>(null); const [samplingError, setSamplingError] = useState<string | null>(null); const [isSampling, setIsSampling] = useState(false); // State for sampling prompt and auto-update const [samplingPrompt, setSamplingPrompt] = useState<string>("What can you see?"); const [autoUpdate, setAutoUpdate] = useState<boolean>(false); // Explicitly false const [updateInterval, setUpdateInterval] = useState<number>(30); const autoUpdateIntervalRef = useRef<NodeJS.Timeout | null>(null); // State for session management const [sessions, setSessions] = useState<Session[]>([]); const [selectedSessionId, setSelectedSessionId] = useState<string | null>( null ); const sessionPollIntervalRef = useRef<NodeJS.Timeout | null>(null); // Get the currently selected session const selectedSession = sessions.find((s) => s.id === selectedSessionId); const getImage = useCallback(() => { console.log("getImage called, frozenFrame state:", frozenFrame); if (frozenFrame) { console.log("Using frozen frame"); return frozenFrame; } console.log("Getting live screenshot"); const screenshot = webcamInstance?.getScreenshot(); return screenshot || null; }, [frozenFrame, webcamInstance]); const toggleFreeze = () => { console.log("toggleFreeze called, current frozenFrame:", frozenFrame); if (frozenFrame) { console.log("Unfreezing frame"); setFrozenFrame(null); } else if (webcamInstance) { console.log("Freezing new frame"); const screenshot = webcamInstance.getScreenshot(); if (screenshot) { console.log("New frame captured successfully"); setFrozenFrame(screenshot); } } }; const handleScreenCapture = async () => { console.log("Screen capture button clicked"); try { const screenImage = await captureScreen(); console.log("Got screen image, length:", screenImage.length); // Test if we can even get this far alert("Screen captured! Check console for details."); if (!clientIdRef.current) { console.error("No client ID available"); return; } const response = await fetch(`/api/capture-result?user=${encodeURIComponent(userParam)}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ clientId: clientIdRef.current, image: screenImage, type: "screen", }), }); console.log("Server response:", response.status); } catch (error) { console.error("Screen capture error:", error); alert("Screen capture failed: " + (error as Error).message); } }; // New function to handle sampling with callback for auto-update const handleSample = async (onComplete?: () => void) => { console.log("Sample button clicked"); setSamplingError(null); setSamplingResult(null); setIsSampling(true); try { const imageSrc = getImage(); if (!imageSrc) { throw new Error("Failed to capture image for sampling"); } console.log("Sending image for sampling..."); // Add timeout to prevent hanging requests const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 30000); // 30 second timeout const response = await fetch(`/api/process-sample?user=${encodeURIComponent(userParam)}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ image: imageSrc, prompt: samplingPrompt, sessionId: selectedSessionId, }), signal: controller.signal, }).catch((error) => { clearTimeout(timeoutId); if (error.name === "AbortError") { throw new Error("Request timed out after 30 seconds"); } throw error; }); clearTimeout(timeoutId); if (!response.ok) { const errorData = await response.json(); throw new Error(errorData.error || "Failed to process sample"); } const data = await response.json(); console.log("Sampling response:", data); if (data.success && data.result && data.result.content?.type === "text") { setSamplingResult(data.result.content.text); // Call the completion callback on success if (onComplete) { onComplete(); } } else { throw new Error("Invalid sampling result format"); } } catch (error) { console.error("Sampling error:", error); setSamplingError((error as Error).message || "An unknown error occurred"); } finally { setIsSampling(false); } }; // Fetch configuration on mount useEffect(() => { const fetchConfig = async () => { try { const response = await fetch('/api/config'); const configData = await response.json(); setConfig(configData); } catch (error) { console.error('Error fetching config:', error); } }; fetchConfig(); }, []); useEffect(() => { const getDevices = async () => { try { const devices = await navigator.mediaDevices.enumerateDevices(); const videoDevices = devices.filter( (device) => device.kind === "videoinput" ); setDevices(videoDevices); setSelectedDevice("default"); } catch (error) { console.error("Error getting devices:", error); } }; getDevices(); navigator.mediaDevices.addEventListener("devicechange", getDevices); return () => { navigator.mediaDevices.removeEventListener("devicechange", getDevices); }; }, []); useEffect(() => { console.error("Setting up EventSource..."); const eventSource = new EventSource(`/api/events?user=${encodeURIComponent(userParam)}`); eventSource.onopen = () => { console.error("SSE connection opened successfully"); }; eventSource.onerror = (error) => { console.error("SSE connection error:", error); }; eventSource.onmessage = async (event) => { console.log("Received message:", event.data); try { const data = JSON.parse(event.data); switch (data.type) { case "connected": console.log("Connected with client ID:", data.clientId); clientIdRef.current = data.clientId; // Store in ref setClientId(data.clientId); // Keep state in sync if needed for UI break; case "capture": console.log(`Capture triggered - webcam status:`, !!webcamInstance); if (!webcamInstance || !clientIdRef.current) { const error = !webcamInstance ? "Webcam not initialized" : "Client ID not set"; await fetch(`/api/capture-error?user=${encodeURIComponent(userParam)}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ clientId: clientIdRef.current, error: { message: error }, }), }); return; } console.log("Taking webcam image..."); const imageSrc = getImage(); if (!imageSrc) { await fetch(`/api/capture-error?user=${encodeURIComponent(userParam)}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ clientId: clientIdRef.current, error: { message: "Failed to capture image" }, }), }); return; } await fetch(`/api/capture-result?user=${encodeURIComponent(userParam)}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ clientId: clientIdRef.current, image: imageSrc, }), }); console.log("Image sent to server"); break; case "screenshot": console.log("Screen capture triggered"); if (!clientIdRef.current) { console.error("Cannot capture - client ID not set"); return; } try { const screenImage = await captureScreen(); await fetch(`/api/capture-result?user=${encodeURIComponent(userParam)}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ clientId: clientIdRef.current, image: screenImage, type: "screen", }), }); console.log("Screen capture sent to server"); } catch (error) { console.error("Screen capture failed:", error); await fetch(`/api/capture-error?user=${encodeURIComponent(userParam)}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ clientId: clientIdRef.current, error: { message: (error as Error).message || "Screen capture failed", }, }), }); } break; case "sample": // Handle sample event if needed (currently handled directly by handle Sample function) break; default: console.warn("Unknown message type:", data.type); } } catch (error) { console.error( "Error processing message:", error, "Raw message:", event.data ); } }; return () => { console.error("Cleaning up EventSource connection"); eventSource.close(); }; }, [webcamInstance, getImage, userParam]); // Add userParam to dependencies // Handle auto-update with recursive timeout after successful requests useEffect(() => { console.log("Auto-update effect running:", { autoUpdate, updateInterval, hasSampling: selectedSession?.capabilities.sampling, sessionId: selectedSession?.id }); // Clear any existing timer first if (autoUpdateIntervalRef.current) { clearTimeout(autoUpdateIntervalRef.current); autoUpdateIntervalRef.current = null; } // Recursive function to handle auto-update const scheduleNextUpdate = () => { // Ensure minimum 5 seconds between requests const delayMs = Math.max(updateInterval * 1000, 5000); autoUpdateIntervalRef.current = setTimeout(() => { if (autoUpdate === true && selectedSession?.capabilities.sampling) { console.log("Auto-update triggered after", delayMs, "ms"); handleSample(() => { // On successful completion, schedule the next update if (autoUpdate === true) { scheduleNextUpdate(); } }); } }, delayMs); }; // Only start auto-update if explicitly enabled by user if (autoUpdate === true && updateInterval > 0 && selectedSession?.capabilities.sampling) { console.log("Starting auto-update"); // Initial sample when auto-update is enabled handleSample(() => { // Schedule next update after successful initial sample if (autoUpdate === true) { scheduleNextUpdate(); } }); } // Cleanup function return () => { if (autoUpdateIntervalRef.current) { console.log("Cleaning up auto-update timer"); clearTimeout(autoUpdateIntervalRef.current); autoUpdateIntervalRef.current = null; } }; }, [autoUpdate, updateInterval, selectedSession?.id]); // Only depend on session ID, not the whole object // State for all sessions count const [totalSessions, setTotalSessions] = useState<number>(0); // Poll for active sessions useEffect(() => { const fetchSessions = async () => { try { // Fetch sessions for current user const response = await fetch(`/api/sessions?user=${encodeURIComponent(userParam)}`); if (response.ok) { const data = await response.json(); setSessions(data.sessions); // Auto-select the most recent session if none selected if (!selectedSessionId && data.sessions.length > 0) { // Sort by connection time and select the most recent const sortedSessions = [...data.sessions].sort( (a, b) => new Date(b.connectedAt).getTime() - new Date(a.connectedAt).getTime() ); setSelectedSessionId(sortedSessions[0].id); } // Clean up selected session if it's no longer available if ( selectedSessionId && !data.sessions.find((s: Session) => s.id === selectedSessionId) ) { setSelectedSessionId(null); } } // Fetch total sessions count (only if showing banner) if (showBanner) { const totalResponse = await fetch(`/api/sessions?all=true`); if (totalResponse.ok) { const totalData = await totalResponse.json(); setTotalSessions(totalData.sessions.length); } } } catch (error) { console.error("Error fetching sessions:", error); } }; // Initial fetch fetchSessions(); // Poll every 2 seconds sessionPollIntervalRef.current = setInterval(fetchSessions, 2000); return () => { if (sessionPollIntervalRef.current) { clearInterval(sessionPollIntervalRef.current); } }; }, [selectedSessionId, userParam, showBanner]); return ( <div> {showBanner && ( <> {/* Fixed position connection badge in top right corner */} <div className="fixed top-2 right-2 sm:top-4 sm:right-4 z-50 flex items-center gap-1 bg-white dark:bg-slate-800 rounded-md border px-2 py-1 shadow-lg"> <Users className="h-3 w-3 text-green-600" /> <span className="text-xs font-medium text-slate-700 dark:text-slate-300"> {sessions.length}/{totalSessions} </span> </div> {/* Main banner content */} <div className="border-b bg-slate-50 dark:bg-slate-900/50"> <div className="w-full px-3 sm:px-6 py-3"> <div className="flex flex-col sm:flex-row sm:items-center gap-3 sm:gap-6"> <div className="flex items-center gap-2"> <Info className="h-4 w-4 text-blue-600" /> <span className="text-sm font-medium">Connected as</span> <Badge variant="default" className="bg-blue-600 hover:bg-blue-700"> {userParam} </Badge> </div> {/* MCP URL - stacks on mobile */} <div className="flex flex-col sm:flex-row sm:items-center gap-2 sm:gap-2 min-w-0 sm:flex-1"> <div className="flex items-center gap-2"> <Link2 className="h-4 w-4 text-blue-600 flex-shrink-0" /> <span className="text-sm font-medium flex-shrink-0">MCP URL:</span> </div> <div className="flex items-center gap-2 min-w-0 flex-1"> <code className="text-xs font-mono bg-slate-100 dark:bg-slate-900 px-2 py-1 rounded border select-all truncate flex-1 text-slate-700 dark:text-slate-300"> {config?.mcpHost || window.location.origin}/mcp{config?.mcpHostConfigured && userParam !== 'default' ? `?user=${userParam}` : ''} </code> <Button variant="outline" size="sm" onClick={() => copyToClipboard(`${config?.mcpHost || window.location.origin}/mcp${config?.mcpHostConfigured && userParam !== 'default' ? `?user=${userParam}` : ''}`)} className="h-7 px-2 flex-shrink-0" > {copied ? ( <Check className="h-3 w-3" /> ) : ( <Copy className="h-3 w-3" /> )} </Button> </div> </div> </div> {/* Helper text */} <div className="mt-2 text-xs text-muted-foreground"> Add <code className="bg-muted px-1 py-0.5 rounded font-mono text-xs">?user=YOUR_ID</code> to change user </div> </div> </div> </> )} <Card className={`w-full max-w-2xl mx-auto ${showBanner ? 'mt-4' : ''}`}> <CardHeader> <div className="relative"> <a href="https://github.com/evalstate/mcp-webcam" target="_blank" rel="noopener noreferrer" className="absolute left-0 top-0 flex items-center gap-2 text-xs sm:text-sm text-muted-foreground hover:text-foreground transition-colors" > <Github className="h-3 w-3 sm:h-4 sm:w-4" /> <span className="hidden sm:inline">github.com/evalstate</span> </a> <CardTitle className="text-lg sm:text-xl font-bold text-center pt-6 sm:pt-0"> mcp-webcam </CardTitle> </div> <div className="w-full max-w-2xl mx-auto mt-4 space-y-2"> <div className="grid grid-cols-1 sm:grid-cols-2 gap-4"> {/* Camera selector */} <div className="space-y-2"> <label className="text-sm font-medium">Camera</label> <Select value={selectedDevice} onValueChange={setSelectedDevice} > <SelectTrigger className="w-full"> <SelectValue placeholder="Select camera" /> </SelectTrigger> <SelectContent> <SelectItem value="default">Default camera</SelectItem> {devices.map((device) => { const deviceId = device.deviceId || `device-${devices.indexOf(device)}`; return ( <SelectItem key={deviceId} value={deviceId}> {device.label || `Camera ${devices.indexOf(device) + 1}`} </SelectItem> ); })} </SelectContent> </Select> </div> {/* Session selector - always visible */} <div className="space-y-2"> <label className="text-sm font-medium"> {userParam === 'default' ? 'MCP Session' : `MCP Session (${userParam})`} </label> <Select value={selectedSessionId || ""} onValueChange={setSelectedSessionId} disabled={sessions.length === 0} > <SelectTrigger className="w-full"> <SelectValue placeholder={ sessions.length === 0 ? "No connections" : "Select MCP session" } /> </SelectTrigger> <SelectContent> {sessions.length === 0 ? ( <div className="p-2 text-center text-muted-foreground text-sm"> No MCP connections available </div> ) : ( sessions.map((session) => { const connectedTime = new Date(session.connectedAt); const timeString = connectedTime.toLocaleTimeString(); // Determine color based on status let colorClass = "bg-red-500"; // Default: stale if (!session.isStale) { if (session.capabilities.sampling) { colorClass = "bg-green-500"; // Active with sampling } else { colorClass = "bg-amber-500"; // Active without sampling } } return ( <SelectItem key={session.id} value={session.id}> <div className="flex items-center gap-2"> <div className={`w-2 h-2 rounded-full ${colorClass}`} /> <span> {session.clientInfo ? `${session.clientInfo.name} v${session.clientInfo.version}` : `Session ${session.id.slice(0, 8)}`} </span> <span className="text-xs text-muted-foreground"> ({timeString}) </span> </div> </SelectItem> ); }) )} </SelectContent> </Select> </div> </div> {sessions.length > 0 && ( <div className="text-xs text-muted-foreground text-center"> <span className="inline-flex items-center gap-1"> <div className="w-2 h-2 rounded-full bg-green-500" /> Active with sampling </span> <span className="inline-flex items-center gap-1 ml-3"> <div className="w-2 h-2 rounded-full bg-amber-500" /> Active, no sampling </span> <span className="inline-flex items-center gap-1 ml-3"> <div className="w-2 h-2 rounded-full bg-red-500" /> Stale connection </span> </div> )} </div> </CardHeader> <CardContent className="px-3 sm:px-6 pt-3 pb-6"> <div className="rounded-lg overflow-hidden border border-border relative"> <Webcam ref={(webcam) => setWebcamInstance(webcam)} screenshotFormat="image/jpeg" className="w-full" videoConstraints={{ width: 1280, height: 720, ...(selectedDevice !== "default" ? { deviceId: selectedDevice } : { facingMode: "user" }), }} /> {frozenFrame && ( <img src={frozenFrame} alt="Frozen frame" className="absolute inset-0 w-full h-full object-cover" /> )} <div className="absolute top-4 right-4"> <Button onClick={toggleFreeze} variant={frozenFrame ? "destructive" : "outline"} size="sm" > {frozenFrame ? "Unfreeze" : "Freeze"} </Button> </div> </div> </CardContent> <CardFooter className="flex flex-col gap-4 pb-6"> <div className="w-full space-y-4"> {selectedSession && !selectedSession.capabilities.sampling && ( <Alert className="mb-4"> <AlertDescription> The selected MCP session does not support sampling. Please connect a client with sampling capabilities. </AlertDescription> </Alert> )} <div className="flex flex-col sm:flex-row gap-2"> <Input type="text" value={samplingPrompt} onChange={(e) => setSamplingPrompt(e.target.value)} placeholder="Enter your question..." className="flex-1" /> <Button onClick={() => handleSample()} variant="default" disabled={ isSampling || autoUpdate || !selectedSession?.capabilities.sampling } title={ !selectedSession?.capabilities.sampling ? "Selected session does not support sampling" : "" } className="w-full sm:w-auto" > {isSampling ? "Sampling..." : "Sample"} </Button> </div> {/* Sampling results display - always visible */} <div className="mt-4 min-h-[80px]"> {samplingResult && ( <Alert> <AlertTitle>Analysis Result</AlertTitle> <AlertDescription>{samplingResult}</AlertDescription> </Alert> )} {samplingError && ( <Alert variant="destructive"> <AlertTitle>Sampling Error</AlertTitle> <AlertDescription>{samplingError}</AlertDescription> </Alert> )} {!samplingResult && !samplingError && !isSampling && ( <div className="text-center text-muted-foreground text-sm p-4 border rounded-lg"> Sampling results will appear here </div> )} {isSampling && ( <div className="text-center text-muted-foreground text-sm p-4 border rounded-lg"> Processing image... </div> )} </div> {/* Auto-update and Screen Capture controls */} <div className="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-4 mt-4"> <div className="flex flex-col sm:flex-row sm:items-center gap-3 sm:gap-4"> <div className="flex items-center space-x-2"> <Checkbox id="auto-update" checked={autoUpdate} onCheckedChange={(checked) => setAutoUpdate(checked as boolean) } disabled={!selectedSession?.capabilities.sampling} /> <label htmlFor="auto-update" className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70" > Auto-update </label> </div> <div className="flex items-center gap-2"> <Input type="number" value={updateInterval} onChange={(e) => setUpdateInterval(parseInt(e.target.value) || 30) } className="w-20" min="1" disabled={ !autoUpdate || !selectedSession?.capabilities.sampling } /> <span className="text-sm text-muted-foreground">seconds</span> </div> </div> <Button onClick={handleScreenCapture} variant="secondary" className="w-full sm:w-auto"> Test Screen Capture </Button> </div> </div> </CardFooter> </Card> </div> ); }