Peekaboo MCP

import Algorithms
import AppKit
import AXorcist
import Commander
import CoreGraphics
import Foundation
import PeekabooCore
import PeekabooFoundation
import ScreenCaptureKit

@MainActor
extension SeeCommand {
    func detectElements(
        imageData: Data,
        windowContext: WindowContext?
    ) async throws -> ElementDetectionResult {
        self.logger.operationStart("element_detection")
        defer { self.logger.operationComplete("element_detection") }

        do {
            return try await Self.withWallClockTimeout(seconds: 20.0) {
                try await AutomationServiceBridge.detectElements(
                    automation: self.services.automation,
                    imageData: imageData,
                    snapshotId: nil,
                    windowContext: windowContext
                )
            }
        } catch is TimeoutError {
            throw CaptureError.detectionTimedOut(20.0)
        }
    }

    func resolveCaptureContext() async throws -> CaptureContext {
        if self.menubar {
            if let popover = try await self.captureMenuBarPopover() {
                return CaptureContext(
                    captureResult: popover.captureResult,
                    captureBounds: popover.windowBounds,
                    prefersOCR: true,
                    ocrMethod: "OCR",
                    windowIdOverride: popover.windowId
                )
            }

            if let appHint = self.menuBarAppHint() {
                self.logger.verbose("Attempting to open menu extra for capture", category: "Capture", metadata: [
                    "app": appHint
                ])
                let clickResult = try? await MenuServiceBridge.clickMenuBarItem(
                    named: appHint,
                    menu: self.services.menu
                )
                try? await Task.sleep(nanoseconds: 350_000_000)
                if let preferredX = clickResult?.location?.x,
                   let quickAreaCapture = try await self.captureMenuBarPopoverByArea(
                       preferredX: preferredX,
                       hints: MenuBarPopoverResolverContext.normalizedHints([appHint])
                   ) {
                    return CaptureContext(
                        captureResult: quickAreaCapture.captureResult,
                        captureBounds: quickAreaCapture.windowBounds,
                        prefersOCR: true,
                        ocrMethod: "OCR",
                        windowIdOverride: quickAreaCapture.windowId
                    )
                }
                if let popover = try await self.captureMenuBarPopover(allowAreaFallback: true) {
                    return CaptureContext(
                        captureResult: popover.captureResult,
                        captureBounds: popover.windowBounds,
                        prefersOCR: true,
                        ocrMethod: "OCR",
                        windowIdOverride: popover.windowId
                    )
                }
            }

            self.logger.verbose("No menu bar popover detected; capturing menu bar area", category: "Capture")
            let rect = try self.menuBarRect()
            let result = try await ScreenCaptureBridge.captureArea(services: self.services, rect: rect)
            return CaptureContext(
                captureResult: result,
                captureBounds: rect,
                prefersOCR: true,
                ocrMethod: "OCR",
                windowIdOverride: nil
            )
        }

        if let appName = self.app?.lowercased() {
            switch appName {
            case "menubar":
                self.logger.verbose("Capturing menu bar area", category: "Capture")
                let rect = try self.menuBarRect()
                let result = try await ScreenCaptureBridge.captureArea(services: self.services, rect: rect)
                return CaptureContext(
                    captureResult: result,
                    captureBounds: rect,
                    prefersOCR: false,
                    ocrMethod: nil,
                    windowIdOverride: nil
                )
            case "frontmost":
                self.logger.verbose("Capturing frontmost window (via --app frontmost)", category: "Capture")
                let result = try await ScreenCaptureBridge.captureFrontmost(services: self.services)
                return CaptureContext(
                    captureResult: result,
                    captureBounds: nil,
                    prefersOCR: false,
                    ocrMethod: nil,
                    windowIdOverride: nil
                )
            default:
                let result = try await self.performStandardCapture()
                return CaptureContext(
                    captureResult: result,
                    captureBounds: nil,
                    prefersOCR: false,
                    ocrMethod: nil,
                    windowIdOverride: nil
                )
            }
        }

        let result = try await self.performStandardCapture()
        return CaptureContext(
            captureResult: result,
            captureBounds: nil,
            prefersOCR: false,
            ocrMethod: nil,
            windowIdOverride: nil
        )
    }

    private func performStandardCapture() async throws -> CaptureResult {
        let effectiveMode = self.determineMode()
        self.logger.verbose(
            "Determined capture mode",
            category: "Capture",
            metadata: ["mode": effectiveMode.rawValue]
        )

        self.logger.operationStart("capture_phase", metadata: ["mode": effectiveMode.rawValue])
        switch effectiveMode {
        case .screen:
            // Handle screen capture with multi-screen support
            let result = try await self.performScreenCapture()
            self.logger.operationComplete("capture_phase", metadata: ["mode": effectiveMode.rawValue])
            return result

        case .multi:
            // Commander currently treats multi captures as multi-display screen grabs
            let result = try await self.performScreenCapture()
            self.logger.operationComplete("capture_phase", metadata: ["mode": effectiveMode.rawValue])
            return result

        case .window:
            if let windowId = self.windowId {
                self.logger.verbose("Initiating window capture (by id)", category: "Capture", metadata: [
                    "windowId": windowId,
                ])

                self.logger.startTimer("window_capture")
                let result = try await ScreenCaptureBridge.captureWindowById(
                    services: self.services,
                    windowId: windowId
                )
                self.logger.stopTimer("window_capture")
                self.logger.operationComplete("capture_phase", metadata: ["mode": effectiveMode.rawValue])
                return result
            } else if self.app != nil || self.pid != nil {
                let appIdentifier = try self.resolveApplicationIdentifier()
                self.logger.verbose("Initiating window capture", category: "Capture", metadata: [
                    "app": appIdentifier,
                    "windowTitle": self.windowTitle ?? "any",
                ])

                if let resolvedWindowId = try await self.resolveWindowId(
                    appIdentifier: appIdentifier,
                    titleFragment: self.windowTitle
                ) {
                    self.logger.verbose("Resolved window id for capture", category: "Capture", metadata: [
                        "windowId": resolvedWindowId
                    ])

                    self.logger.startTimer("window_capture")
                    let result = try await ScreenCaptureBridge.captureWindowById(
                        services: self.services,
                        windowId: resolvedWindowId
                    )
                    self.logger.stopTimer("window_capture")
                    self.logger.operationComplete("capture_phase", metadata: ["mode": effectiveMode.rawValue])
                    return result
                }

                let windowIndex = try await self.resolveSeeWindowIndex(
                    appIdentifier: appIdentifier,
                    titleFragment: self.windowTitle
                )

                self.logger.startTimer("window_capture")
                let result = try await ScreenCaptureBridge.captureWindow(
                    services: self.services,
                    appIdentifier: appIdentifier,
                    windowIndex: windowIndex
                )
                self.logger.stopTimer("window_capture")
                self.logger.operationComplete("capture_phase", metadata: ["mode": effectiveMode.rawValue])
                return result
            } else {
                throw ValidationError("Provide --window-id, or --app/--pid for window mode")
            }

        case .frontmost:
            self.logger.verbose("Capturing frontmost window")
            let result = try await ScreenCaptureBridge.captureFrontmost(services: self.services)
            self.logger.operationComplete("capture_phase", metadata: ["mode": effectiveMode.rawValue])
            return result

        case .area:
            throw ValidationError("Area capture mode is not supported for 'see' yet. Use --mode screen or window")
        }
    }
}