Android MCP Server

Instructions

Implementation Reference

• src/accessibility/smart-executor.ts:40-158 (handler)

  The core implementation of the `smart_click` tool, which attempts multiple strategies (UIAutomator, Accessibility, Vision, Coordinates) to click an element.

  export async function smartClick(
    selector: ElementSelector,
    fallbackCoordinates?: { x: number; y: number },
    deviceId?: string
  ): Promise<SmartClickResult> {
    const resolved = await deviceManager.resolveDeviceId(deviceId);
    deviceManager.checkRateLimit(resolved);
  
    const strategyChain: SmartClickResult['strategyChain'] = [];
  
    // Strategy 1: UIAutomator
    try {
      log.debug('Attempting UIAutomator click', { selector });
      const element = await clickElement(selector, resolved);
      strategyChain.push({ method: 'uiautomator', attempted: true, succeeded: true });
      return {
        success: true,
        method: 'uiautomator',
        element,
        coordinates: { x: element.bounds.centerX, y: element.bounds.centerY },
        requiresVerification: false,
        strategyChain,
      };
    } catch (uiError) {
      const reason = uiError instanceof Error ? uiError.message : String(uiError);
      strategyChain.push({ method: 'uiautomator', attempted: true, succeeded: false, failureReason: reason });
      log.debug('UIAutomator failed', { error: reason });
    }
  
    // Strategy 2: Accessibility fallback
    try {
      log.debug('Attempting Accessibility-based search', { selector });
      const accessibilityData = await getAccessibilityTree(resolved);
  
      const searchText = selector.text || selector.contentDesc || selector.textContains || '';
      if (searchText && accessibilityData.includes(searchText)) {
        await new Promise(resolve => setTimeout(resolve, 500));
        try {
          const element = await clickElement(selector, resolved);
          strategyChain.push({ method: 'accessibility', attempted: true, succeeded: true });
          return {
            success: true,
            method: 'accessibility',
            element,
            coordinates: { x: element.bounds.centerX, y: element.bounds.centerY },
            requiresVerification: false,
            strategyChain,
          };
        } catch {
          strategyChain.push({ method: 'accessibility', attempted: true, succeeded: false, failureReason: 'Element found in accessibility but UIAutomator retry failed' });
        }
      } else {
        strategyChain.push({ method: 'accessibility', attempted: true, succeeded: false, failureReason: 'Element not found in accessibility data' });
      }
    } catch (accError) {
      const reason = accError instanceof Error ? accError.message : String(accError);
      strategyChain.push({ method: 'accessibility', attempted: true, succeeded: false, failureReason: reason });
      log.debug('Accessibility failed', { error: reason });
    }
  
    // Strategy 3: Vision
    try {
      log.debug('Attempting Vision-based detection');
      const analysis = await analyzeScreen(resolved);
  
      if (analysis.uiSummary) {
        const searchText = selector.text || selector.contentDesc || selector.textContains || '';
        if (searchText && analysis.uiSummary.toLowerCase().includes(searchText.toLowerCase())) {
          try {
            const element = await clickElement(selector, resolved);
            strategyChain.push({ method: 'vision', attempted: true, succeeded: true });
            return {
              success: true,
              method: 'vision',
              element,
              coordinates: { x: element.bounds.centerX, y: element.bounds.centerY },
              requiresVerification: false,
              strategyChain,
            };
          } catch {
            strategyChain.push({ method: 'vision', attempted: true, succeeded: false, failureReason: 'Element visible in analysis but click failed' });
          }
        } else {
          strategyChain.push({ method: 'vision', attempted: true, succeeded: false, failureReason: 'Element not found in visual analysis' });
        }
      } else {
        strategyChain.push({ method: 'vision', attempted: true, succeeded: false, failureReason: 'No UI summary available' });
      }
    } catch (visionError) {
      const reason = visionError instanceof Error ? visionError.message : String(visionError);
      strategyChain.push({ method: 'vision', attempted: true, succeeded: false, failureReason: reason });
      log.debug('Vision failed', { error: reason });
    }
  
    // Strategy 4: Raw coordinates fallback
    if (fallbackCoordinates) {
      log.info('Using fallback coordinates', { fallbackCoordinates });
      await tap(fallbackCoordinates.x, fallbackCoordinates.y, resolved);
      strategyChain.push({ method: 'coordinates', attempted: true, succeeded: true });
      return {
        success: true,
        method: 'coordinates',
        coordinates: fallbackCoordinates,
        fallbackReason: 'All structured methods failed; used fallback coordinates',
        requiresVerification: true, // Coordinate clicks must be verified
        strategyChain,
      };
    }
  
    strategyChain.push({ method: 'coordinates', attempted: false, succeeded: false, failureReason: 'No fallback coordinates provided' });
  
    return {
      success: false,
      method: 'coordinates',
      fallbackReason: 'All strategies failed and no fallback coordinates provided',
      requiresVerification: false,
      strategyChain,
    };
  }

• src/accessibility/smart-executor.ts:14-29 (schema)

  The type definition for the result returned by `smartClick`.

  export interface SmartClickResult {
    success: boolean;
    method: ExecutionMethod;
    element?: FoundElement;
    coordinates?: { x: number; y: number };
    fallbackReason?: string;
    /** Whether the result needs verification (true for coordinate-only clicks) */
    requiresVerification: boolean;
    /** Full strategy chain showing what was attempted and why each failed */
    strategyChain: Array<{
      method: ExecutionMethod;
      attempted: boolean;
      succeeded: boolean;
      failureReason?: string;
    }>;
  }

• src/controllers/automation-tools.ts:40-46 (registration)

  The automation controller where the `smart_click` tool is likely registered and invoked.

  return await metrics.measure('smart_click', device_id || 'default', async () => {
    const fallback = (fallback_x !== undefined && fallback_y !== undefined)
      ? { x: fallback_x, y: fallback_y }
      : undefined;
  
    const result = await smartClick(selector as ElementSelector, fallback, device_id);
    return {