execute-browser-commands

Automates browser interactions by executing predefined commands like clicking, typing, navigating, and waiting to perform web testing or automation tasks.

Instructions

Executes a sequence of predefined browser commands safely. Available commands:

click: Clicks on an element matching the selector or at specified coordinates
type: Types text into an input element
wait: Waits for an element, a specified time period, or a condition
navigate: Navigates to a specified URL
select: Selects an option in a dropdown
check: Checks or unchecks a checkbox
hover: Hovers over an element
focus: Focuses an element
blur: Removes focus from an element
keypress: Simulates pressing a keyboard key
scroll: Scrolls the page or an element
getAttribute: Gets an attribute value from an element
getProperty: Gets a property value from an element
drag: Performs a drag operation from one position to another
refresh: Refreshes the current page

Note on coordinates: For all mouse-related commands (click, drag, etc.), coordinates are relative to the browser viewport where (0,0) is the top-left corner. X increases to the right, Y increases downward.

Examples are available in the schema definition.

Input Schema

TableJSON Schema

Name	Required	Description
`commands`	Yes	Array of commands to execute in sequence
`timeout`	No	Overall timeout in milliseconds (default: 30000)
`contextId`	No	Browser ID to execute commands on (uses most recent browser if not provided)

Implementation Reference

src/index.ts:87-92 (registration)
Calls registerBrowserTools which registers the execute-browser-commands tool among others
```
registerBrowserTools(
  server,
  contextManager,
  lastHMREvents,
  screenshotHelpers
);
```

src/tools/browser-tools.ts:935-1334 (handler)

The main handler function that executes sequences of browser commands (click, type, navigate, etc.) using Playwright, with support for continueOnError and timeout.

async ({ commands, timeout = 30000, contextId }) => {
  try {
    // Check browser status
    const browserStatus = getContextForOperation(contextId);
    if (!browserStatus.isStarted) {
      return browserStatus.error;
    }

    // Get current checkpoint ID
    const checkpointId = await getCurrentCheckpointId(browserStatus.page);

    // Define command handler type
    type CommandArgs = Record<string, unknown>;
    type CommandHandler = (page: Page, selector: string | undefined, args: CommandArgs) => Promise<string | Record<string, unknown>>;

    // Command handler mapping
    const commandHandlers: Record<string, CommandHandler> = {
      click: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        if (!selector) throw new Error('Selector is required for click command');
        await page.waitForSelector(selector, {
          state: 'visible',
          timeout: args.timeout as number || 5000
        });
        await page.click(selector, {
          button: (args.button as ('left' | 'right' | 'middle')) || 'left',
          clickCount: args.clickCount as number || 1,
          delay: args.delay as number || 0
        });
        return `Clicked on ${selector}`;
      },

      type: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        if (!selector) throw new Error('Selector is required for type command');
        if (!args.text) throw new Error('Text is required for type command');

        await page.waitForSelector(selector, {
          state: 'visible',
          timeout: args.timeout as number || 5000
        });

        if (args.clearFirst) {
          await page.evaluate((sel) => {
            const element = document.querySelector(sel);
            if (element) {
              (element as HTMLInputElement).value = '';
            }
          }, selector);
        }

        await page.type(selector, args.text as string, {
          delay: args.delay as number || 0
        });

        return `Typed "${args.text}" into ${selector}`;
      },

      wait: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        if (selector) {
          await page.waitForSelector(selector, {
            state: args.visible !== false ? 'visible' : 'attached',
            timeout: args.timeout as number || 5000
          });
          return `Waited for element ${selector}`;
        } else if (args.time) {
          await new Promise(resolve => setTimeout(resolve, args.time as number));
          return `Waited for ${args.time}ms`;
        } else if (args.function) {
          // Only allow limited wait conditions
          await page.waitForFunction(
            `document.querySelectorAll('${args.functionSelector}').length ${args.functionOperator || '>'} ${args.functionValue || 0}`,
            { timeout: args.timeout as number || 5000 }
          );
          return `Waited for function condition on ${args.functionSelector}`;
        } else {
          throw new Error('Either selector, time, or function parameters are required for wait command');
        }
      },

      navigate: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        if (!args.url) throw new Error('URL is required for navigate command');

        await page.goto(args.url as string, {
          waitUntil: args.waitUntil as ('load' | 'domcontentloaded' | 'networkidle' | 'commit') || 'networkidle0',
          timeout: args.timeout as number || 30000
        });

        return `Navigated to ${args.url}`;
      },

      select: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        if (!selector) throw new Error('Selector is required for select command');
        if (!args.value) throw new Error('Value is required for select command');

        await page.waitForSelector(selector, {
          state: 'visible',
          timeout: args.timeout as number || 5000
        });

        await page.selectOption(selector, args.value as string);

        return `Selected value "${args.value}" in ${selector}`;
      },

      check: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        if (!selector) throw new Error('Selector is required for check command');

        await page.waitForSelector(selector, {
          state: 'visible',
          timeout: args.timeout as number || 5000
        });

        const checked = args.checked !== false;

        if (checked) {
          await page.check(selector);
        } else {
          await page.uncheck(selector);
        }

        return `${checked ? 'Checked' : 'Unchecked'} checkbox ${selector}`;
      },

      hover: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        if (!selector) throw new Error('Selector is required for hover command');

        await page.waitForSelector(selector, {
          state: 'visible',
          timeout: args.timeout as number || 5000
        });

        await page.hover(selector as string);

        return `Hovered over ${selector}`;
      },

      focus: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        if (!selector) throw new Error('Selector is required for focus command');

        await page.waitForSelector(selector, {
          state: 'visible',
          timeout: args.timeout as number || 5000
        });

        await page.focus(selector as string);

        return `Focused on ${selector}`;
      },

      blur: async (page: Page, selector: string | undefined, _args: CommandArgs = {}) => {
        if (!selector) throw new Error('Selector is required for blur command');

        await page.evaluate((sel) => {
          const element = document.querySelector(sel);
          if (element && 'blur' in element) {
            (element as HTMLElement).blur();
          }
        }, selector as string);

        return `Removed focus from ${selector}`;
      },

      keypress: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        if (!args.key) throw new Error('Key is required for keypress command');

        if (selector) {
          await page.waitForSelector(selector, {
            state: 'visible',
            timeout: args.timeout as number || 5000
          });
          await page.focus(selector as string);
        }

        await page.keyboard.press(args.key as string);

        return `Pressed key ${args.key}${selector ? ` on ${selector}` : ''}`;
      },

      scroll: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        const x = args.x as number || 0;
        const y = args.y as number || 0;

        if (selector) {
          await page.waitForSelector(selector, {
            state: 'visible',
            timeout: args.timeout as number || 5000
          });

          await page.evaluate(({ sel, xPos, yPos }: { sel: string; xPos: number; yPos: number }) => {
            const element = document.querySelector(sel);
            if (element) {
              element.scrollBy(xPos, yPos);
            }
          }, { sel: selector, xPos: x, yPos: y });

          return `Scrolled element ${selector} by (${x}, ${y})`;
        } else {
          await page.evaluate(({ xPos, yPos }: { xPos: number; yPos: number }) => {
            window.scrollBy(xPos, yPos);
          }, { xPos: x, yPos: y });

          return `Scrolled window by (${x}, ${y})`;
        }
      },

      getAttribute: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        if (!selector) throw new Error('Selector is required for getAttribute command');
        if (!args.name) throw new Error('Attribute name is required for getAttribute command');

        await page.waitForSelector(selector, {
          state: args.visible !== false ? 'visible' : 'attached',
          timeout: args.timeout as number || 5000
        });

        const attributeValue = await page.evaluate(({ sel, attr }: { sel: string; attr: string }) => {
          const element = document.querySelector(sel);
          return element ? element.getAttribute(attr) : null;
        }, { sel: selector, attr: args.name as string });

        return {
          selector,
          attribute: args.name,
          value: attributeValue
        };
      },

      getProperty: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        if (!selector) throw new Error('Selector is required for getProperty command');
        if (!args.name) throw new Error('Property name is required for getProperty command');

        await page.waitForSelector(selector, {
          state: args.visible !== false ? 'visible' : 'attached',
          timeout: args.timeout as number || 5000
        });

        const propertyValue = await page.evaluate(({ sel, prop }: { sel: string; prop: string }) => {
          const element = document.querySelector(sel);
          return element ? (element as unknown as Record<string, unknown>)[prop] : null;
        }, { sel: selector, prop: args.name as string });

        return {
          selector,
          property: args.name,
          value: propertyValue
        };
      },

      refresh: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        await page.reload({
          waitUntil: args.waitUntil as ('load' | 'domcontentloaded' | 'networkidle' | 'commit') || 'networkidle0',
          timeout: args.timeout as number || 30000
        });

        return 'Refreshed current page';
      },

      drag: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
        // Validate required arguments
        const sourceX = args.sourceX as number | undefined;
        const sourceY = args.sourceY as number | undefined;
        const offsetX = args.offsetX as number | undefined;
        const offsetY = args.offsetY as number | undefined;

        if (sourceX === undefined || sourceY === undefined) {
          throw new Error('sourceX and sourceY are required for drag command');
        }

        if (offsetX === undefined || offsetY === undefined) {
          throw new Error('offsetX and offsetY are required for drag command');
        }

        const smoothDrag = args.smoothDrag === true;
        const steps = args.steps as number || 10;

        // Calculate target coordinates
        const targetX = sourceX + offsetX;
        const targetY = sourceY + offsetY;

        // Perform the drag operation
        await page.mouse.move(sourceX, sourceY);
        await page.mouse.down();

        // Optional: Implement a gradual movement for more realistic drag
        if (smoothDrag) {
          const stepX = offsetX / steps;
          const stepY = offsetY / steps;

          for (let i = 1; i <= steps; i++) {
            await page.mouse.move(
              sourceX + stepX * i,
              sourceY + stepY * i,
              { steps: 1 }
            );
            // Small delay between steps for more natural movement
            await new Promise(resolve => setTimeout(resolve, 10));
          }
        } else {
          // Direct movement
          await page.mouse.move(targetX, targetY);
        }

        // Release the mouse button
        await page.mouse.up();

        return `Dragged from (${sourceX}, ${sourceY}) to (${targetX}, ${targetY}) with offset (${offsetX}, ${offsetY})`;
      }
    };

    // Execute commands sequentially
    const startTime = Date.now();
    const results = [];
    for (const [index, cmd] of commands.entries()) {
      // Check overall timeout
      if (Date.now() - startTime > timeout) {
        results.push({
          commandIndex: index,
          command: cmd.command,
          description: cmd.description,
          status: 'error',
          error: 'Execution timed out'
        });
        break;
      }

      try {
        if (!commandHandlers[cmd.command]) {
          throw new Error(`Unknown command: ${cmd.command}`);
        }

        // Handle selector and args for all commands
        const selector = 'selector' in cmd ? cmd.selector : undefined;
        const args = cmd.args || {};

        const result = await commandHandlers[cmd.command](
          browserStatus.page,
          selector,
          args
        );

        results.push({
          commandIndex: index,
          command: cmd.command,
          description: cmd.description,
          status: 'success',
          result
        });
      } catch (error) {
        const errorMessage = error instanceof Error ? error.message : String(error);
        Logger.error(`Command execution failed: ${errorMessage}`);

        results.push({
          commandIndex: index,
          command: cmd.command,
          description: cmd.description,
          status: 'error',
          error: errorMessage
        });

        // Determine whether to continue based on option
        // Check continueOnError property
        const continueOnError = cmd.args && 'continueOnError' in cmd.args ?
          (cmd.args as Record<string, unknown>).continueOnError === true : false;
        if (!continueOnError) {
          break;
        }
      }
    }

    // Return results
    const resultMessage = {
      totalCommands: commands.length,
      executedCommands: results.length,
      successCount: results.filter(r => r.status === 'success').length,
      failureCount: results.filter(r => r.status === 'error').length,
      elapsedTime: Date.now() - startTime,
      results,
      checkpointId
    };

    return {
      content: [
        {
          type: 'text',
          text: JSON.stringify(resultMessage, null, 2)
        }
      ]
    };
  } catch (error) {
    const errorMessage = error instanceof Error ? error.message : String(error);
    Logger.error(`Failed to execute browser commands: ${errorMessage}`);
    return {
      content: [
        {
          type: 'text',
          text: `Failed to execute browser commands: ${errorMessage}`
        }
      ],
      isError: true
    };
  }
}

src/tools/browser-tools.ts:781-934 (schema)

Comprehensive Zod schema for input validation, using discriminated union to define all supported browser command types with their specific parameters.

{
  commands: z.array(
    z.discriminatedUnion('command', [
      z.object({
        command: z.literal('click'),
        selector: z.string().optional().describe('CSS selector of element to click (required unless x,y coordinates are provided)'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          button: z.enum(['left', 'right', 'middle']).optional().describe('Mouse button to use (default: left)'),
          clickCount: z.number().optional().describe('Number of clicks (default: 1)'),
          delay: z.number().optional().describe('Delay between mousedown and mouseup in ms (default: 0)'),
          x: z.number().optional().describe('X coordinate to click (used instead of selector)'),
          y: z.number().describe('Y coordinate to click (used instead of selector)'),
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        }).optional()
      }),
      z.object({
        command: z.literal('type'),
        selector: z.string().describe('CSS selector of input element to type into'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          text: z.string().describe('Text to type into the element'),
          delay: z.number().optional().describe('Delay between keystrokes in ms (default: 0)'),
          clearFirst: z.boolean().optional().describe('Whether to clear the input field before typing (default: false)'),
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        }).optional()
      }),
      z.object({
        command: z.literal('wait'),
        selector: z.string().optional().describe('CSS selector to wait for'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          time: z.number().optional().describe('Time to wait in milliseconds (use this or selector)'),
          visible: z.boolean().optional().describe('Wait for element to be visible (default: true)'),
          timeout: z.number().optional().describe('Maximum time to wait in ms (default: 5000)'),
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        }).optional()
      }),
      z.object({
        command: z.literal('navigate'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          url: z.string().describe('URL to navigate to'),
          waitUntil: z.enum(['load', 'domcontentloaded', 'networkidle0', 'networkidle2']).optional()
            .describe('Navigation wait condition (default: networkidle0)'),
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        })
      }),
      z.object({
        command: z.literal('drag'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          sourceX: z.number().describe('X coordinate to start the drag from (distance from left edge of viewport)'),
          sourceY: z.number().describe('Y coordinate to start the drag from (distance from top edge of viewport)'),
          offsetX: z.number().describe('Horizontal distance to drag (positive for right, negative for left)'),
          offsetY: z.number().describe('Vertical distance to drag (positive for down, negative for up)'),
          smoothDrag: z.boolean().optional().describe('Whether to perform a smooth, gradual drag movement (default: false)'),
          steps: z.number().optional().describe('Number of intermediate steps for smooth drag (default: 10)'),
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        })
      }),
      z.object({
        command: z.literal('select'),
        selector: z.string().describe('CSS selector of select element'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          value: z.string().describe('Value of the option to select'),
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        })
      }),
      z.object({
        command: z.literal('check'),
        selector: z.string().describe('CSS selector of checkbox element'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          checked: z.boolean().optional().describe('Whether to check or uncheck the box (default: true)'),
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        }).optional()
      }),
      z.object({
        command: z.literal('hover'),
        selector: z.string().describe('CSS selector of element to hover over'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        }).optional()
      }),
      z.object({
        command: z.literal('focus'),
        selector: z.string().describe('CSS selector of element to focus'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        }).optional()
      }),
      z.object({
        command: z.literal('blur'),
        selector: z.string().describe('CSS selector of element to blur'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        }).optional()
      }),
      z.object({
        command: z.literal('keypress'),
        selector: z.string().optional().describe('CSS selector of element to target (optional)'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          key: z.string().describe("Key to press (e.g., 'Enter', 'Tab', 'ArrowDown')"),
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        })
      }),
      z.object({
        command: z.literal('scroll'),
        selector: z.string().optional().describe('CSS selector of element to scroll (scrolls page if not provided)'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          x: z.number().optional().describe('Horizontal scroll amount in pixels (default: 0)'),
          y: z.number().optional().describe('Vertical scroll amount in pixels (default: 0)'),
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        }).optional()
      }),
      z.object({
        command: z.literal('getAttribute'),
        selector: z.string().describe('CSS selector of element'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          name: z.string().describe('Name of the attribute to retrieve'),
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        })
      }),
      z.object({
        command: z.literal('getProperty'),
        selector: z.string().describe('CSS selector of element'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          name: z.string().describe('Name of the property to retrieve'),
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        })
      }),
      z.object({
        command: z.literal('refresh'),
        description: z.string().optional().describe('Description of this command step'),
        args: z.object({
          waitUntil: z.enum(['load', 'domcontentloaded', 'networkidle0', 'networkidle2']).optional()
            .describe('Navigation wait condition (default: networkidle0)'),
          continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
        }).optional()
      })
    ])
  ).describe('Array of commands to execute in sequence'),
  timeout: z.number().optional().describe('Overall timeout in milliseconds (default: 30000)'),
  contextId: z.string().optional().describe('Browser ID to execute commands on (uses most recent browser if not provided)')
},

src/tools/browser-tools.ts:757-1335 (registration)

Direct registration of the tool using server.tool() within the registerBrowserTools function.

  server.tool(
    'execute-browser-commands',
    `Executes a sequence of predefined browser commands safely. Available commands:

- click: Clicks on an element matching the selector or at specified coordinates
- type: Types text into an input element
- wait: Waits for an element, a specified time period, or a condition
- navigate: Navigates to a specified URL
- select: Selects an option in a dropdown
- check: Checks or unchecks a checkbox
- hover: Hovers over an element
- focus: Focuses an element
- blur: Removes focus from an element
- keypress: Simulates pressing a keyboard key
- scroll: Scrolls the page or an element
- getAttribute: Gets an attribute value from an element
- getProperty: Gets a property value from an element
- drag: Performs a drag operation from one position to another
- refresh: Refreshes the current page

Note on coordinates: For all mouse-related commands (click, drag, etc.), coordinates are relative to the browser viewport
where (0,0) is the top-left corner. X increases to the right, Y increases downward.

Examples are available in the schema definition.`,
    {
      commands: z.array(
        z.discriminatedUnion('command', [
          z.object({
            command: z.literal('click'),
            selector: z.string().optional().describe('CSS selector of element to click (required unless x,y coordinates are provided)'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              button: z.enum(['left', 'right', 'middle']).optional().describe('Mouse button to use (default: left)'),
              clickCount: z.number().optional().describe('Number of clicks (default: 1)'),
              delay: z.number().optional().describe('Delay between mousedown and mouseup in ms (default: 0)'),
              x: z.number().optional().describe('X coordinate to click (used instead of selector)'),
              y: z.number().describe('Y coordinate to click (used instead of selector)'),
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            }).optional()
          }),
          z.object({
            command: z.literal('type'),
            selector: z.string().describe('CSS selector of input element to type into'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              text: z.string().describe('Text to type into the element'),
              delay: z.number().optional().describe('Delay between keystrokes in ms (default: 0)'),
              clearFirst: z.boolean().optional().describe('Whether to clear the input field before typing (default: false)'),
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            }).optional()
          }),
          z.object({
            command: z.literal('wait'),
            selector: z.string().optional().describe('CSS selector to wait for'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              time: z.number().optional().describe('Time to wait in milliseconds (use this or selector)'),
              visible: z.boolean().optional().describe('Wait for element to be visible (default: true)'),
              timeout: z.number().optional().describe('Maximum time to wait in ms (default: 5000)'),
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            }).optional()
          }),
          z.object({
            command: z.literal('navigate'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              url: z.string().describe('URL to navigate to'),
              waitUntil: z.enum(['load', 'domcontentloaded', 'networkidle0', 'networkidle2']).optional()
                .describe('Navigation wait condition (default: networkidle0)'),
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            })
          }),
          z.object({
            command: z.literal('drag'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              sourceX: z.number().describe('X coordinate to start the drag from (distance from left edge of viewport)'),
              sourceY: z.number().describe('Y coordinate to start the drag from (distance from top edge of viewport)'),
              offsetX: z.number().describe('Horizontal distance to drag (positive for right, negative for left)'),
              offsetY: z.number().describe('Vertical distance to drag (positive for down, negative for up)'),
              smoothDrag: z.boolean().optional().describe('Whether to perform a smooth, gradual drag movement (default: false)'),
              steps: z.number().optional().describe('Number of intermediate steps for smooth drag (default: 10)'),
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            })
          }),
          z.object({
            command: z.literal('select'),
            selector: z.string().describe('CSS selector of select element'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              value: z.string().describe('Value of the option to select'),
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            })
          }),
          z.object({
            command: z.literal('check'),
            selector: z.string().describe('CSS selector of checkbox element'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              checked: z.boolean().optional().describe('Whether to check or uncheck the box (default: true)'),
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            }).optional()
          }),
          z.object({
            command: z.literal('hover'),
            selector: z.string().describe('CSS selector of element to hover over'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            }).optional()
          }),
          z.object({
            command: z.literal('focus'),
            selector: z.string().describe('CSS selector of element to focus'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            }).optional()
          }),
          z.object({
            command: z.literal('blur'),
            selector: z.string().describe('CSS selector of element to blur'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            }).optional()
          }),
          z.object({
            command: z.literal('keypress'),
            selector: z.string().optional().describe('CSS selector of element to target (optional)'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              key: z.string().describe("Key to press (e.g., 'Enter', 'Tab', 'ArrowDown')"),
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            })
          }),
          z.object({
            command: z.literal('scroll'),
            selector: z.string().optional().describe('CSS selector of element to scroll (scrolls page if not provided)'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              x: z.number().optional().describe('Horizontal scroll amount in pixels (default: 0)'),
              y: z.number().optional().describe('Vertical scroll amount in pixels (default: 0)'),
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            }).optional()
          }),
          z.object({
            command: z.literal('getAttribute'),
            selector: z.string().describe('CSS selector of element'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              name: z.string().describe('Name of the attribute to retrieve'),
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            })
          }),
          z.object({
            command: z.literal('getProperty'),
            selector: z.string().describe('CSS selector of element'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              name: z.string().describe('Name of the property to retrieve'),
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            })
          }),
          z.object({
            command: z.literal('refresh'),
            description: z.string().optional().describe('Description of this command step'),
            args: z.object({
              waitUntil: z.enum(['load', 'domcontentloaded', 'networkidle0', 'networkidle2']).optional()
                .describe('Navigation wait condition (default: networkidle0)'),
              continueOnError: z.boolean().optional().describe('Whether to continue executing commands if this command fails')
            }).optional()
          })
        ])
      ).describe('Array of commands to execute in sequence'),
      timeout: z.number().optional().describe('Overall timeout in milliseconds (default: 30000)'),
      contextId: z.string().optional().describe('Browser ID to execute commands on (uses most recent browser if not provided)')
    },
    async ({ commands, timeout = 30000, contextId }) => {
      try {
        // Check browser status
        const browserStatus = getContextForOperation(contextId);
        if (!browserStatus.isStarted) {
          return browserStatus.error;
        }

        // Get current checkpoint ID
        const checkpointId = await getCurrentCheckpointId(browserStatus.page);

        // Define command handler type
        type CommandArgs = Record<string, unknown>;
        type CommandHandler = (page: Page, selector: string | undefined, args: CommandArgs) => Promise<string | Record<string, unknown>>;

        // Command handler mapping
        const commandHandlers: Record<string, CommandHandler> = {
          click: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            if (!selector) throw new Error('Selector is required for click command');
            await page.waitForSelector(selector, {
              state: 'visible',
              timeout: args.timeout as number || 5000
            });
            await page.click(selector, {
              button: (args.button as ('left' | 'right' | 'middle')) || 'left',
              clickCount: args.clickCount as number || 1,
              delay: args.delay as number || 0
            });
            return `Clicked on ${selector}`;
          },

          type: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            if (!selector) throw new Error('Selector is required for type command');
            if (!args.text) throw new Error('Text is required for type command');

            await page.waitForSelector(selector, {
              state: 'visible',
              timeout: args.timeout as number || 5000
            });

            if (args.clearFirst) {
              await page.evaluate((sel) => {
                const element = document.querySelector(sel);
                if (element) {
                  (element as HTMLInputElement).value = '';
                }
              }, selector);
            }

            await page.type(selector, args.text as string, {
              delay: args.delay as number || 0
            });

            return `Typed "${args.text}" into ${selector}`;
          },

          wait: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            if (selector) {
              await page.waitForSelector(selector, {
                state: args.visible !== false ? 'visible' : 'attached',
                timeout: args.timeout as number || 5000
              });
              return `Waited for element ${selector}`;
            } else if (args.time) {
              await new Promise(resolve => setTimeout(resolve, args.time as number));
              return `Waited for ${args.time}ms`;
            } else if (args.function) {
              // Only allow limited wait conditions
              await page.waitForFunction(
                `document.querySelectorAll('${args.functionSelector}').length ${args.functionOperator || '>'} ${args.functionValue || 0}`,
                { timeout: args.timeout as number || 5000 }
              );
              return `Waited for function condition on ${args.functionSelector}`;
            } else {
              throw new Error('Either selector, time, or function parameters are required for wait command');
            }
          },

          navigate: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            if (!args.url) throw new Error('URL is required for navigate command');

            await page.goto(args.url as string, {
              waitUntil: args.waitUntil as ('load' | 'domcontentloaded' | 'networkidle' | 'commit') || 'networkidle0',
              timeout: args.timeout as number || 30000
            });

            return `Navigated to ${args.url}`;
          },

          select: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            if (!selector) throw new Error('Selector is required for select command');
            if (!args.value) throw new Error('Value is required for select command');

            await page.waitForSelector(selector, {
              state: 'visible',
              timeout: args.timeout as number || 5000
            });

            await page.selectOption(selector, args.value as string);

            return `Selected value "${args.value}" in ${selector}`;
          },

          check: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            if (!selector) throw new Error('Selector is required for check command');

            await page.waitForSelector(selector, {
              state: 'visible',
              timeout: args.timeout as number || 5000
            });

            const checked = args.checked !== false;

            if (checked) {
              await page.check(selector);
            } else {
              await page.uncheck(selector);
            }

            return `${checked ? 'Checked' : 'Unchecked'} checkbox ${selector}`;
          },

          hover: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            if (!selector) throw new Error('Selector is required for hover command');

            await page.waitForSelector(selector, {
              state: 'visible',
              timeout: args.timeout as number || 5000
            });

            await page.hover(selector as string);

            return `Hovered over ${selector}`;
          },

          focus: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            if (!selector) throw new Error('Selector is required for focus command');

            await page.waitForSelector(selector, {
              state: 'visible',
              timeout: args.timeout as number || 5000
            });

            await page.focus(selector as string);

            return `Focused on ${selector}`;
          },

          blur: async (page: Page, selector: string | undefined, _args: CommandArgs = {}) => {
            if (!selector) throw new Error('Selector is required for blur command');

            await page.evaluate((sel) => {
              const element = document.querySelector(sel);
              if (element && 'blur' in element) {
                (element as HTMLElement).blur();
              }
            }, selector as string);

            return `Removed focus from ${selector}`;
          },

          keypress: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            if (!args.key) throw new Error('Key is required for keypress command');

            if (selector) {
              await page.waitForSelector(selector, {
                state: 'visible',
                timeout: args.timeout as number || 5000
              });
              await page.focus(selector as string);
            }

            await page.keyboard.press(args.key as string);

            return `Pressed key ${args.key}${selector ? ` on ${selector}` : ''}`;
          },

          scroll: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            const x = args.x as number || 0;
            const y = args.y as number || 0;

            if (selector) {
              await page.waitForSelector(selector, {
                state: 'visible',
                timeout: args.timeout as number || 5000
              });

              await page.evaluate(({ sel, xPos, yPos }: { sel: string; xPos: number; yPos: number }) => {
                const element = document.querySelector(sel);
                if (element) {
                  element.scrollBy(xPos, yPos);
                }
              }, { sel: selector, xPos: x, yPos: y });

              return `Scrolled element ${selector} by (${x}, ${y})`;
            } else {
              await page.evaluate(({ xPos, yPos }: { xPos: number; yPos: number }) => {
                window.scrollBy(xPos, yPos);
              }, { xPos: x, yPos: y });

              return `Scrolled window by (${x}, ${y})`;
            }
          },

          getAttribute: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            if (!selector) throw new Error('Selector is required for getAttribute command');
            if (!args.name) throw new Error('Attribute name is required for getAttribute command');

            await page.waitForSelector(selector, {
              state: args.visible !== false ? 'visible' : 'attached',
              timeout: args.timeout as number || 5000
            });

            const attributeValue = await page.evaluate(({ sel, attr }: { sel: string; attr: string }) => {
              const element = document.querySelector(sel);
              return element ? element.getAttribute(attr) : null;
            }, { sel: selector, attr: args.name as string });

            return {
              selector,
              attribute: args.name,
              value: attributeValue
            };
          },

          getProperty: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            if (!selector) throw new Error('Selector is required for getProperty command');
            if (!args.name) throw new Error('Property name is required for getProperty command');

            await page.waitForSelector(selector, {
              state: args.visible !== false ? 'visible' : 'attached',
              timeout: args.timeout as number || 5000
            });

            const propertyValue = await page.evaluate(({ sel, prop }: { sel: string; prop: string }) => {
              const element = document.querySelector(sel);
              return element ? (element as unknown as Record<string, unknown>)[prop] : null;
            }, { sel: selector, prop: args.name as string });

            return {
              selector,
              property: args.name,
              value: propertyValue
            };
          },

          refresh: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            await page.reload({
              waitUntil: args.waitUntil as ('load' | 'domcontentloaded' | 'networkidle' | 'commit') || 'networkidle0',
              timeout: args.timeout as number || 30000
            });

            return 'Refreshed current page';
          },

          drag: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
            // Validate required arguments
            const sourceX = args.sourceX as number | undefined;
            const sourceY = args.sourceY as number | undefined;
            const offsetX = args.offsetX as number | undefined;
            const offsetY = args.offsetY as number | undefined;

            if (sourceX === undefined || sourceY === undefined) {
              throw new Error('sourceX and sourceY are required for drag command');
            }

            if (offsetX === undefined || offsetY === undefined) {
              throw new Error('offsetX and offsetY are required for drag command');
            }

            const smoothDrag = args.smoothDrag === true;
            const steps = args.steps as number || 10;

            // Calculate target coordinates
            const targetX = sourceX + offsetX;
            const targetY = sourceY + offsetY;

            // Perform the drag operation
            await page.mouse.move(sourceX, sourceY);
            await page.mouse.down();

            // Optional: Implement a gradual movement for more realistic drag
            if (smoothDrag) {
              const stepX = offsetX / steps;
              const stepY = offsetY / steps;

              for (let i = 1; i <= steps; i++) {
                await page.mouse.move(
                  sourceX + stepX * i,
                  sourceY + stepY * i,
                  { steps: 1 }
                );
                // Small delay between steps for more natural movement
                await new Promise(resolve => setTimeout(resolve, 10));
              }
            } else {
              // Direct movement
              await page.mouse.move(targetX, targetY);
            }

            // Release the mouse button
            await page.mouse.up();

            return `Dragged from (${sourceX}, ${sourceY}) to (${targetX}, ${targetY}) with offset (${offsetX}, ${offsetY})`;
          }
        };

        // Execute commands sequentially
        const startTime = Date.now();
        const results = [];
        for (const [index, cmd] of commands.entries()) {
          // Check overall timeout
          if (Date.now() - startTime > timeout) {
            results.push({
              commandIndex: index,
              command: cmd.command,
              description: cmd.description,
              status: 'error',
              error: 'Execution timed out'
            });
            break;
          }

          try {
            if (!commandHandlers[cmd.command]) {
              throw new Error(`Unknown command: ${cmd.command}`);
            }

            // Handle selector and args for all commands
            const selector = 'selector' in cmd ? cmd.selector : undefined;
            const args = cmd.args || {};

            const result = await commandHandlers[cmd.command](
              browserStatus.page,
              selector,
              args
            );

            results.push({
              commandIndex: index,
              command: cmd.command,
              description: cmd.description,
              status: 'success',
              result
            });
          } catch (error) {
            const errorMessage = error instanceof Error ? error.message : String(error);
            Logger.error(`Command execution failed: ${errorMessage}`);

            results.push({
              commandIndex: index,
              command: cmd.command,
              description: cmd.description,
              status: 'error',
              error: errorMessage
            });

            // Determine whether to continue based on option
            // Check continueOnError property
            const continueOnError = cmd.args && 'continueOnError' in cmd.args ?
              (cmd.args as Record<string, unknown>).continueOnError === true : false;
            if (!continueOnError) {
              break;
            }
          }
        }

        // Return results
        const resultMessage = {
          totalCommands: commands.length,
          executedCommands: results.length,
          successCount: results.filter(r => r.status === 'success').length,
          failureCount: results.filter(r => r.status === 'error').length,
          elapsedTime: Date.now() - startTime,
          results,
          checkpointId
        };

        return {
          content: [
            {
              type: 'text',
              text: JSON.stringify(resultMessage, null, 2)
            }
          ]
        };
      } catch (error) {
        const errorMessage = error instanceof Error ? error.message : String(error);
        Logger.error(`Failed to execute browser commands: ${errorMessage}`);
        return {
          content: [
            {
              type: 'text',
              text: `Failed to execute browser commands: ${errorMessage}`
            }
          ],
          isError: true
        };
      }
    }
  );

src/tools/browser-tools.ts:951-1240 (helper)

Mapping of individual command handlers that implement specific Playwright operations for each command type.

const commandHandlers: Record<string, CommandHandler> = {
  click: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    if (!selector) throw new Error('Selector is required for click command');
    await page.waitForSelector(selector, {
      state: 'visible',
      timeout: args.timeout as number || 5000
    });
    await page.click(selector, {
      button: (args.button as ('left' | 'right' | 'middle')) || 'left',
      clickCount: args.clickCount as number || 1,
      delay: args.delay as number || 0
    });
    return `Clicked on ${selector}`;
  },

  type: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    if (!selector) throw new Error('Selector is required for type command');
    if (!args.text) throw new Error('Text is required for type command');

    await page.waitForSelector(selector, {
      state: 'visible',
      timeout: args.timeout as number || 5000
    });

    if (args.clearFirst) {
      await page.evaluate((sel) => {
        const element = document.querySelector(sel);
        if (element) {
          (element as HTMLInputElement).value = '';
        }
      }, selector);
    }

    await page.type(selector, args.text as string, {
      delay: args.delay as number || 0
    });

    return `Typed "${args.text}" into ${selector}`;
  },

  wait: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    if (selector) {
      await page.waitForSelector(selector, {
        state: args.visible !== false ? 'visible' : 'attached',
        timeout: args.timeout as number || 5000
      });
      return `Waited for element ${selector}`;
    } else if (args.time) {
      await new Promise(resolve => setTimeout(resolve, args.time as number));
      return `Waited for ${args.time}ms`;
    } else if (args.function) {
      // Only allow limited wait conditions
      await page.waitForFunction(
        `document.querySelectorAll('${args.functionSelector}').length ${args.functionOperator || '>'} ${args.functionValue || 0}`,
        { timeout: args.timeout as number || 5000 }
      );
      return `Waited for function condition on ${args.functionSelector}`;
    } else {
      throw new Error('Either selector, time, or function parameters are required for wait command');
    }
  },

  navigate: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    if (!args.url) throw new Error('URL is required for navigate command');

    await page.goto(args.url as string, {
      waitUntil: args.waitUntil as ('load' | 'domcontentloaded' | 'networkidle' | 'commit') || 'networkidle0',
      timeout: args.timeout as number || 30000
    });

    return `Navigated to ${args.url}`;
  },

  select: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    if (!selector) throw new Error('Selector is required for select command');
    if (!args.value) throw new Error('Value is required for select command');

    await page.waitForSelector(selector, {
      state: 'visible',
      timeout: args.timeout as number || 5000
    });

    await page.selectOption(selector, args.value as string);

    return `Selected value "${args.value}" in ${selector}`;
  },

  check: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    if (!selector) throw new Error('Selector is required for check command');

    await page.waitForSelector(selector, {
      state: 'visible',
      timeout: args.timeout as number || 5000
    });

    const checked = args.checked !== false;

    if (checked) {
      await page.check(selector);
    } else {
      await page.uncheck(selector);
    }

    return `${checked ? 'Checked' : 'Unchecked'} checkbox ${selector}`;
  },

  hover: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    if (!selector) throw new Error('Selector is required for hover command');

    await page.waitForSelector(selector, {
      state: 'visible',
      timeout: args.timeout as number || 5000
    });

    await page.hover(selector as string);

    return `Hovered over ${selector}`;
  },

  focus: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    if (!selector) throw new Error('Selector is required for focus command');

    await page.waitForSelector(selector, {
      state: 'visible',
      timeout: args.timeout as number || 5000
    });

    await page.focus(selector as string);

    return `Focused on ${selector}`;
  },

  blur: async (page: Page, selector: string | undefined, _args: CommandArgs = {}) => {
    if (!selector) throw new Error('Selector is required for blur command');

    await page.evaluate((sel) => {
      const element = document.querySelector(sel);
      if (element && 'blur' in element) {
        (element as HTMLElement).blur();
      }
    }, selector as string);

    return `Removed focus from ${selector}`;
  },

  keypress: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    if (!args.key) throw new Error('Key is required for keypress command');

    if (selector) {
      await page.waitForSelector(selector, {
        state: 'visible',
        timeout: args.timeout as number || 5000
      });
      await page.focus(selector as string);
    }

    await page.keyboard.press(args.key as string);

    return `Pressed key ${args.key}${selector ? ` on ${selector}` : ''}`;
  },

  scroll: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    const x = args.x as number || 0;
    const y = args.y as number || 0;

    if (selector) {
      await page.waitForSelector(selector, {
        state: 'visible',
        timeout: args.timeout as number || 5000
      });

      await page.evaluate(({ sel, xPos, yPos }: { sel: string; xPos: number; yPos: number }) => {
        const element = document.querySelector(sel);
        if (element) {
          element.scrollBy(xPos, yPos);
        }
      }, { sel: selector, xPos: x, yPos: y });

      return `Scrolled element ${selector} by (${x}, ${y})`;
    } else {
      await page.evaluate(({ xPos, yPos }: { xPos: number; yPos: number }) => {
        window.scrollBy(xPos, yPos);
      }, { xPos: x, yPos: y });

      return `Scrolled window by (${x}, ${y})`;
    }
  },

  getAttribute: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    if (!selector) throw new Error('Selector is required for getAttribute command');
    if (!args.name) throw new Error('Attribute name is required for getAttribute command');

    await page.waitForSelector(selector, {
      state: args.visible !== false ? 'visible' : 'attached',
      timeout: args.timeout as number || 5000
    });

    const attributeValue = await page.evaluate(({ sel, attr }: { sel: string; attr: string }) => {
      const element = document.querySelector(sel);
      return element ? element.getAttribute(attr) : null;
    }, { sel: selector, attr: args.name as string });

    return {
      selector,
      attribute: args.name,
      value: attributeValue
    };
  },

  getProperty: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    if (!selector) throw new Error('Selector is required for getProperty command');
    if (!args.name) throw new Error('Property name is required for getProperty command');

    await page.waitForSelector(selector, {
      state: args.visible !== false ? 'visible' : 'attached',
      timeout: args.timeout as number || 5000
    });

    const propertyValue = await page.evaluate(({ sel, prop }: { sel: string; prop: string }) => {
      const element = document.querySelector(sel);
      return element ? (element as unknown as Record<string, unknown>)[prop] : null;
    }, { sel: selector, prop: args.name as string });

    return {
      selector,
      property: args.name,
      value: propertyValue
    };
  },

  refresh: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    await page.reload({
      waitUntil: args.waitUntil as ('load' | 'domcontentloaded' | 'networkidle' | 'commit') || 'networkidle0',
      timeout: args.timeout as number || 30000
    });

    return 'Refreshed current page';
  },

  drag: async (page: Page, selector: string | undefined, args: CommandArgs = {}) => {
    // Validate required arguments
    const sourceX = args.sourceX as number | undefined;
    const sourceY = args.sourceY as number | undefined;
    const offsetX = args.offsetX as number | undefined;
    const offsetY = args.offsetY as number | undefined;

    if (sourceX === undefined || sourceY === undefined) {
      throw new Error('sourceX and sourceY are required for drag command');
    }

    if (offsetX === undefined || offsetY === undefined) {
      throw new Error('offsetX and offsetY are required for drag command');
    }

    const smoothDrag = args.smoothDrag === true;
    const steps = args.steps as number || 10;

    // Calculate target coordinates
    const targetX = sourceX + offsetX;
    const targetY = sourceY + offsetY;

    // Perform the drag operation
    await page.mouse.move(sourceX, sourceY);
    await page.mouse.down();

    // Optional: Implement a gradual movement for more realistic drag
    if (smoothDrag) {
      const stepX = offsetX / steps;
      const stepY = offsetY / steps;

      for (let i = 1; i <= steps; i++) {
        await page.mouse.move(
          sourceX + stepX * i,
          sourceY + stepY * i,
          { steps: 1 }
        );
        // Small delay between steps for more natural movement
        await new Promise(resolve => setTimeout(resolve, 10));
      }
    } else {
      // Direct movement
      await page.mouse.move(targetX, targetY);
    }

    // Release the mouse button
    await page.mouse.up();

    return `Dragged from (${sourceX}, ${sourceY}) to (${targetX}, ${targetY}) with offset (${offsetX}, ${offsetY})`;
  }
};

Vite MCP Server