desktop-client.ts•6.09 kB
/**
* ByteBot Desktop API HTTP Client
*
* Provides HTTP client for interacting with ByteBot Desktop API (computer control)
* Includes error handling, retry logic, and action routing
*/
import axios, { AxiosInstance } from 'axios';
import {
DesktopActionRequest,
DesktopActionResponse,
Coordinates,
} from '../types/bytebot.js';
import { EnvironmentConfig, DesktopActionResult } from '../types/mcp.js';
import {
handleAxiosError,
withRetry,
logError,
} from '../utils/error-handler.js';
/**
* Desktop API Client
*/
export class DesktopClient {
private client: AxiosInstance;
private config: EnvironmentConfig;
constructor(config: EnvironmentConfig) {
this.config = config;
this.client = axios.create({
baseURL: config.desktopUrl,
timeout: config.desktopActionTimeout,
headers: {
'Content-Type': 'application/json',
},
});
// Log client initialization
console.log(
`[ByteBot MCP] Desktop API client initialized: ${config.desktopUrl}`
);
}
/**
* Execute a desktop action
*/
async executeAction(
request: DesktopActionRequest
): Promise<DesktopActionResult> {
const startTime = Date.now();
try {
console.log(
`[ByteBot MCP] Executing desktop action: ${request.action}`
);
// Make request with retry logic
const response = await withRetry(
async () => {
return await this.client.post<DesktopActionResponse>(
'/computer-use',
request
);
},
{
maxRetries: this.config.maxRetries,
delay: this.config.retryDelay,
backoffMultiplier: 2,
maxDelay: 10000,
}
);
const duration = Date.now() - startTime;
console.log(
`[ByteBot MCP] Desktop action ${request.action} completed in ${duration}ms`
);
return {
...response.data,
action: request.action,
executedAt: new Date().toISOString(),
duration,
};
} catch (error) {
logError(`executeAction(${request.action})`, error);
throw handleAxiosError(error);
}
}
/**
* Mouse actions
*/
async moveMouse(x: number, y: number): Promise<DesktopActionResult> {
return this.executeAction({
action: 'move_mouse',
x,
y,
});
}
async clickMouse(
x: number,
y: number,
button: 'left' | 'right' | 'middle' = 'left',
count = 1
): Promise<DesktopActionResult> {
return this.executeAction({
action: 'click_mouse',
x,
y,
button,
count,
});
}
async dragMouse(
fromX: number,
fromY: number,
toX: number,
toY: number
): Promise<DesktopActionResult> {
return this.executeAction({
action: 'drag_mouse',
from_x: fromX,
from_y: fromY,
to_x: toX,
to_y: toY,
});
}
async pressMouse(
button: 'left' | 'right' | 'middle' = 'left',
down = true
): Promise<DesktopActionResult> {
return this.executeAction({
action: 'press_mouse',
button,
down,
});
}
async scroll(
direction: 'up' | 'down' | 'left' | 'right',
count = 1
): Promise<DesktopActionResult> {
return this.executeAction({
action: 'scroll',
direction,
count,
});
}
async traceMouse(coordinates: Coordinates[]): Promise<DesktopActionResult> {
return this.executeAction({
action: 'trace_mouse',
coordinates,
});
}
/**
* Keyboard actions
*/
async typeText(text: string, delay?: number): Promise<DesktopActionResult> {
return this.executeAction({
action: 'type_text',
text,
delay,
});
}
async pasteText(text: string): Promise<DesktopActionResult> {
return this.executeAction({
action: 'paste_text',
text,
});
}
async pressKeys(keys: string[]): Promise<DesktopActionResult> {
return this.executeAction({
action: 'press_keys',
keys,
});
}
async typeKeys(keys: string[]): Promise<DesktopActionResult> {
return this.executeAction({
action: 'type_keys',
keys,
});
}
/**
* Screen actions
*/
async screenshot(): Promise<DesktopActionResult> {
return this.executeAction({
action: 'screenshot',
});
}
async getCursorPosition(): Promise<DesktopActionResult> {
return this.executeAction({
action: 'cursor_position',
});
}
/**
* File I/O actions
*/
async readFile(path: string): Promise<DesktopActionResult> {
return this.executeAction({
action: 'read_file',
path,
});
}
async writeFile(
path: string,
content: string
): Promise<DesktopActionResult> {
return this.executeAction({
action: 'write_file',
path,
content,
});
}
/**
* System actions
*/
async switchApplication(name: string): Promise<DesktopActionResult> {
return this.executeAction({
action: 'application',
name,
});
}
async wait(duration: number): Promise<DesktopActionResult> {
return this.executeAction({
action: 'wait',
duration,
});
}
/**
* Health check
*/
async healthCheck(): Promise<boolean> {
try {
const result = await this.getCursorPosition();
return result.success;
} catch (error) {
console.error('[ByteBot MCP] Desktop API health check failed:', error);
return false;
}
}
/**
* Helper: Validate screenshot size and provide warnings
*/
validateScreenshotSize(base64Data: string): void {
const sizeBytes = (base64Data.length * 3) / 4;
const sizeMB = sizeBytes / (1024 * 1024);
if (sizeMB > 5) {
console.warn(
`[ByteBot MCP] Warning: Screenshot size is ${sizeMB.toFixed(2)}MB. Consider using image compression or reducing screen resolution.`
);
}
}
/**
* Helper: Convert base64 screenshot to data URL
*/
screenshotToDataUrl(base64Data: string, mimeType = 'image/png'): string {
return `data:${mimeType};base64,${base64Data}`;
}
}