detect-all-objects
Detect and identify all objects in an image, providing their categories, counts, coordinates, and detailed descriptions using advanced visual analysis for precise object recognition.
Instructions
Analyze an image to detect all identifiable objects, returning the category, count, coordinate positions and detailed descriptions for each object.
Input Schema
TableJSON Schema
| Name | Required | Description | Default |
|---|---|---|---|
| imageFileUri | Yes | URI of the input image. Preferred for remote or local files. Must start with "https://" or "file://". | |
| includeDescription | Yes | Whether to return a description of the objects detected in the image, but will take longer to process. |
Implementation Reference
- src/constants/tool.ts:16-19 (schema)Tool schema configuration defining the name and description for the 'detect-all-objects' tool used across servers.[Tool.DETECT_ALL_OBJECTS]: { name: Tool.DETECT_ALL_OBJECTS, description: "Analyze an image to detect all identifiable objects, returning the category, count, coordinate positions and detailed descriptions for each object.", },
- src/servers/http-server.ts:153-228 (handler)Primary handler implementation for 'detect-all-objects' tool in HTTP MCP server. Includes inline Zod input schema, API client call, bbox parsing, categorization, and multi-part text response.private registerDetectAllObjectsTool(): void { const { name, description } = ToolConfigs[Tool.DETECT_ALL_OBJECTS]; this.server.tool( name, description, { imageFileUri: z.string().describe("URI of the input image. Preferred for remote or local files. Must start with 'https://'."), includeDescription: z.boolean().describe("Whether to return a description of the objects detected in the image, but will take longer to process."), }, async (args) => { try { const { imageFileUri, includeDescription } = args; if (!imageFileUri) { return { content: [ { type: 'text', text: 'Image file URI is required', }, ], } } const { objects } = await this.api.detectAllObjects(imageFileUri, includeDescription); const categories: ResultCategory = {}; for (const object of objects) { if (!categories[object.category]) { categories[object.category] = []; } categories[object.category].push(object); } const objectsInfo = objects.map(obj => { const bbox = parseBbox(obj.bbox); return { name: obj.category, bbox, ...(includeDescription ? { description: obj.caption, } : {}), } }); return { content: [ { type: "text", text: `Objects detected in image: ${Object.keys(categories).map(cat => `${cat} (${categories[cat].length})` )?.join(', ')}.` }, { type: "text", text: `Detailed object detection results: ${JSON.stringify(objectsInfo, null, 2)}` }, { type: "text", text: `Note: The bbox coordinates are in [xmin, ymin, xmax, ymax] format, where the origin (0,0) is at the top-left corner of the image. These coordinates help determine the exact position and spatial relationships of objects in the image.` }, ] }; } catch (error) { return { content: [ { type: 'text', text: `Failed to detect objects from image: ${error instanceof Error ? error.message : String(error)}`, }, ], }; } } ) }
- src/servers/stdio-server.ts:120-195 (handler)Identical handler implementation for 'detect-all-objects' tool in STDIO MCP server.private registerDetectAllObjectsTool(): void { const { name, description } = ToolConfigs[Tool.DETECT_ALL_OBJECTS]; this.server.tool( name, description, { imageFileUri: z.string().describe("URI of the input image. Preferred for remote or local files. Must start with 'https://' or 'file://'."), includeDescription: z.boolean().describe("Whether to return a description of the objects detected in the image, but will take longer to process."), }, async (args) => { try { const { imageFileUri, includeDescription } = args; if (!imageFileUri) { return { content: [ { type: 'text', text: 'Image file URI is required', }, ], } } const { objects } = await this.api.detectAllObjects(imageFileUri, includeDescription); const categories: ResultCategory = {}; for (const object of objects) { if (!categories[object.category]) { categories[object.category] = []; } categories[object.category].push(object); } const objectsInfo = objects.map(obj => { const bbox = parseBbox(obj.bbox); return { name: obj.category, bbox, ...(includeDescription ? { description: obj.caption, } : {}), } }); return { content: [ { type: "text", text: `Objects detected in image: ${Object.keys(categories).map(cat => `${cat} (${categories[cat].length})` )?.join(', ')}.` }, { type: "text", text: `Detailed object detection results: ${JSON.stringify(objectsInfo, null, 2)}` }, { type: "text", text: `Note: The bbox coordinates are in [xmin, ymin, xmax, ymax] format, where the origin (0,0) is at the top-left corner of the image. These coordinates help determine the exact position and spatial relationships of objects in the image.` }, ] }; } catch (error) { return { content: [ { type: 'text', text: `Failed to detect objects from image: ${error instanceof Error ? error.message : String(error)}`, }, ], }; } } ) }
- src/dino-x/client.ts:203-217 (helper)DinoX API client helper method that performs the detection-all-objects API request using a universal prompt.async detectAllObjects( imageFileUri: string, includeDescription: boolean ): Promise<DetectionResult> { return this.performDetection(imageFileUri, includeDescription, { model: "DINO-X-1.0", prompt: { type: "universal", universal: 1 }, targets: ["bbox"], bbox_threshold: 0.25, iou_threshold: 0.8 }); }
- src/utils/index.ts:82-86 (helper)Utility helper to parse bbox array into object format used in tool handlers.export const parseBbox = (bbox: number[]) => { return { xmin: parseFloat(bbox[0].toFixed(1)), ymin: parseFloat(bbox[1].toFixed(1)), xmax: parseFloat(bbox[2].toFixed(1)),