Bright Data Web MCP

Instructions

Implementation Reference

• server.js:687-743 (handler)

  Core handler logic for the web_data_apple_app_store tool (shared with other web_data_* tools). Triggers a BrightData dataset collection using the specific dataset_id, polls the snapshot status up to 600 seconds, and returns the structured JSON data.
  execute: tool_fn(`web_data_${id}`, async(data, ctx)=>{ let trigger_response = await axios({ url: 'https://api.brightdata.com/datasets/v3/trigger', params: {dataset_id, include_errors: true}, method: 'POST', data: [data], headers: api_headers(), }); if (!trigger_response.data?.snapshot_id) throw new Error('No snapshot ID returned from request'); let snapshot_id = trigger_response.data.snapshot_id; console.error(`[web_data_${id}] triggered collection with ` +`snapshot ID: ${snapshot_id}`); let max_attempts = 600; let attempts = 0; while (attempts < max_attempts) { try { if (ctx && ctx.reportProgress) { await ctx.reportProgress({ progress: attempts, total: max_attempts, message: `Polling for data (attempt ` +`${attempts + 1}/${max_attempts})`, }); } let snapshot_response = await axios({ url: `https://api.brightdata.com/datasets/v3` +`/snapshot/${snapshot_id}`, params: {format: 'json'}, method: 'GET', headers: api_headers(), }); if (['running', 'building'].includes(snapshot_response.data?.status)) { console.error(`[web_data_${id}] snapshot not ready, ` +`polling again (attempt ` +`${attempts + 1}/${max_attempts})`); attempts++; await new Promise(resolve=>setTimeout(resolve, 1000)); continue; } console.error(`[web_data_${id}] snapshot data received ` +`after ${attempts + 1} attempts`); let result_data = JSON.stringify(snapshot_response.data); return result_data; } catch(e){ console.error(`[web_data_${id}] polling error: ` +`${e.message}`); attempts++; await new Promise(resolve=>setTimeout(resolve, 1000)); } } throw new Error(`Timeout after ${max_attempts} seconds waiting ` +`for data`); }),
• server.js:570-578 (registration)

  Dataset configuration entry for 'apple_app_store', which defines the dataset_id 'gd_lsk9ki3u2iishmwrui', description, and input schema (url), used to register the web_data_apple_app_store tool.
  id: 'apple_app_store', dataset_id: 'gd_lsk9ki3u2iishmwrui', description: [ 'Quickly read structured apple app store data.', 'Requires a valid apple app store app URL.', 'This can be a cache lookup, so it can be more reliable than scraping', ].join('\n'), inputs: ['url'], }, {
• server.js:674-745 (registration)

  Dynamic registration loop that creates and registers the web_data_apple_app_store tool (and others) with FastMCP server.addTool(), including dynamic schema from inputs and shared handler.
  for (let {dataset_id, id, description, inputs, defaults = {}} of datasets) { let parameters = {}; for (let input of inputs) { let param_schema = input=='url' ? z.string().url() : z.string(); parameters[input] = defaults[input] !== undefined ? param_schema.default(defaults[input]) : param_schema; } addTool({ name: `web_data_${id}`, description, parameters: z.object(parameters), execute: tool_fn(`web_data_${id}`, async(data, ctx)=>{ let trigger_response = await axios({ url: 'https://api.brightdata.com/datasets/v3/trigger', params: {dataset_id, include_errors: true}, method: 'POST', data: [data], headers: api_headers(), }); if (!trigger_response.data?.snapshot_id) throw new Error('No snapshot ID returned from request'); let snapshot_id = trigger_response.data.snapshot_id; console.error(`[web_data_${id}] triggered collection with ` +`snapshot ID: ${snapshot_id}`); let max_attempts = 600; let attempts = 0; while (attempts < max_attempts) { try { if (ctx && ctx.reportProgress) { await ctx.reportProgress({ progress: attempts, total: max_attempts, message: `Polling for data (attempt ` +`${attempts + 1}/${max_attempts})`, }); } let snapshot_response = await axios({ url: `https://api.brightdata.com/datasets/v3` +`/snapshot/${snapshot_id}`, params: {format: 'json'}, method: 'GET', headers: api_headers(), }); if (['running', 'building'].includes(snapshot_response.data?.status)) { console.error(`[web_data_${id}] snapshot not ready, ` +`polling again (attempt ` +`${attempts + 1}/${max_attempts})`); attempts++; await new Promise(resolve=>setTimeout(resolve, 1000)); continue; } console.error(`[web_data_${id}] snapshot data received ` +`after ${attempts + 1} attempts`); let result_data = JSON.stringify(snapshot_response.data); return result_data; } catch(e){ console.error(`[web_data_${id}] polling error: ` +`${e.message}`); attempts++; await new Promise(resolve=>setTimeout(resolve, 1000)); } } throw new Error(`Timeout after ${max_attempts} seconds waiting ` +`for data`); }), }); }
• server.js:676-682 (schema)

  Dynamic schema construction for tool parameters using Zod. For apple_app_store, results in z.object({url: z.string().url()}).
  let parameters = {}; for (let input of inputs) { let param_schema = input=='url' ? z.string().url() : z.string(); parameters[input] = defaults[input] !== undefined ? param_schema.default(defaults[input]) : param_schema; }
• server.js:752-778 (helper)

  Wrapper function tool_fn that adds rate limiting, stats tracking, logging, error handling, and timing around the core handler for all tools including web_data_apple_app_store.
  function tool_fn(name, fn){ return async(data, ctx)=>{ check_rate_limit(); debug_stats.tool_calls[name] = debug_stats.tool_calls[name]||0; debug_stats.tool_calls[name]++; debug_stats.session_calls++; let ts = Date.now(); console.error(`[%s] executing %s`, name, JSON.stringify(data)); try { return await fn(data, ctx); } catch(e){ if (e.response) { console.error(`[%s] error %s %s: %s`, name, e.response.status, e.response.statusText, e.response.data); let message = e.response.data; if (message?.length) throw new Error(`HTTP ${e.response.status}: ${message}`); } else console.error(`[%s] error %s`, name, e.stack); throw e; } finally { let dur = Date.now()-ts; console.error(`[%s] tool finished in %sms`, name, dur); } }; }