Bright Data Web MCP

Instructions

Implementation Reference

• server.js:296-305 (registration)

  Dataset configuration defining the 'amazon_product_search' id (used for tool name 'web_data_amazon_product_search'), BrightData dataset_id, description, input parameters (keyword, url, pages_to_search with default), used for registration and schema.

      id: 'amazon_product_search',
      dataset_id: 'gd_lwdb4vjm1ehb499uxs',
      description: [
          'Quickly read structured amazon product search data.',
          'Requires a valid search keyword and amazon domain URL.',
          'This can be a cache lookup, so it can be more reliable than scraping',
      ].join('\n'),
      inputs: ['keyword', 'url', 'pages_to_search'],
      defaults: {pages_to_search: '1'},
  }, {

• server.js:683-686 (schema)

  Dynamic schema and registration snippet showing tool name construction `web_data_${id}` where id='amazon_product_search', and parameters schema built from inputs array as z.object() with Zod validators.

  addTool({
      name: `web_data_${id}`,
      description,
      parameters: z.object(parameters),

• server.js:687-743 (handler)

  Core handler logic for the tool: POSTs input data to BrightData datasets/v3/trigger using specific dataset_id 'gd_lwdb4vjm1ehb499uxs', polls /snapshot/{id} up to 600s until ready, returns JSON.stringify(response.data). Interpolates tool-specific name and dataset_id.

  execute: tool_fn(`web_data_${id}`, async(data, ctx)=>{
      let trigger_response = await axios({
          url: 'https://api.brightdata.com/datasets/v3/trigger',
          params: {dataset_id, include_errors: true},
          method: 'POST',
          data: [data],
          headers: api_headers(),
      });
      if (!trigger_response.data?.snapshot_id)
          throw new Error('No snapshot ID returned from request');
      let snapshot_id = trigger_response.data.snapshot_id;
      console.error(`[web_data_${id}] triggered collection with `
          +`snapshot ID: ${snapshot_id}`);
      let max_attempts = 600;
      let attempts = 0;
      while (attempts < max_attempts)
      {
          try {
              if (ctx && ctx.reportProgress)
              {
                  await ctx.reportProgress({
                      progress: attempts,
                      total: max_attempts,
                      message: `Polling for data (attempt `
                          +`${attempts + 1}/${max_attempts})`,
                  });
              }
              let snapshot_response = await axios({
                  url: `https://api.brightdata.com/datasets/v3`
                      +`/snapshot/${snapshot_id}`,
                  params: {format: 'json'},
                  method: 'GET',
                  headers: api_headers(),
              });
              if (['running', 'building'].includes(snapshot_response.data?.status))
              {
                  console.error(`[web_data_${id}] snapshot not ready, `
                      +`polling again (attempt `
                      +`${attempts + 1}/${max_attempts})`);
                  attempts++;
                  await new Promise(resolve=>setTimeout(resolve, 1000));
                  continue;
              }
              console.error(`[web_data_${id}] snapshot data received `
                  +`after ${attempts + 1} attempts`);
              let result_data = JSON.stringify(snapshot_response.data);
              return result_data;
          } catch(e){
              console.error(`[web_data_${id}] polling error: `
                  +`${e.message}`);
              attempts++;
              await new Promise(resolve=>setTimeout(resolve, 1000));
          }
      }
      throw new Error(`Timeout after ${max_attempts} seconds waiting `
          +`for data`);
  }),

• server.js:752-778 (helper)

  Helper wrapper tool_fn used in execute, provides rate limiting (check_rate_limit), stats tracking, execution logging with tool name 'web_data_amazon_product_search', error handling/logging, timing.

  function tool_fn(name, fn){
      return async(data, ctx)=>{
          check_rate_limit();
          debug_stats.tool_calls[name] = debug_stats.tool_calls[name]||0;
          debug_stats.tool_calls[name]++;
          debug_stats.session_calls++;
          let ts = Date.now();
          console.error(`[%s] executing %s`, name, JSON.stringify(data));
          try { return await fn(data, ctx); }
          catch(e){
              if (e.response)
              {
                  console.error(`[%s] error %s %s: %s`, name, e.response.status,
                      e.response.statusText, e.response.data);
                  let message = e.response.data;
                  if (message?.length)
                      throw new Error(`HTTP ${e.response.status}: ${message}`);
              }
              else
                  console.error(`[%s] error %s`, name, e.stack);
              throw e;
          } finally {
              let dur = Date.now()-ts;
              console.error(`[%s] tool finished in %sms`, name, dur);
          }
      };
  }