Bright Data Web MCP

Instructions

Implementation Reference

• server.js:687-746 (handler)

  The execute handler for the web_data_google_play_store tool (shared with other web_data_* tools). Triggers a BrightData dataset snapshot using the provided inputs (e.g., app URL), polls until ready, and returns the structured JSON data.

          execute: tool_fn(`web_data_${id}`, async(data, ctx)=>{
              let trigger_response = await axios({
                  url: 'https://api.brightdata.com/datasets/v3/trigger',
                  params: {dataset_id, include_errors: true},
                  method: 'POST',
                  data: [data],
                  headers: api_headers(),
              });
              if (!trigger_response.data?.snapshot_id)
                  throw new Error('No snapshot ID returned from request');
              let snapshot_id = trigger_response.data.snapshot_id;
              console.error(`[web_data_${id}] triggered collection with `
                  +`snapshot ID: ${snapshot_id}`);
              let max_attempts = 600;
              let attempts = 0;
              while (attempts < max_attempts)
              {
                  try {
                      if (ctx && ctx.reportProgress)
                      {
                          await ctx.reportProgress({
                              progress: attempts,
                              total: max_attempts,
                              message: `Polling for data (attempt `
                                  +`${attempts + 1}/${max_attempts})`,
                          });
                      }
                      let snapshot_response = await axios({
                          url: `https://api.brightdata.com/datasets/v3`
                              +`/snapshot/${snapshot_id}`,
                          params: {format: 'json'},
                          method: 'GET',
                          headers: api_headers(),
                      });
                      if (['running', 'building'].includes(snapshot_response.data?.status))
                      {
                          console.error(`[web_data_${id}] snapshot not ready, `
                              +`polling again (attempt `
                              +`${attempts + 1}/${max_attempts})`);
                          attempts++;
                          await new Promise(resolve=>setTimeout(resolve, 1000));
                          continue;
                      }
                      console.error(`[web_data_${id}] snapshot data received `
                          +`after ${attempts + 1} attempts`);
                      let result_data = JSON.stringify(snapshot_response.data);
                      return result_data;
                  } catch(e){
                      console.error(`[web_data_${id}] polling error: `
                          +`${e.message}`);
                      attempts++;
                      await new Promise(resolve=>setTimeout(resolve, 1000));
                  }
              }
              throw new Error(`Timeout after ${max_attempts} seconds waiting `
                  +`for data`);
          }),
      });
  }

• server.js:561-569 (schema)

  Dataset configuration defining the inputs (url) and dataset_id for google_play_store, used to generate the Zod input schema z.object({url: z.string().url()}) and description for the web_data_google_play_store tool.

      id: 'google_play_store',
      dataset_id: 'gd_lsk382l8xei8vzm4u',
      description: [
          'Quickly read structured Google play store data.',
          'Requires a valid Google play store app URL.',
          'This can be a cache lookup, so it can be more reliable than scraping',
      ].join('\n'),
      inputs: ['url'],
  }, {

• server.js:674-747 (registration)

  Registration loop that dynamically creates and registers the web_data_google_play_store tool using the google_play_store dataset config, assigning name 'web_data_google_play_store', Zod schema from inputs, and shared handler.

  for (let {dataset_id, id, description, inputs, defaults = {}} of datasets)
  {
      let parameters = {};
      for (let input of inputs)
      {
          let param_schema = input=='url' ? z.string().url() : z.string();
          parameters[input] = defaults[input] !== undefined ?
              param_schema.default(defaults[input]) : param_schema;
      }
      addTool({
          name: `web_data_${id}`,
          description,
          parameters: z.object(parameters),
          execute: tool_fn(`web_data_${id}`, async(data, ctx)=>{
              let trigger_response = await axios({
                  url: 'https://api.brightdata.com/datasets/v3/trigger',
                  params: {dataset_id, include_errors: true},
                  method: 'POST',
                  data: [data],
                  headers: api_headers(),
              });
              if (!trigger_response.data?.snapshot_id)
                  throw new Error('No snapshot ID returned from request');
              let snapshot_id = trigger_response.data.snapshot_id;
              console.error(`[web_data_${id}] triggered collection with `
                  +`snapshot ID: ${snapshot_id}`);
              let max_attempts = 600;
              let attempts = 0;
              while (attempts < max_attempts)
              {
                  try {
                      if (ctx && ctx.reportProgress)
                      {
                          await ctx.reportProgress({
                              progress: attempts,
                              total: max_attempts,
                              message: `Polling for data (attempt `
                                  +`${attempts + 1}/${max_attempts})`,
                          });
                      }
                      let snapshot_response = await axios({
                          url: `https://api.brightdata.com/datasets/v3`
                              +`/snapshot/${snapshot_id}`,
                          params: {format: 'json'},
                          method: 'GET',
                          headers: api_headers(),
                      });
                      if (['running', 'building'].includes(snapshot_response.data?.status))
                      {
                          console.error(`[web_data_${id}] snapshot not ready, `
                              +`polling again (attempt `
                              +`${attempts + 1}/${max_attempts})`);
                          attempts++;
                          await new Promise(resolve=>setTimeout(resolve, 1000));
                          continue;
                      }
                      console.error(`[web_data_${id}] snapshot data received `
                          +`after ${attempts + 1} attempts`);
                      let result_data = JSON.stringify(snapshot_response.data);
                      return result_data;
                  } catch(e){
                      console.error(`[web_data_${id}] polling error: `
                          +`${e.message}`);
                      attempts++;
                      await new Promise(resolve=>setTimeout(resolve, 1000));
                  }
              }
              throw new Error(`Timeout after ${max_attempts} seconds waiting `
                  +`for data`);
          }),
      });
  }
  
  for (let tool of browser_tools)

• server.js:752-778 (helper)

  Wrapper function used for all tools, including web_data_google_play_store, that adds rate limiting, stats tracking, logging, and error handling around the core execute function.

  function tool_fn(name, fn){
      return async(data, ctx)=>{
          check_rate_limit();
          debug_stats.tool_calls[name] = debug_stats.tool_calls[name]||0;
          debug_stats.tool_calls[name]++;
          debug_stats.session_calls++;
          let ts = Date.now();
          console.error(`[%s] executing %s`, name, JSON.stringify(data));
          try { return await fn(data, ctx); }
          catch(e){
              if (e.response)
              {
                  console.error(`[%s] error %s %s: %s`, name, e.response.status,
                      e.response.statusText, e.response.data);
                  let message = e.response.data;
                  if (message?.length)
                      throw new Error(`HTTP ${e.response.status}: ${message}`);
              }
              else
                  console.error(`[%s] error %s`, name, e.stack);
              throw e;
          } finally {
              let dur = Date.now()-ts;
              console.error(`[%s] tool finished in %sms`, name, dur);
          }
      };
  }