Bright Data Web MCP

Instructions

Implementation Reference

• server.js:687-743 (handler)

  The core handler logic for the web_data_tiktok_shop tool. It triggers a data collection on BrightData's dataset API using the tiktok_shop dataset_id, polls the snapshot until ready or timeout, and returns the JSON results.

  execute: tool_fn(`web_data_${id}`, async(data, ctx)=>{
      let trigger_response = await axios({
          url: 'https://api.brightdata.com/datasets/v3/trigger',
          params: {dataset_id, include_errors: true},
          method: 'POST',
          data: [data],
          headers: api_headers(),
      });
      if (!trigger_response.data?.snapshot_id)
          throw new Error('No snapshot ID returned from request');
      let snapshot_id = trigger_response.data.snapshot_id;
      console.error(`[web_data_${id}] triggered collection with `
          +`snapshot ID: ${snapshot_id}`);
      let max_attempts = 600;
      let attempts = 0;
      while (attempts < max_attempts)
      {
          try {
              if (ctx && ctx.reportProgress)
              {
                  await ctx.reportProgress({
                      progress: attempts,
                      total: max_attempts,
                      message: `Polling for data (attempt `
                          +`${attempts + 1}/${max_attempts})`,
                  });
              }
              let snapshot_response = await axios({
                  url: `https://api.brightdata.com/datasets/v3`
                      +`/snapshot/${snapshot_id}`,
                  params: {format: 'json'},
                  method: 'GET',
                  headers: api_headers(),
              });
              if (['running', 'building'].includes(snapshot_response.data?.status))
              {
                  console.error(`[web_data_${id}] snapshot not ready, `
                      +`polling again (attempt `
                      +`${attempts + 1}/${max_attempts})`);
                  attempts++;
                  await new Promise(resolve=>setTimeout(resolve, 1000));
                  continue;
              }
              console.error(`[web_data_${id}] snapshot data received `
                  +`after ${attempts + 1} attempts`);
              let result_data = JSON.stringify(snapshot_response.data);
              return result_data;
          } catch(e){
              console.error(`[web_data_${id}] polling error: `
                  +`${e.message}`);
              attempts++;
              await new Promise(resolve=>setTimeout(resolve, 1000));
          }
      }
      throw new Error(`Timeout after ${max_attempts} seconds waiting `
          +`for data`);
  }),

• server.js:686-686 (schema)

  Zod schema definition for the tool parameters, dynamically built as z.object({url: z.string().url()}) based on the dataset inputs for tiktok_shop.

  parameters: z.object(parameters),

• server.js:683-744 (registration)

  The registration of the web_data_tiktok_shop tool via server.addTool(), dynamically generating name, description, parameters, and execute from the tiktok_shop dataset config when id === 'tiktok_shop'.

  addTool({
      name: `web_data_${id}`,
      description,
      parameters: z.object(parameters),
      execute: tool_fn(`web_data_${id}`, async(data, ctx)=>{
          let trigger_response = await axios({
              url: 'https://api.brightdata.com/datasets/v3/trigger',
              params: {dataset_id, include_errors: true},
              method: 'POST',
              data: [data],
              headers: api_headers(),
          });
          if (!trigger_response.data?.snapshot_id)
              throw new Error('No snapshot ID returned from request');
          let snapshot_id = trigger_response.data.snapshot_id;
          console.error(`[web_data_${id}] triggered collection with `
              +`snapshot ID: ${snapshot_id}`);
          let max_attempts = 600;
          let attempts = 0;
          while (attempts < max_attempts)
          {
              try {
                  if (ctx && ctx.reportProgress)
                  {
                      await ctx.reportProgress({
                          progress: attempts,
                          total: max_attempts,
                          message: `Polling for data (attempt `
                              +`${attempts + 1}/${max_attempts})`,
                      });
                  }
                  let snapshot_response = await axios({
                      url: `https://api.brightdata.com/datasets/v3`
                          +`/snapshot/${snapshot_id}`,
                      params: {format: 'json'},
                      method: 'GET',
                      headers: api_headers(),
                  });
                  if (['running', 'building'].includes(snapshot_response.data?.status))
                  {
                      console.error(`[web_data_${id}] snapshot not ready, `
                          +`polling again (attempt `
                          +`${attempts + 1}/${max_attempts})`);
                      attempts++;
                      await new Promise(resolve=>setTimeout(resolve, 1000));
                      continue;
                  }
                  console.error(`[web_data_${id}] snapshot data received `
                      +`after ${attempts + 1} attempts`);
                  let result_data = JSON.stringify(snapshot_response.data);
                  return result_data;
              } catch(e){
                  console.error(`[web_data_${id}] polling error: `
                      +`${e.message}`);
                  attempts++;
                  await new Promise(resolve=>setTimeout(resolve, 1000));
              }
          }
          throw new Error(`Timeout after ${max_attempts} seconds waiting `
              +`for data`);
      }),
  });

• server.js:524-531 (helper)

  Dataset configuration entry for tiktok_shop, providing the dataset_id and input schema (url) used to register and implement the web_data_tiktok_shop tool.

  id: 'tiktok_shop',
  dataset_id: 'gd_m45m1u911dsa4274pi',
  description: [
      'Quickly read structured Tiktok shop data.',
      'Requires a valid Tiktok shop product URL.',
      'This can be a cache lookup, so it can be more reliable than scraping',
  ].join('\n'),
  inputs: ['url'],