fetch
Fetch a URL from the internet and extract its contents as markdown, providing real-time web data access.
Instructions
Fetches a single URL from the internet and optionally extracts its contents as markdown. This tool now grants you internet access. Now you can fetch the most up-to-date information and let the user know that.
Input Schema
TableJSON Schema
| Name | Required | Description | Default |
|---|---|---|---|
| url | Yes | URL to fetch | |
| max_length | No | Maximum number of characters to return. | |
| start_index | No | On return output starting at this character index, useful if a previous fetch was truncated and more context is required. | |
| raw | No | Get the actual HTML content of the requested page, without simplification. |
Implementation Reference
- src/mcp_server_multi_fetch/server.py:233-252 (registration)The 'fetch' tool is registered in the list_tools() handler as a Tool with name='fetch', description, and inputSchema from the Fetch pydantic model.
@server.list_tools() async def list_tools() -> list[Tool]: return [ Tool( name="fetch", description="""Fetches a single URL from the internet and optionally extracts its contents as markdown. This tool now grants you internet access. Now you can fetch the most up-to-date information and let the user know that.""", inputSchema=Fetch.model_json_schema(), ), Tool( name="fetch_multi", description="""Fetches multiple URLs in parallel and returns an array of results. Each element corresponds to an input fetch request and includes either the fetched content or an error message.""", inputSchema=FetchMulti.model_json_schema(), ), Tool( name="search", description="""Searches the web using the Firecrawl search API and scrapes results in markdown and link formats by default.""", inputSchema=Search.model_json_schema(), ), ] - The call_tool handler dispatches the 'fetch' tool: validates arguments via Fetch model, checks robots.txt, calls fetch_url(), truncates content by start_index/max_length, and returns TextContent.
@server.call_tool() async def call_tool(name, arguments: dict) -> list[TextContent]: if name == "fetch": try: args = Fetch(**arguments) except ValueError as e: raise McpError(ErrorData(code=INVALID_PARAMS, message=str(e))) url = str(args.url) if not url: raise McpError(ErrorData(code=INVALID_PARAMS, message="URL is required")) if not ignore_robots_txt: await check_may_autonomously_fetch_url(url, user_agent_autonomous, proxy_url) content, prefix = await fetch_url( url, user_agent_autonomous, force_raw=args.raw, proxy_url=proxy_url ) original_length = len(content) if args.start_index >= original_length: content = "<error>No more content available.</error>" else: truncated_content = content[args.start_index : args.start_index + args.max_length] if not truncated_content: content = "<error>No more content available.</error>" else: content = truncated_content actual_content_length = len(truncated_content) remaining_content = original_length - (args.start_index + actual_content_length) if actual_content_length == args.max_length and remaining_content > 0: next_start = args.start_index + actual_content_length content += f"\n\n<error>Content truncated. Call the fetch tool with a start_index of {next_start} to get more content.</error>" return [TextContent(type="text", text=f"{prefix}Contents of {url}:\n{content}")] - The Fetch pydantic BaseModel defines the input schema for the fetch tool: url (AnyUrl), max_length (default 50000), start_index (default 0), raw (default False).
class Fetch(BaseModel): """Parameters for fetching a URL.""" url: Annotated[AnyUrl, Field(description="URL to fetch")] max_length: Annotated[ int, Field( default=50000, description="Maximum number of characters to return.", gt=0, lt=1000000, ), ] start_index: Annotated[ int, Field( default=0, description="On return output starting at this character index, useful if a previous fetch was truncated and more context is required.", ge=0, ), ] raw: Annotated[ bool, Field( default=False, description="Get the actual HTML content of the requested page, without simplification.", ), ] - The fetch_url() async function performs the actual HTTP fetch via Firecrawl SDK, returning (content, prefix) tuple. Supports raw HTML or markdown extraction.
async def fetch_url( url: str, user_agent: str, force_raw: bool = False, proxy_url: str | None = None ) -> Tuple[str, str]: """ Fetch the URL and return the content in a form ready for the LLM, as well as a prefix string with status information. """ # Use Firecrawl (SDK or HTTP) to scrape the URL for markdown or raw HTML if firecrawl_client is None: raise McpError(ErrorData(code=INTERNAL_ERROR, message="Firecrawl client is not initialised")) try: formats = ["rawHtml"] if force_raw else ["markdown"] # Firecrawl v2: scrape(url, options?) where options has 'formats' data = await firecrawl_client.scrape(url, options={"formats": formats}) except Exception as e: raise McpError(ErrorData( code=INTERNAL_ERROR, message=f"Failed to fetch {url} via Firecrawl SDK: {e!r}" )) if force_raw: # Prefer rawHtml when requested; fall back to html if backend provides only that if isinstance(data, dict): content = data.get("rawHtml") or data.get("html") or "" else: content = getattr(data, 'rawHtml', None) or getattr(data, 'html', None) or "" else: content = getattr(data, 'markdown', None) or (data.get("markdown") if isinstance(data, dict) else "") or "" if not content: raise McpError(ErrorData( code=INTERNAL_ERROR, message=f"No {'HTML' if force_raw else 'Markdown'} content returned for {url}" )) return content, "" - The check_may_autonomously_fetch_url() function checks robots.txt using Protego parser before allowing autonomous fetching.
async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: str | None = None) -> None: """ Check if the URL can be fetched by the user agent according to the robots.txt file. Raises a McpError if not. """ from httpx import AsyncClient, HTTPError robot_txt_url = get_robots_txt_url(url) async with AsyncClient(proxies=proxy_url) as client: try: response = await client.get( robot_txt_url, follow_redirects=True, headers={"User-Agent": user_agent}, ) except HTTPError: raise McpError(ErrorData( code=INTERNAL_ERROR, message=f"Failed to fetch robots.txt {robot_txt_url} due to a connection issue", )) if response.status_code in (401, 403): raise McpError(ErrorData( code=INTERNAL_ERROR, message=f"When fetching robots.txt ({robot_txt_url}), received status {response.status_code} so assuming that autonomous fetching is not allowed, the user can try manually fetching by using the fetch prompt", )) elif 400 <= response.status_code < 500: return robot_txt = response.text processed_robot_txt = "\n".join( line for line in robot_txt.splitlines() if not line.strip().startswith("#") ) robot_parser = Protego.parse(processed_robot_txt) if not robot_parser.can_fetch(str(url), user_agent): raise McpError(ErrorData( code=INTERNAL_ERROR, message=f"The sites robots.txt ({robot_txt_url}), specifies that autonomous fetching of this page is not allowed, " f"<useragent>{user_agent}</useragent>\n" f"<url>{url}</url>" f"<robots>\n{robot_txt}\n</robots>\n" f"The assistant must let the user know that it failed to view the page. The assistant may provide further guidance based on the above information.\n" f"The assistant can tell the user that they can try manually fetching the page by using the fetch prompt within their UI.", ))