mcp-server-webcrawl

{ "archive_path": "archive/1756357684.130171", "base_url": "pragmar.com/mcp-server-webcrawl", "basename": "", "bookmarked_date": "2025-08-28 05:08", "canonical": { "archive_org_path": "https://web.archive.org/web/pragmar.com/mcp-server-webcrawl", "dom_path": "output.html", "favicon_path": "favicon.ico", "git_path": "git/", "google_favicon_path": "https://www.google.com/s2/favicons?domain=pragmar.com", "headers_path": "headers.json", "index_path": "index.html", "media_path": "media/", "mercury_path": "mercury/content.html", "pdf_path": "output.pdf", "readability_path": "readability/content.html", "screenshot_path": "screenshot.png", "singlefile_path": "singlefile.html", "warc_path": "warc/", "wget_path": "pragmar.com/mcp-server-webcrawl/index.html" }, "domain": "pragmar.com", "extension": "", "hash": "CMDEG16YJESE4F438YA0", "history": { "archive_org": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--head", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.1 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.5.0 (x86_64-pc-linux-gnu)", "https://web.archive.org/save/https://pragmar.com/mcp-server-webcrawl/" ], "cmd_version": "curl 8.5.0 (x86_64-pc-linux-gnu)", "end_ts": "2025-08-28T05:16:12.905346+00:00", "index_texts": null, "output": "TimeoutExpired: Command '['curl', '--silent', '--location', '--compressed', '--head', '--max-time', '60', '--user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.1 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.5.0 (x86_64-pc-linux-gnu)', 'https://web.archive.org/save/https://pragmar.com/mcp-server-webcrawl/']' timed out after 60 seconds", "pwd": "/home/archivebox-data/pragmar/archive/1756357684.130171", "schema": "ArchiveResult", "start_ts": "2025-08-28T05:15:12.894318+00:00", "status": "failed" } ], "dom": [], "favicon": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--max-time", "60", "--output", "favicon.ico", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.1 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.5.0 (x86_64-pc-linux-gnu)", "https://www.google.com/s2/favicons?domain=pragmar.com" ], "cmd_version": "curl 8.5.0 (x86_64-pc-linux-gnu)", "end_ts": "2025-08-28T05:13:41.313213+00:00", "index_texts": null, "output": "favicon.ico", "pwd": "/home/archivebox-data/pragmar/archive/1756357684.130171", "schema": "ArchiveResult", "start_ts": "2025-08-28T05:13:41.152004+00:00", "status": "succeeded" } ], "git": [], "headers": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--head", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.1 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.5.0 (x86_64-pc-linux-gnu)", "https://pragmar.com/mcp-server-webcrawl/" ], "cmd_version": "curl 8.5.0 (x86_64-pc-linux-gnu)", "end_ts": "2025-08-28T05:13:41.467978+00:00", "index_texts": null, "output": "headers.json", "pwd": "/home/archivebox-data/pragmar/archive/1756357684.130171", "schema": "ArchiveResult", "start_ts": "2025-08-28T05:13:41.326476+00:00", "status": "succeeded" } ], "media": [ { "cmd": [ "yt-dlp", "--write-description", "--write-info-json", "--write-annotations", "--write-thumbnail", "--no-call-home", "--write-sub", "--write-auto-subs", "--convert-subs=srt", "--yes-playlist", "--continue", "--no-abort-on-error", "--ignore-errors", "--geo-bypass", "--add-metadata", "--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)", "https://pragmar.com/mcp-server-webcrawl/" ], "cmd_version": "2025.08.27", "end_ts": "2025-08-28T05:15:12.881648+00:00", "index_texts": [ "\ud83d\udce6 Get mcp-server-webcrawl: https://pragmar.com/mcp-server-webcrawl/\n\ud83d\udd78\ufe0f Get InterroBot: https://interro.bot/\n\ud83d\udc0d Get Python: https://www.python.org/downloads/\n\ud83d\udd2e Get Claude Desktop: https://claude.ai/download\n\nTimestamps:\n00:00 Introduction\n00:36 Requirements and installations\n00:50 Creating sample crawls with InterroBot\n01:38 Finding the InterroBot database path\n01:53 Installing MCP Server Web Crawl\n02:42 Configuring Claude Desktop MCP servers\n03:42 mcp-server-webcrawl demonstration\n\nThis tutorial sets up a mcp-server-webcrawl search connection from InterroBot web crawler and analyzer, to Claude Desktop. mcp-server-webcrawl handles search and retrieval of the archive, allowing Claude to filter and analyze web content archived with SiteOne.\n\nWhat you'll learn:\n- Installing mcp-server-webcrawl via pip\n- Configuring Cloud desktop for your MCP server\n- Using InterroBot to create local web archives\n- Enabling your LLM to search and retrieve content\n- Testing your setup with example.com and pragmar.com\n\nThis open-source tool is for developers, SEOs, web content administrators--anyone looking to leverage web content, work with archived web content, or create specialized knowledge bases for their LLMs.\n\n#mcp #InterroBot #ClaudeAI #WebCrawl #AITools #mcp-server-webcrawl", "\ud83d\udce6 Get mcp-server-webcrawl: https://pragmar.com/mcp-server-webcrawl/\n\ud83d\udd78\ufe0f Get SiteOne: https://crawler.siteone.io/\n\ud83d\udc0d Get Python: https://www.python.org/downloads/\n\ud83d\udd2e Get Claude Desktop: https://claude.ai/download\n\nTimestamps:\n00:00 Introduction\n00:30 Creating sample crawls with Site One\n01:47 Finding the Site One output directory\n01:59 Installing MCP Server Web Crawl\n02:54 Configuring Claude Desktop MCP servers\n04:18 mcp-server-webcrawl demonstration\n\nThis tutorial sets up a mcp-server-webcrawl search connection from SiteOne web archives to Claude Desktop. mcp-server-webcrawl acts as a search and retrieval interface, allowing Claude to filter and analyze web content archived with SiteOne.\n\nWhat you'll learn:\n- Installing mcp-server-webcrawl via pip\n- Configuring Cloud desktop for your MCP server\n- Using SiteOne to create local web archives\n- Enabling your LLM to search and retrieve content autonomously\n- Testing your setup with example.com and pragmar.com\n\nThis open-source tool is for developers, SEOs, web content administrators--anyone looking to leverage web content, work with archived web content, or create specialized knowledge bases for their LLMs.\n\n#mcp #SiteOne #ClaudeAI #WebCrawl #AITools #mcp-server-webcrawl", "\ud83d\udce6 Get mcp-server-webcrawl: https://pragmar.com/mcp-server-webcrawl/\n\ud83d\udd78\ufe0f Get WGet: macOS use brew, Windows use Ubuntu (see MS Store)\n\ud83d\udc0d Get Python: https://www.python.org/downloads/\n\ud83d\udd2e Get Claude Desktop: https://claude.ai/download\nTimestamps:\n00:00 Introduction\n00:38 Understanding WARC vs WGet mirror mode\n01:32 Installing Python and server requirements\n02:13 Installing MCP Server WebCrawl\n02:42 Creating WARC archives with WGet\n04:34 Configuring Claude Desktop MCP servers\n05:39 MCP Server WebCrawl demonstration\n\nThis tutorial sets up a mcp-server-webcrawl search connection from WARC (Web ARChive) files directly to Claude Desktop. mcp-server-webcrawl acts as a search and retrieval interface, allowing Claude to filter and analyze web content archived in the comprehensive WARC format, including status codes and headers.\nWhat you'll learn:\n\n- Installing mcp-server-webcrawl via pip\n- Configuring Claude Desktop for your MCP server\n- Understanding the differences between WARC and mirror archiving\n- Enabling your LLM to search and retrieve content autonomously\n- Testing your setup with example.com and pragmar.com\n\nThis open-source tool is for developers, SEOs, web content administrators--anyone looking to leverage web content, work with archived web content, or create specialized knowledge bases for their LLMs.\n#mcp #WARC #ClaudeAI #LLMResearch #WebCrawling #AITools #mcp-server-webcrawl", "\ud83d\udce6 Get mcp-server-webcrawl: https://pragmar.com/mcp-server-webcrawl/ \n\ud83d\udd78\ufe0f Get Katana: https://github.com/projectdiscovery/katana \n\ud83d\udc0d Get Python: https://www.python.org/downloads/ \n\ud83c\udf10 Get Golang: https://go.dev/ \n\ud83d\udd2e Get Claude Desktop: https://claude.ai/download \n\nTimestamps:\n00:00 Introduction\n00:30 Installing Python and mcp-server-webcrawl\n01:46 Installing Go and Katana\n03:03 Crawling example websites\n05:15 Configuring Claude Desktop MCP servers\n07:01 mcp-server-webcrawl demonstration\n\nThis tutorial sets up a mcp-server-webcrawl search connection from Katana web crawler and analyzer, to Claude Desktop. mcp-server-webcrawl handles search and retrieval of the archive, allowing Claude to filter and analyze web content archived with Katana.\n\nWhat you'll learn:\n- Installing mcp-server-webcrawl via pip\n- Configuring Claude desktop for your MCP server\n- Using Katana to create local web archives\n- Enabling your LLM to search and retrieve content\n- Testing your setup with example.com and pragmar.com\n\nThis open-source tool is for developers, SEOs, web content administrators--anyone looking to leverage web content, work with archived web content, or create specialized knowledge bases for their LLMs.\n\n#mcp #Katana #ClaudeAI #WebCrawl #AITools #mcp-server-webcrawl", "\ud83d\udce6 Get mcp-server-webcrawl: https://pragmar.com/mcp-server-webcrawl/\n\ud83d\udc0d Get Python: https://www.python.org/downloads/\n\ud83d\udd2e Get Claude Desktop: https://claude.ai/download\n\ud83d\udd78\ufe0fwget: use macOS use brew, Windows use Ubuntu (see MS Store)\n\nTimestamps:\n00:00 Introduction\n00:45 Configuring the MCP server\n02:38 Installing mcp-server-webcrawl\n03:22 Verifying MCP server setup\n04:03 Creating sample crawls with wget\n05:15 mcp-server-webcrawl demonstration\n\nThis tutorial sets up a mcp-server-webcrawl search connection from wget web archives, directly to Claude Desktop. mcp-server-webcrawl acts as a search and retrieval interface, allowing Claude to filter and analyze web content archived with wget.\n\nWhat you'll learn:\n- Installing mcp-server-webcrawl via pip\n- Configuring Cloud desktop for your MCP server\n- Using wget to create local web archives\n- Enabling your LLM to search and retrieve content autonomously\n- Testing your setup with example.com and pragmar.com\n\nThis open-source tool is for developers, SEOs, web content administrators--anyone looking to leverage web content, work with archived web content, or create specialized knowledge bases for their LLMs.\n\n#mcp #wget #ClaudeAI #WebCrawl #AITools #mcp-server-webcrawl", "AI search and retrieval for web crawlers", "1\n00:00:09,280 --> 00:00:12,560\nHey, this is Ben from InterroBot. In my other videos\n\n2\n00:00:12,561 --> 00:00:15,840\nI've gone over crawler and MCP configuration,\n\n3\n00:00:15,841 --> 00:00:18,120\nso I thought it was time to go a little bit deeper\n\n4\n00:00:18,121 --> 00:00:20,400\ninto the practical aspects of the server.\n\n5\n00:00:22,160 --> 00:00:25,160\nmcp-server-webcrawl is, at a fundamental level,\n\n6\n00:00:25,161 --> 00:00:28,160\na search interface.\n\n7\n00:00:28,161 --> 00:00:30,360\nYou use it to filter pages and return content\n\n8\n00:00:30,361 --> 00:00:32,560\nfor review or additional processing.\n\n9\n00:00:34,480 --> 00:00:37,760\nWhere Google is an all-of-web search interface\n\n10\n00:00:37,761 --> 00:00:41,040\ndesigned for humans, mcp-server-webcrawl is a website\n\n11\n00:00:41,041 --> 00:00:44,640\nlevel interface designed for AI language models.\n\n12\n00:00:44,641 --> 00:00:48,240\nYou can direct search as much as you want.\n\n13\n00:00:48,241 --> 00:00:51,920\nThe LLM is capable of searching and processing\n\n14\n00:00:51,921 --> 00:00:55,600\nautonomously. So you can, for example, give it a goal,\n\n15\n00:00:56,880 --> 00:00:58,000\nand just let it have at it.\n\n16\n00:01:02,320 --> 00:01:04,960\nIn fact, you can use the server as a custom site search,\n\n17\n00:01:04,961 --> 00:01:07,600\nand it's a good introduction to the server\n\n18\n00:01:07,601 --> 00:01:11,200\nto see it operate with results in full view.\n\n19\n00:01:11,201 --> 00:01:14,800\nSo say I've got this NASA SOHO site crawled,\n\n20\n00:01:14,801 --> 00:01:18,920\nand I'm looking for \"dopplergram\"\n\n21\n00:01:18,921 --> 00:01:23,040\nAnd if I get a noisy results at back,\n\n22\n00:01:24,320 --> 00:01:26,880\nand I don't want these Joint Observing Plan (JOP) pages\n\n23\n00:01:26,881 --> 00:01:29,440\nwith a little bit of Boolean search I can\n\n24\n00:01:29,441 --> 00:01:32,360\nfill through them out. And I can even tell Claude,\n\n25\n00:01:32,361 --> 00:01:35,280\nlook, I don't know the actual Boolean,\n\n26\n00:01:35,281 --> 00:01:39,200\nbut just get as close as you can to this.\n\n27\n00:01:39,201 --> 00:01:43,120\nAnd I haven't had bad results doing that.\n\n28\n00:01:43,121 --> 00:01:46,320\nLike, Claude is actually pretty good at Boolean queries.\n\n29\n00:01:46,321 --> 00:01:49,520\nIn this case, I'm searching \"dopplergram\"\n\n30\n00:01:49,521 --> 00:01:52,760\nCram, but I mean, you might be searching a product skew,\n\n31\n00:01:52,761 --> 00:01:56,000\nor someone's name, or a JavaScript\n\n32\n00:01:56,001 --> 00:01:59,200\npreference, because it's not just all about the HTML pages.\n\n33\n00:01:59,201 --> 00:02:02,400\nThere's a world of CSS,\n\n34\n00:02:02,401 --> 00:02:05,440\nJavaScript, you name it as just as valuable\n\n35\n00:02:05,441 --> 00:02:08,480\nto some of you out there as the HTML itself.\n\n36\n00:02:08,481 --> 00:02:12,000\nAnd of course, you see these search results as snippets,\n\n37\n00:02:12,001 --> 00:02:15,520\nbut snippets is just one perspective on the\n\n38\n00:02:15,521 --> 00:02:19,800\ncontent. You can alternatively retrieve results\n\n39\n00:02:19,801 --> 00:02:24,080\nas raw HTTP content, or page mark down,\n\n40\n00:02:25,120 --> 00:02:29,000\nor the latest features you can even zero in on XPath.\n\n41\n00:02:29,001 --> 00:02:32,880\nSo you can collect just what you need,\n\n42\n00:02:32,881 --> 00:02:37,000\nand really just manage that token load.\n\n43\n00:02:37,001 --> 00:02:41,120\nOften you don't need the HTTP body at all.\n\n44\n00:02:42,400 --> 00:02:45,160\nmcp-server-webcrawl encourages efficient token usage.\n\n45\n00:02:45,161 --> 00:02:47,920\nYou can control the context window\n\n46\n00:02:48,080 --> 00:02:50,080\nby allowing the query results to be as light\n\n47\n00:02:50,081 --> 00:02:52,080\nor as dense as you need.\n\n48\n00:02:52,081 --> 00:03:00,320\nI can find the largest images, CSS, or JavaScript.\n\n49\n00:03:09,920 --> 00:03:13,680\nLikewise, I could target 404 errors using HTTP status,\n\n50\n00:03:13,681 --> 00:03:17,440\nor a particular character set using headers.\n\n51\n00:03:17,441 --> 00:03:20,760\nOr mix and match field level and full text search.\n\n52\n00:03:20,761 --> 00:03:24,080\nThe Boolean search foundation of mcp-server-webcrawl\n\n53\n00:03:24,081 --> 00:03:27,120\nis solid. So you can search crawls with high precision,\n\n54\n00:03:27,121 --> 00:03:30,160\nand you've seen the snippets\n\n55\n00:03:30,161 --> 00:03:32,760\npresent in clear view. But in most cases,\n\n56\n00:03:32,761 --> 00:03:35,360\nsearch results are processed in the background.\n\n57\n00:03:36,560 --> 00:03:39,240\nYou might expand the MCP interaction to verify\n\n58\n00:03:39,241 --> 00:03:41,920\na search query or the results return,\n\n59\n00:03:41,921 --> 00:03:45,320\nbut if the AI is searching predictably and it's giving you\n\n60\n00:03:45,321 --> 00:03:48,720\nwhat you need, this isn't necessary.\n\n61\n00:03:48,721 --> 00:04:01,460\nSo the server provides both search and content retrieval,\n\n62\n00:04:01,461 --> 00:04:11,200\nalong with these content extras to\n\n63\n00:04:11,201 --> 00:04:15,240\ngive a lot of degrees of freedom to get what you're after.\n\n64\n00:04:15,241 --> 00:04:19,280\nYou can embed these crawl queries in complex\n\n65\n00:04:19,281 --> 00:04:21,640\nprompts. You can write these yourself,\n\n66\n00:04:21,641 --> 00:04:24,000\nor if you just want to get your feet wet,\n\n67\n00:04:24,001 --> 00:04:27,560\nthere are examples that I wrote in the GitHub repo.\n\n68\n00:04:27,561 --> 00:04:31,120\nSo like the site selection, all the interaction\n\n69\n00:04:31,121 --> 00:04:35,200\nbecomes guided. It's an efficient way to script a test\n\n70\n00:04:35,201 --> 00:04:39,280\nor report, because the queries in the\n\n71\n00:04:39,281 --> 00:04:44,360\nlogic can be expressed in broad strokes\n\n72\n00:04:44,361 --> 00:04:49,440\nusing plain language. If you haven't checked these prompts\n\n73\n00:04:49,441 --> 00:04:53,160\nout, there worth a look. I develop InterroBot crawler.\n\n74\n00:04:53,161 --> 00:04:56,880\nI wrote a plug-in system in a series of plugins.\n\n75\n00:04:56,881 --> 00:05:00,960\nDon't get me wrong. The InterroBot core plugins are more\n\n76\n00:05:00,961 --> 00:05:05,040\nadvanced, more deterministic, and the UI is a\n\n77\n00:05:05,041 --> 00:05:09,120\nlot more polished than these prompts, but they took\n\n78\n00:05:09,121 --> 00:05:13,200\nso much longer to write. Like two, three months\n\n79\n00:05:13,201 --> 00:05:16,760\nfor the InterroBot plugins compared to like a couple\n\n80\n00:05:16,761 --> 00:05:20,320\nof days for the prompts. And the prompts get you\n\n81\n00:05:20,321 --> 00:05:25,720\n80% there. The LLM can be glitchy on occasion,\n\n82\n00:05:25,721 --> 00:05:31,120\nand presentation is chat. It's nothing\n\n83\n00:05:31,121 --> 00:05:34,000\nnot right home about, but the fact that it can do\n\n84\n00:05:34,001 --> 00:05:36,800\na reasonable job of reviewing any number of\n\n85\n00:05:36,801 --> 00:05:42,860\nwebsite issue classes is notable.\n\n86\n00:05:42,861 --> 00:05:49,920\nWhat is that last 20% worth? Does it outweigh the\n\n87\n00:05:49,921 --> 00:05:53,640\nconvenience of a direct API? I'm still trying to figure\n\n88\n00:05:53,641 --> 00:05:57,360\nthat out. But let me know what you think in the\n\n89\n00:05:57,361 --> 00:06:01,720\ncomments. mcp-server-webcrawl is a search tool\n\n90\n00:06:01,721 --> 00:06:06,080\nfor web data integrates with your LLM.\n\n91\n00:06:06,081 --> 00:06:18,560\nIt's all I have for you today. Stay tuned.", "1\n00:00:00,700 --> 00:00:07,110\n\n[Music]\n\n2\n00:00:07,110 --> 00:00:07,120\n\n \n\n3\n00:00:07,120 --> 00:00:09,190\n\nHey, this is Ben from Interobot and\n\n4\n00:00:09,190 --> 00:00:09,200\nHey, this is Ben from Interobot and\n \n\n5\n00:00:09,200 --> 00:00:11,549\nHey, this is Ben from Interobot and\ntoday I'm going to talk about MCP server\n\n6\n00:00:11,549 --> 00:00:11,559\ntoday I'm going to talk about MCP server\n \n\n7\n00:00:11,559 --> 00:00:14,430\ntoday I'm going to talk about MCP server\nwebcrawl along with site one\n\n8\n00:00:14,430 --> 00:00:14,440\nwebcrawl along with site one\n \n\n9\n00:00:14,440 --> 00:00:16,550\nwebcrawl along with site one\ncrawler. What we're going to do is we're\n\n10\n00:00:16,550 --> 00:00:16,560\ncrawler. What we're going to do is we're\n \n\n11\n00:00:16,560 --> 00:00:19,630\ncrawler. What we're going to do is we're\ngoing to use cloud desktop as our uh MCP\n\n12\n00:00:19,630 --> 00:00:19,640\ngoing to use cloud desktop as our uh MCP\n \n\n13\n00:00:19,640 --> 00:00:22,390\ngoing to use cloud desktop as our uh MCP\nclient and then we're going to set up an\n\n14\n00:00:22,390 --> 00:00:22,400\nclient and then we're going to set up an\n \n\n15\n00:00:22,400 --> 00:00:25,029\nclient and then we're going to set up an\nMCP server web crawl and we're going to\n\n16\n00:00:25,029 --> 00:00:25,039\nMCP server web crawl and we're going to\n \n\n17\n00:00:25,039 --> 00:00:27,670\nMCP server web crawl and we're going to\nattach it to the crawls that we execute\n\n18\n00:00:27,670 --> 00:00:27,680\nattach it to the crawls that we execute\n \n\n19\n00:00:27,680 --> 00:00:29,830\nattach it to the crawls that we execute\nwithin site one. This is the directory\n\n20\n00:00:29,830 --> 00:00:29,840\nwithin site one. This is the directory\n \n\n21\n00:00:29,840 --> 00:00:32,069\nwithin site one. This is the directory\nwhere site one is storing uh the\n\n22\n00:00:32,069 --> 00:00:32,079\nwhere site one is storing uh the\n \n\n23\n00:00:32,079 --> 00:00:35,950\nwhere site one is storing uh the\ngenerated offline web\n\n24\n00:00:35,950 --> 00:00:35,960\n\n \n\n25\n00:00:35,960 --> 00:00:39,869\n\ncontent. And so I'm just going to\n\n26\n00:00:39,869 --> 00:00:39,879\ncontent. And so I'm just going to\n \n\n27\n00:00:39,879 --> 00:00:43,950\ncontent. And so I'm just going to\ncrawl\n\n28\n00:00:43,950 --> 00:00:43,960\n\n \n\n29\n00:00:43,960 --> 00:00:46,549\n\nexample.com and just make sure you have\n\n30\n00:00:46,549 --> 00:00:46,559\nexample.com and just make sure you have\n \n\n31\n00:00:46,559 --> 00:00:48,389\nexample.com and just make sure you have\nthis checked. This is the uh the\n\n32\n00:00:48,389 --> 00:00:48,399\nthis checked. This is the uh the\n \n\n33\n00:00:48,399 --> 00:00:50,590\nthis checked. This is the uh the\nimportant\n\n34\n00:00:50,590 --> 00:00:50,600\nimportant\n \n\n35\n00:00:50,600 --> 00:00:57,510\nimportant\noption. We're going to grab that.\n\n36\n00:00:57,510 --> 00:00:57,520\n\n \n\n37\n00:00:57,520 --> 00:00:59,110\n\nIt's going to do its thing. It's going\n\n38\n00:00:59,110 --> 00:00:59,120\nIt's going to do its thing. It's going\n \n\n39\n00:00:59,120 --> 00:01:01,349\nIt's going to do its thing. It's going\nto generate some other um reports and\n\n40\n00:01:01,349 --> 00:01:01,359\nto generate some other um reports and\n \n\n41\n00:01:01,359 --> 00:01:02,869\nto generate some other um reports and\nstuff that you might want to look at,\n\n42\n00:01:02,869 --> 00:01:02,879\nstuff that you might want to look at,\n \n\n43\n00:01:02,879 --> 00:01:05,509\nstuff that you might want to look at,\nbut what we're really after is that\n\n44\n00:01:05,509 --> 00:01:05,519\nbut what we're really after is that\n \n\n45\n00:01:05,519 --> 00:01:09,950\nbut what we're really after is that\noffline content that it just\n\n46\n00:01:09,950 --> 00:01:09,960\n\n \n\n47\n00:01:09,960 --> 00:01:11,990\n\ngrabbed. So, after that, I'm going to\n\n48\n00:01:11,990 --> 00:01:12,000\ngrabbed. So, after that, I'm going to\n \n\n49\n00:01:12,000 --> 00:01:17,310\ngrabbed. So, after that, I'm going to\ngrab another website,\n\n50\n00:01:17,310 --> 00:01:17,320\n\n \n\n51\n00:01:17,320 --> 00:01:19,950\n\nPragmar. And again, all of this is going\n\n52\n00:01:19,950 --> 00:01:19,960\nPragmar. And again, all of this is going\n \n\n53\n00:01:19,960 --> 00:01:25,590\nPragmar. And again, all of this is going\ninto these directories here.\n\n54\n00:01:25,590 --> 00:01:25,600\n\n \n\n55\n00:01:25,600 --> 00:01:29,270\n\nthe um when we configure our MCP server,\n\n56\n00:01:29,270 --> 00:01:29,280\nthe um when we configure our MCP server,\n \n\n57\n00:01:29,280 --> 00:01:45,550\nthe um when we configure our MCP server,\nwe'll use this path to point to the\n\n58\n00:01:45,550 --> 00:01:45,560\n\n \n\n59\n00:01:45,560 --> 00:01:47,830\n\ndirectory. All right. So, we got our\n\n60\n00:01:47,830 --> 00:01:47,840\ndirectory. All right. So, we got our\n \n\n61\n00:01:47,840 --> 00:01:49,590\ndirectory. All right. So, we got our\nreports again. If you want you can look\n\n62\n00:01:49,590 --> 00:01:49,600\nreports again. If you want you can look\n \n\n63\n00:01:49,600 --> 00:01:51,510\nreports again. If you want you can look\nat those.\n\n64\n00:01:51,510 --> 00:01:51,520\nat those.\n \n\n65\n00:01:51,520 --> 00:01:53,350\nat those.\nuh also can see the options that were\n\n66\n00:01:53,350 --> 00:01:53,360\nuh also can see the options that were\n \n\n67\n00:01:53,360 --> 00:01:56,710\nuh also can see the options that were\nused in the crawl. But other than that,\n\n68\n00:01:56,710 --> 00:01:56,720\nused in the crawl. But other than that,\n \n\n69\n00:01:56,720 --> 00:01:58,709\nused in the crawl. But other than that,\nwe're done with site one. We have all\n\n70\n00:01:58,709 --> 00:01:58,719\nwe're done with site one. We have all\n \n\n71\n00:01:58,719 --> 00:02:01,990\nwe're done with site one. We have all\nour files that we need and we're going\n\n72\n00:02:01,990 --> 00:02:02,000\nour files that we need and we're going\n \n\n73\n00:02:02,000 --> 00:02:04,950\nour files that we need and we're going\nto take a look at Claude and MCP server\n\n74\n00:02:04,950 --> 00:02:04,960\nto take a look at Claude and MCP server\n \n\n75\n00:02:04,960 --> 00:02:06,670\nto take a look at Claude and MCP server\nweb crawl\n\n76\n00:02:06,670 --> 00:02:06,680\nweb crawl\n \n\n77\n00:02:06,680 --> 00:02:09,029\nweb crawl\nnow. Okay. So, at this point, I'm going\n\n78\n00:02:09,029 --> 00:02:09,039\nnow. Okay. So, at this point, I'm going\n \n\n79\n00:02:09,039 --> 00:02:11,029\nnow. Okay. So, at this point, I'm going\nto assume that you have Python installed\n\n80\n00:02:11,029 --> 00:02:11,039\nto assume that you have Python installed\n \n\n81\n00:02:11,039 --> 00:02:14,110\nto assume that you have Python installed\nand also a MCP client like Claude\n\n82\n00:02:14,110 --> 00:02:14,120\nand also a MCP client like Claude\n \n\n83\n00:02:14,120 --> 00:02:18,589\nand also a MCP client like Claude\ndesktop. For this demo, I'm using\n\n84\n00:02:18,589 --> 00:02:18,599\ndesktop. For this demo, I'm using\n \n\n85\n00:02:18,599 --> 00:02:21,270\ndesktop. For this demo, I'm using\nClaude. Once you have Python, that means\n\n86\n00:02:21,270 --> 00:02:21,280\nClaude. Once you have Python, that means\n \n\n87\n00:02:21,280 --> 00:02:25,070\nClaude. Once you have Python, that means\nyou have\n\n88\n00:02:25,070 --> 00:02:25,080\n\n \n\n89\n00:02:25,080 --> 00:02:31,229\n\npip and you'd install pip mcp server\n\n90\n00:02:31,229 --> 00:02:31,239\n\n \n\n91\n00:02:31,239 --> 00:02:33,430\n\nwebcwl. At this point, if you want, you\n\n92\n00:02:33,430 --> 00:02:33,440\nwebcwl. At this point, if you want, you\n \n\n93\n00:02:33,440 --> 00:02:40,910\nwebcwl. At this point, if you want, you\ncan just check and see if it's\n\n94\n00:02:40,910 --> 00:02:40,920\n\n \n\n95\n00:02:40,920 --> 00:02:44,750\n\nthere. And it is.\n\n96\n00:02:44,750 --> 00:02:44,760\nthere. And it is.\n \n\n97\n00:02:44,760 --> 00:02:48,229\nthere. And it is.\nUm, so we've got our server and we've\n\n98\n00:02:48,229 --> 00:02:48,239\nUm, so we've got our server and we've\n \n\n99\n00:02:48,239 --> 00:02:52,190\nUm, so we've got our server and we've\ngot our crawls. It's time to work with\n\n100\n00:02:52,190 --> 00:02:52,200\ngot our crawls. It's time to work with\n \n\n101\n00:02:52,200 --> 00:02:54,710\ngot our crawls. It's time to work with\nClaude. If we come over to Claude,\n\n102\n00:02:54,710 --> 00:02:54,720\nClaude. If we come over to Claude,\n \n\n103\n00:02:54,720 --> 00:02:56,470\nClaude. If we come over to Claude,\nyou'll notice that there there isn't a\n\n104\n00:02:56,470 --> 00:02:56,480\nyou'll notice that there there isn't a\n \n\n105\n00:02:56,480 --> 00:02:59,509\nyou'll notice that there there isn't a\nhammer button here. And uh it's just\n\n106\n00:02:59,509 --> 00:02:59,519\nhammer button here. And uh it's just\n \n\n107\n00:02:59,519 --> 00:03:01,550\nhammer button here. And uh it's just\nthese two buttons. This is like\n\n108\n00:03:01,550 --> 00:03:01,560\nthese two buttons. This is like\n \n\n109\n00:03:01,560 --> 00:03:03,790\nthese two buttons. This is like\nuploads. This is\n\n110\n00:03:03,790 --> 00:03:03,800\nuploads. This is\n \n\n111\n00:03:03,800 --> 00:03:06,229\nuploads. This is\noptions. There is no hammer. And the\n\n112\n00:03:06,229 --> 00:03:06,239\noptions. There is no hammer. And the\n \n\n113\n00:03:06,239 --> 00:03:12,030\noptions. There is no hammer. And the\nreason is you need to go up\n\n114\n00:03:12,030 --> 00:03:12,040\n\n \n\n115\n00:03:12,040 --> 00:03:17,149\n\nhere and you need to edit your\n\n116\n00:03:17,149 --> 00:03:17,159\n\n \n\n117\n00:03:17,159 --> 00:03:20,070\n\nconfig. So if we look back at our MCP\n\n118\n00:03:20,070 --> 00:03:20,080\nconfig. So if we look back at our MCP\n \n\n119\n00:03:20,080 --> 00:03:21,990\nconfig. So if we look back at our MCP\nwebsite, we already have a site one\n\n120\n00:03:21,990 --> 00:03:22,000\nwebsite, we already have a site one\n \n\n121\n00:03:22,000 --> 00:03:24,309\nwebsite, we already have a site one\noption here and this is the config that\n\n122\n00:03:24,309 --> 00:03:24,319\noption here and this is the config that\n \n\n123\n00:03:24,319 --> 00:03:26,470\noption here and this is the config that\nwe're going to use, but we're going to\n\n124\n00:03:26,470 --> 00:03:26,480\nwe're going to use, but we're going to\n \n\n125\n00:03:26,480 --> 00:03:43,589\nwe're going to use, but we're going to\nhave to modify this path right here.\n\n126\n00:03:43,589 --> 00:03:43,599\n\n \n\n127\n00:03:43,599 --> 00:03:46,229\n\nYou can see that I've modified the path\n\n128\n00:03:46,229 --> 00:03:46,239\nYou can see that I've modified the path\n \n\n129\n00:03:46,239 --> 00:03:48,630\nYou can see that I've modified the path\nto reflect where site one is dropping\n\n130\n00:03:48,630 --> 00:03:48,640\nto reflect where site one is dropping\n \n\n131\n00:03:48,640 --> 00:03:51,949\nto reflect where site one is dropping\nthe uh\n\n132\n00:03:51,949 --> 00:03:51,959\n\n \n\n133\n00:03:51,959 --> 00:03:56,309\n\narchives. All right. Very good.\n\n134\n00:03:56,309 --> 00:03:56,319\narchives. All right. Very good.\n \n\n135\n00:03:56,319 --> 00:03:58,869\narchives. All right. Very good.\nSo let's save this. And then when we\n\n136\n00:03:58,869 --> 00:03:58,879\nSo let's save this. And then when we\n \n\n137\n00:03:58,879 --> 00:04:02,509\nSo let's save this. And then when we\ncome back to claude, it's very\n\n138\n00:04:02,509 --> 00:04:02,519\ncome back to claude, it's very\n \n\n139\n00:04:02,519 --> 00:04:04,949\ncome back to claude, it's very\nimportant. You need to fully exit. If\n\n140\n00:04:04,949 --> 00:04:04,959\nimportant. You need to fully exit. If\n \n\n141\n00:04:04,959 --> 00:04:07,949\nimportant. You need to fully exit. If\nyou don't fully\n\n142\n00:04:07,949 --> 00:04:07,959\n\n \n\n143\n00:04:07,959 --> 00:04:10,630\n\nexit, the new configuration won't get\n\n144\n00:04:10,630 --> 00:04:10,640\nexit, the new configuration won't get\n \n\n145\n00:04:10,640 --> 00:04:14,789\nexit, the new configuration won't get\nloaded up. All right. So at this point,\n\n146\n00:04:14,789 --> 00:04:14,799\nloaded up. All right. So at this point,\n \n\n147\n00:04:14,799 --> 00:04:17,670\nloaded up. All right. So at this point,\nwe have our configuration all set up.\n\n148\n00:04:17,670 --> 00:04:17,680\nwe have our configuration all set up.\n \n\n149\n00:04:17,680 --> 00:04:21,030\nwe have our configuration all set up.\nWe've reloaded Claude and lo and behold,\n\n150\n00:04:21,030 --> 00:04:21,040\nWe've reloaded Claude and lo and behold,\n \n\n151\n00:04:21,040 --> 00:04:24,990\nWe've reloaded Claude and lo and behold,\nwe've got a hammer and we've got\n\n152\n00:04:24,990 --> 00:04:25,000\nwe've got a hammer and we've got\n \n\n153\n00:04:25,000 --> 00:04:29,350\nwe've got a hammer and we've got\nour two MCP tools. Just a heads up, the\n\n154\n00:04:29,350 --> 00:04:29,360\nour two MCP tools. Just a heads up, the\n \n\n155\n00:04:29,360 --> 00:04:31,990\nour two MCP tools. Just a heads up, the\ncloud desktop uh interface has changed\n\n156\n00:04:31,990 --> 00:04:32,000\ncloud desktop uh interface has changed\n \n\n157\n00:04:32,000 --> 00:04:36,629\ncloud desktop uh interface has changed\nsince I recorded the uh demo. And MCP\n\n158\n00:04:36,629 --> 00:04:36,639\nsince I recorded the uh demo. And MCP\n \n\n159\n00:04:36,639 --> 00:04:39,430\nsince I recorded the uh demo. And MCP\nservers are now under search and tools.\n\n160\n00:04:39,430 --> 00:04:39,440\nservers are now under search and tools.\n \n\n161\n00:04:39,440 --> 00:04:41,909\nservers are now under search and tools.\nSo like right here, they can be toggled\n\n162\n00:04:41,909 --> 00:04:41,919\nSo like right here, they can be toggled\n \n\n163\n00:04:41,919 --> 00:04:43,830\nSo like right here, they can be toggled\non and off.\n\n164\n00:04:43,830 --> 00:04:43,840\non and off.\n \n\n165\n00:04:43,840 --> 00:04:45,990\non and off.\nBut the button that I'm referring to,\n\n166\n00:04:45,990 --> 00:04:46,000\nBut the button that I'm referring to,\n \n\n167\n00:04:46,000 --> 00:04:47,510\nBut the button that I'm referring to,\nthe hammer button that I'm getting\n\n168\n00:04:47,510 --> 00:04:47,520\nthe hammer button that I'm getting\n \n\n169\n00:04:47,520 --> 00:04:50,550\nthe hammer button that I'm getting\nexcited about perhaps is no longer part\n\n170\n00:04:50,550 --> 00:04:50,560\nexcited about perhaps is no longer part\n \n\n171\n00:04:50,560 --> 00:04:54,710\nexcited about perhaps is no longer part\nof the interface. So, as of May 25, uh\n\n172\n00:04:54,710 --> 00:04:54,720\nof the interface. So, as of May 25, uh\n \n\n173\n00:04:54,720 --> 00:04:56,390\nof the interface. So, as of May 25, uh\nthis is how it works. Things are in\n\n174\n00:04:56,390 --> 00:04:56,400\nthis is how it works. Things are in\n \n\n175\n00:04:56,400 --> 00:04:58,870\nthis is how it works. Things are in\nflux, so it could change again, but uh\n\n176\n00:04:58,870 --> 00:04:58,880\nflux, so it could change again, but uh\n \n\n177\n00:04:58,880 --> 00:05:00,830\nflux, so it could change again, but uh\njust wanted to give you that heads\n\n178\n00:05:00,830 --> 00:05:00,840\njust wanted to give you that heads\n \n\n179\n00:05:00,840 --> 00:05:03,990\njust wanted to give you that heads\nup and we're good to go. So, I'm going\n\n180\n00:05:03,990 --> 00:05:04,000\nup and we're good to go. So, I'm going\n \n\n181\n00:05:04,000 --> 00:05:12,830\nup and we're good to go. So, I'm going\nto ask Cloud to list\n\n182\n00:05:12,830 --> 00:05:12,840\n\n \n\n183\n00:05:12,840 --> 00:05:15,990\n\nsites and um hopefully that comes back\n\n184\n00:05:15,990 --> 00:05:16,000\nsites and um hopefully that comes back\n \n\n185\n00:05:16,000 --> 00:05:22,590\nsites and um hopefully that comes back\nas Pragmar an\n\n186\n00:05:22,590 --> 00:05:22,600\n\n \n\n187\n00:05:22,600 --> 00:05:26,230\n\nexample. That's perfect. Okay, so we've\n\n188\n00:05:26,230 --> 00:05:26,240\nexample. That's perfect. Okay, so we've\n \n\n189\n00:05:26,240 --> 00:05:28,230\nexample. That's perfect. Okay, so we've\ngot our two sites. Now, we can search\n\n190\n00:05:28,230 --> 00:05:28,240\ngot our two sites. Now, we can search\n \n\n191\n00:05:28,240 --> 00:05:30,790\ngot our two sites. Now, we can search\nwithin those. Say I'm interested in\n\n192\n00:05:30,790 --> 00:05:30,800\nwithin those. Say I'm interested in\n \n\n193\n00:05:30,800 --> 00:05:33,350\nwithin those. Say I'm interested in\nAbstat which is a software product that\n\n194\n00:05:33,350 --> 00:05:33,360\nAbstat which is a software product that\n \n\n195\n00:05:33,360 --> 00:05:44,430\nAbstat which is a software product that\nI know about on Pragmar.\n\n196\n00:05:44,430 --> 00:05:44,440\n\n \n\n197\n00:05:44,440 --> 00:05:48,590\n\nSo So it's going to launch a permission\n\n198\n00:05:48,590 --> 00:05:48,600\nSo So it's going to launch a permission\n \n\n199\n00:05:48,600 --> 00:05:51,270\nSo So it's going to launch a permission\nhere. This one first one was for sites\n\n200\n00:05:51,270 --> 00:05:51,280\nhere. This one first one was for sites\n \n\n201\n00:05:51,280 --> 00:05:57,870\nhere. This one first one was for sites\nthat was for search.\n\n202\n00:05:57,870 --> 00:05:57,880\n\n \n\n203\n00:05:57,880 --> 00:06:00,230\n\nSo, shows you what's going on behind the\n\n204\n00:06:00,230 --> 00:06:00,240\nSo, shows you what's going on behind the\n \n\n205\n00:06:00,240 --> 00:06:09,950\nSo, shows you what's going on behind the\nscenes if you want to\n\n206\n00:06:09,950 --> 00:06:09,960\n\n \n\n207\n00:06:09,960 --> 00:06:18,270\n\nlook. So, not\n\n208\n00:06:18,270 --> 00:06:18,280\n\n \n\n209\n00:06:18,280 --> 00:06:22,150\n\nbad. Anyway, uh that's the tool. It's\n\n210\n00:06:22,150 --> 00:06:22,160\nbad. Anyway, uh that's the tool. It's\n \n\n211\n00:06:22,160 --> 00:06:25,749\nbad. Anyway, uh that's the tool. It's\ncalled MCP server web crawl and uh it\n\n212\n00:06:25,749 --> 00:06:25,759\ncalled MCP server web crawl and uh it\n \n\n213\n00:06:25,759 --> 00:06:27,430\ncalled MCP server web crawl and uh it\nworks with a bunch of crawlers, not just\n\n214\n00:06:27,430 --> 00:06:27,440\nworks with a bunch of crawlers, not just\n \n\n215\n00:06:27,440 --> 00:06:29,990\nworks with a bunch of crawlers, not just\nsite one. Check it out. Yeah. Have a\n\n216\n00:06:29,990 --> 00:06:30,000\nsite one. Check it out. Yeah. Have a\n \n\n217\n00:06:30,000 --> 00:06:34,280\nsite one. Check it out. Yeah. Have a\ngreat day. Later on.", "1\n00:00:00,700 --> 00:00:05,990\n\n[Music]\n\n2\n00:00:05,990 --> 00:00:06,000\n[Music]\n \n\n3\n00:00:06,000 --> 00:00:07,990\n[Music]\nHey, this is Ben from Interabot and\n\n4\n00:00:07,990 --> 00:00:08,000\nHey, this is Ben from Interabot and\n \n\n5\n00:00:08,000 --> 00:00:11,589\nHey, this is Ben from Interabot and\ntoday I am going over MCP server webcwl\n\n6\n00:00:11,589 --> 00:00:11,599\ntoday I am going over MCP server webcwl\n \n\n7\n00:00:11,599 --> 00:00:14,070\ntoday I am going over MCP server webcwl\nusing interobot. Just so happens I am\n\n8\n00:00:14,070 --> 00:00:14,080\nusing interobot. Just so happens I am\n \n\n9\n00:00:14,080 --> 00:00:16,510\nusing interobot. Just so happens I am\nthe developer of both of these\n\n10\n00:00:16,510 --> 00:00:16,520\nthe developer of both of these\n \n\n11\n00:00:16,520 --> 00:00:19,029\nthe developer of both of these\nprojects. Interabot is a commercial\n\n12\n00:00:19,029 --> 00:00:19,039\nprojects. Interabot is a commercial\n \n\n13\n00:00:19,039 --> 00:00:21,750\nprojects. Interabot is a commercial\ncrawler and MCP server webcwl is an\n\n14\n00:00:21,750 --> 00:00:21,760\ncrawler and MCP server webcwl is an\n \n\n15\n00:00:21,760 --> 00:00:24,109\ncrawler and MCP server webcwl is an\nopen- source LLM\n\n16\n00:00:24,109 --> 00:00:24,119\nopen- source LLM\n \n\n17\n00:00:24,119 --> 00:00:26,470\nopen- source LLM\nconnector. So these things work together\n\n18\n00:00:26,470 --> 00:00:26,480\nconnector. So these things work together\n \n\n19\n00:00:26,480 --> 00:00:29,029\nconnector. So these things work together\nlike peanut butter and jelly.\n\n20\n00:00:29,029 --> 00:00:29,039\nlike peanut butter and jelly.\n \n\n21\n00:00:29,039 --> 00:00:30,230\nlike peanut butter and jelly.\nI'm just going to go over the\n\n22\n00:00:30,230 --> 00:00:30,240\nI'm just going to go over the\n \n\n23\n00:00:30,240 --> 00:00:32,229\nI'm just going to go over the\nrequirements so you get a sense of what\n\n24\n00:00:32,229 --> 00:00:32,239\nrequirements so you get a sense of what\n \n\n25\n00:00:32,239 --> 00:00:34,630\nrequirements so you get a sense of what\nyou need. You're going to need Python\n\n26\n00:00:34,630 --> 00:00:34,640\nyou need. You're going to need Python\n \n\n27\n00:00:34,640 --> 00:00:38,510\nyou need. You're going to need Python\n310 plus. Python you can get at\n\n28\n00:00:38,510 --> 00:00:38,520\n310 plus. Python you can get at\n \n\n29\n00:00:38,520 --> 00:00:41,310\n310 plus. Python you can get at\npython.org. You're going to need\n\n30\n00:00:41,310 --> 00:00:41,320\npython.org. You're going to need\n \n\n31\n00:00:41,320 --> 00:00:43,190\npython.org. You're going to need\nInterabot. I'm going to assume that\n\n32\n00:00:43,190 --> 00:00:43,200\nInterabot. I'm going to assume that\n \n\n33\n00:00:43,200 --> 00:00:44,830\nInterabot. I'm going to assume that\nyou've installed\n\n34\n00:00:44,830 --> 00:00:44,840\nyou've installed\n \n\n35\n00:00:44,840 --> 00:00:47,470\nyou've installed\nInterabot. And we'll head over\n\n36\n00:00:47,470 --> 00:00:47,480\nInterabot. And we'll head over\n \n\n37\n00:00:47,480 --> 00:00:50,229\nInterabot. And we'll head over\nthere. So, this is the screen that you'd\n\n38\n00:00:50,229 --> 00:00:50,239\nthere. So, this is the screen that you'd\n \n\n39\n00:00:50,239 --> 00:00:53,189\nthere. So, this is the screen that you'd\nsee first time you open Interobot. There\n\n40\n00:00:53,189 --> 00:00:53,199\nsee first time you open Interobot. There\n \n\n41\n00:00:53,199 --> 00:00:55,350\nsee first time you open Interobot. There\nare no projects yet. I'm just going to\n\n42\n00:00:55,350 --> 00:00:55,360\nare no projects yet. I'm just going to\n \n\n43\n00:00:55,360 --> 00:01:01,869\nare no projects yet. I'm just going to\nadd um\n\n44\n00:01:01,869 --> 00:01:01,879\n\n \n\n45\n00:01:01,879 --> 00:01:06,460\n\nexample.com and I will also add\n\n46\n00:01:06,460 --> 00:01:06,470\nexample.com and I will also add\n \n\n47\n00:01:06,470 --> 00:01:12,109\nexample.com and I will also add\n[Music]\n\n48\n00:01:12,109 --> 00:01:12,119\n\n \n\n49\n00:01:12,119 --> 00:01:14,149\n\npragmar.com and this should take about\n\n50\n00:01:14,149 --> 00:01:14,159\npragmar.com and this should take about\n \n\n51\n00:01:14,159 --> 00:01:16,710\npragmar.com and this should take about\n20 seconds. If you're not familiar with\n\n52\n00:01:16,710 --> 00:01:16,720\n20 seconds. If you're not familiar with\n \n\n53\n00:01:16,720 --> 00:01:19,510\n20 seconds. If you're not familiar with\nuh webcwlers, what this is doing is it's\n\n54\n00:01:19,510 --> 00:01:19,520\nuh webcwlers, what this is doing is it's\n \n\n55\n00:01:19,520 --> 00:01:21,590\nuh webcwlers, what this is doing is it's\ncollecting all the links in the index\n\n56\n00:01:21,590 --> 00:01:21,600\ncollecting all the links in the index\n \n\n57\n00:01:21,600 --> 00:01:24,390\ncollecting all the links in the index\npage and then uh collecting all of those\n\n58\n00:01:24,390 --> 00:01:24,400\npage and then uh collecting all of those\n \n\n59\n00:01:24,400 --> 00:01:26,230\npage and then uh collecting all of those\npages and then doing the same thing,\n\n60\n00:01:26,230 --> 00:01:26,240\npages and then doing the same thing,\n \n\n61\n00:01:26,240 --> 00:01:28,310\npages and then doing the same thing,\nscanning for more links. So yeah, we're\n\n62\n00:01:28,310 --> 00:01:28,320\nscanning for more links. So yeah, we're\n \n\n63\n00:01:28,320 --> 00:01:30,950\nscanning for more links. So yeah, we're\ndone. We So now we have Pragmar and we\n\n64\n00:01:30,950 --> 00:01:30,960\ndone. We So now we have Pragmar and we\n \n\n65\n00:01:30,960 --> 00:01:33,350\ndone. We So now we have Pragmar and we\nhave example.com. That's pretty much all\n\n66\n00:01:33,350 --> 00:01:33,360\nhave example.com. That's pretty much all\n \n\n67\n00:01:33,360 --> 00:01:35,270\nhave example.com. That's pretty much all\nwe need from interabot if we're using\n\n68\n00:01:35,270 --> 00:01:35,280\nwe need from interabot if we're using\n \n\n69\n00:01:35,280 --> 00:01:36,830\nwe need from interabot if we're using\nMCP server\n\n70\n00:01:36,830 --> 00:01:36,840\nMCP server\n \n\n71\n00:01:36,840 --> 00:01:39,830\nMCP server\nwebcrawl. But I did want to show you uh\n\n72\n00:01:39,830 --> 00:01:39,840\nwebcrawl. But I did want to show you uh\n \n\n73\n00:01:39,840 --> 00:01:42,350\nwebcrawl. But I did want to show you uh\nwhen we set up MCP server\n\n74\n00:01:42,350 --> 00:01:42,360\nwhen we set up MCP server\n \n\n75\n00:01:42,360 --> 00:01:45,149\nwhen we set up MCP server\nwebcwl, we'll want to use this\n\n76\n00:01:45,149 --> 00:01:45,159\nwebcwl, we'll want to use this\n \n\n77\n00:01:45,159 --> 00:01:48,749\nwebcwl, we'll want to use this\num this path here to interabot\n\n78\n00:01:48,749 --> 00:01:48,759\num this path here to interabot\n \n\n79\n00:01:48,759 --> 00:01:50,630\num this path here to interabot\nv2db because that's how we're going to\n\n80\n00:01:50,630 --> 00:01:50,640\nv2db because that's how we're going to\n \n\n81\n00:01:50,640 --> 00:01:52,590\nv2db because that's how we're going to\nuse our\n\n82\n00:01:52,590 --> 00:01:52,600\nuse our\n \n\n83\n00:01:52,600 --> 00:01:55,510\nuse our\nconnector. Okay, let's head over to our\n\n84\n00:01:55,510 --> 00:01:55,520\nconnector. Okay, let's head over to our\n \n\n85\n00:01:55,520 --> 00:01:57,749\nconnector. Okay, let's head over to our\nuh terminal here and we're going to set\n\n86\n00:01:57,749 --> 00:01:57,759\nuh terminal here and we're going to set\n \n\n87\n00:01:57,759 --> 00:02:04,910\nuh terminal here and we're going to set\nup mcp server webcwl. I already have\n\n88\n00:02:04,910 --> 00:02:04,920\n\n \n\n89\n00:02:04,920 --> 00:02:08,229\n\nPython. If you don't have Python, uh,\n\n90\n00:02:08,229 --> 00:02:08,239\nPython. If you don't have Python, uh,\n \n\n91\n00:02:08,239 --> 00:02:10,150\nPython. If you don't have Python, uh,\ncheck out the link in the description,\n\n92\n00:02:10,150 --> 00:02:10,160\ncheck out the link in the description,\n \n\n93\n00:02:10,160 --> 00:02:12,309\ncheck out the link in the description,\nget that installed, and then come back\n\n94\n00:02:12,309 --> 00:02:12,319\nget that installed, and then come back\n \n\n95\n00:02:12,319 --> 00:02:14,949\nget that installed, and then come back\nto this part of the video. But once you\n\n96\n00:02:14,949 --> 00:02:14,959\nto this part of the video. But once you\n \n\n97\n00:02:14,959 --> 00:02:17,030\nto this part of the video. But once you\nhave Python, you have pip, which is the\n\n98\n00:02:17,030 --> 00:02:17,040\nhave Python, you have pip, which is the\n \n\n99\n00:02:17,040 --> 00:02:19,670\nhave Python, you have pip, which is the\npackage manager. And you just want to do\n\n100\n00:02:19,670 --> 00:02:19,680\npackage manager. And you just want to do\n \n\n101\n00:02:19,680 --> 00:02:24,190\npackage manager. And you just want to do\na pip install uh, mcp\n\n102\n00:02:24,190 --> 00:02:24,200\na pip install uh, mcp\n \n\n103\n00:02:24,200 --> 00:02:26,589\na pip install uh, mcp\nserver\n\n104\n00:02:26,589 --> 00:02:26,599\nserver\n \n\n105\n00:02:26,599 --> 00:02:29,030\nserver\nwebcwl. Of course, I already have it,\n\n106\n00:02:29,030 --> 00:02:29,040\nwebcwl. Of course, I already have it,\n \n\n107\n00:02:29,040 --> 00:02:31,430\nwebcwl. Of course, I already have it,\nbut if you don't have it yet, it should\n\n108\n00:02:31,430 --> 00:02:31,440\nbut if you don't have it yet, it should\n \n\n109\n00:02:31,440 --> 00:02:34,070\nbut if you don't have it yet, it should\ninstall.\n\n110\n00:02:34,070 --> 00:02:34,080\ninstall.\n \n\n111\n00:02:34,080 --> 00:02:37,750\ninstall.\nSo, that is all set. And next, we're\n\n112\n00:02:37,750 --> 00:02:37,760\nSo, that is all set. And next, we're\n \n\n113\n00:02:37,760 --> 00:02:38,830\nSo, that is all set. And next, we're\ngoing\n\n114\n00:02:38,830 --> 00:02:38,840\ngoing\n \n\n115\n00:02:38,840 --> 00:02:42,229\ngoing\nto deal with our configuration. And the\n\n116\n00:02:42,229 --> 00:02:42,239\nto deal with our configuration. And the\n \n\n117\n00:02:42,239 --> 00:02:43,430\nto deal with our configuration. And the\nway we're going to do that is we're\n\n118\n00:02:43,430 --> 00:02:43,440\nway we're going to do that is we're\n \n\n119\n00:02:43,440 --> 00:02:44,509\nway we're going to do that is we're\ngoing to head over\n\n120\n00:02:44,509 --> 00:02:44,519\ngoing to head over\n \n\n121\n00:02:44,519 --> 00:02:46,910\ngoing to head over\nto Claw\n\n122\n00:02:46,910 --> 00:02:46,920\nto Claw\n \n\n123\n00:02:46,920 --> 00:02:49,670\nto Claw\nDesktop. And within the settings, you\n\n124\n00:02:49,670 --> 00:02:49,680\nDesktop. And within the settings, you\n \n\n125\n00:02:49,680 --> 00:02:51,470\nDesktop. And within the settings, you\ncan hit this edit\n\n126\n00:02:51,470 --> 00:02:51,480\ncan hit this edit\n \n\n127\n00:02:51,480 --> 00:02:54,350\ncan hit this edit\nconfig. And\n\n128\n00:02:54,350 --> 00:02:54,360\nconfig. And\n \n\n129\n00:02:54,360 --> 00:02:58,830\nconfig. And\nuh I'm just going to open this in VS\n\n130\n00:02:58,830 --> 00:02:58,840\nuh I'm just going to open this in VS\n \n\n131\n00:02:58,840 --> 00:03:07,430\nuh I'm just going to open this in VS\nCode and let's get this in here.\n\n132\n00:03:07,430 --> 00:03:07,440\n\n \n\n133\n00:03:07,440 --> 00:03:09,830\n\nAnd just to be clear, that configuration\n\n134\n00:03:09,830 --> 00:03:09,840\nAnd just to be clear, that configuration\n \n\n135\n00:03:09,840 --> 00:03:12,470\nAnd just to be clear, that configuration\nis coming from the MCP server webcrawl\n\n136\n00:03:12,470 --> 00:03:12,480\nis coming from the MCP server webcrawl\n \n\n137\n00:03:12,480 --> 00:03:16,270\nis coming from the MCP server webcrawl\nuh page and you can grab it here under\n\n138\n00:03:16,270 --> 00:03:16,280\nuh page and you can grab it here under\n \n\n139\n00:03:16,280 --> 00:03:19,830\nuh page and you can grab it here under\ninterobot. You'll just have to add\n\n140\n00:03:19,830 --> 00:03:19,840\ninterobot. You'll just have to add\n \n\n141\n00:03:19,840 --> 00:03:22,309\ninterobot. You'll just have to add\nuh your particular path. And if whether\n\n142\n00:03:22,309 --> 00:03:22,319\nuh your particular path. And if whether\n \n\n143\n00:03:22,319 --> 00:03:25,110\nuh your particular path. And if whether\nyou're Mac or Windows, you can find that\n\n144\n00:03:25,110 --> 00:03:25,120\nyou're Mac or Windows, you can find that\n \n\n145\n00:03:25,120 --> 00:03:28,470\nyou're Mac or Windows, you can find that\npath in the options.\n\n146\n00:03:28,470 --> 00:03:28,480\npath in the options.\n \n\n147\n00:03:28,480 --> 00:03:30,630\npath in the options.\nAnd next thing we got to do is come back\n\n148\n00:03:30,630 --> 00:03:30,640\nAnd next thing we got to do is come back\n \n\n149\n00:03:30,640 --> 00:03:33,149\nAnd next thing we got to do is come back\nto claude. It's very\n\n150\n00:03:33,149 --> 00:03:33,159\nto claude. It's very\n \n\n151\n00:03:33,159 --> 00:03:35,830\nto claude. It's very\nimportant that you exit. If you just\n\n152\n00:03:35,830 --> 00:03:35,840\nimportant that you exit. If you just\n \n\n153\n00:03:35,840 --> 00:03:37,830\nimportant that you exit. If you just\nclose, I don't think the config actually\n\n154\n00:03:37,830 --> 00:03:37,840\nclose, I don't think the config actually\n \n\n155\n00:03:37,840 --> 00:03:40,990\nclose, I don't think the config actually\ngets reloaded. I think you have to\n\n156\n00:03:40,990 --> 00:03:41,000\ngets reloaded. I think you have to\n \n\n157\n00:03:41,000 --> 00:03:43,990\ngets reloaded. I think you have to\nexit. Okay. So once you reload claude,\n\n158\n00:03:43,990 --> 00:03:44,000\nexit. Okay. So once you reload claude,\n \n\n159\n00:03:44,000 --> 00:03:45,670\nexit. Okay. So once you reload claude,\nyou should have this hammer here. And\n\n160\n00:03:45,670 --> 00:03:45,680\nyou should have this hammer here. And\n \n\n161\n00:03:45,680 --> 00:03:47,270\nyou should have this hammer here. And\nthis is a good sign. This means you have\n\n162\n00:03:47,270 --> 00:03:47,280\nthis is a good sign. This means you have\n \n\n163\n00:03:47,280 --> 00:03:49,470\nthis is a good sign. This means you have\nyour webcwl search and your webcrawl\n\n164\n00:03:49,470 --> 00:03:49,480\nyour webcwl search and your webcrawl\n \n\n165\n00:03:49,480 --> 00:03:51,910\nyour webcwl search and your webcrawl\nsites. Those are the actions that we\n\n166\n00:03:51,910 --> 00:03:51,920\nsites. Those are the actions that we\n \n\n167\n00:03:51,920 --> 00:03:53,589\nsites. Those are the actions that we\njust configured. Uh we configured it\n\n168\n00:03:53,589 --> 00:03:53,599\njust configured. Uh we configured it\n \n\n169\n00:03:53,599 --> 00:03:56,470\njust configured. Uh we configured it\nwith that one command. Just a heads up,\n\n170\n00:03:56,470 --> 00:03:56,480\nwith that one command. Just a heads up,\n \n\n171\n00:03:56,480 --> 00:03:58,869\nwith that one command. Just a heads up,\nthe cloud desktop uh interface has\n\n172\n00:03:58,869 --> 00:03:58,879\nthe cloud desktop uh interface has\n \n\n173\n00:03:58,879 --> 00:04:02,710\nthe cloud desktop uh interface has\nchanged since I recorded the uh demo.\n\n174\n00:04:02,710 --> 00:04:02,720\nchanged since I recorded the uh demo.\n \n\n175\n00:04:02,720 --> 00:04:05,710\nchanged since I recorded the uh demo.\nAnd MCP servers are now under search and\n\n176\n00:04:05,710 --> 00:04:05,720\nAnd MCP servers are now under search and\n \n\n177\n00:04:05,720 --> 00:04:08,789\nAnd MCP servers are now under search and\ntools. So like right here, they can be\n\n178\n00:04:08,789 --> 00:04:08,799\ntools. So like right here, they can be\n \n\n179\n00:04:08,799 --> 00:04:10,190\ntools. So like right here, they can be\ntoggled on and\n\n180\n00:04:10,190 --> 00:04:10,200\ntoggled on and\n \n\n181\n00:04:10,200 --> 00:04:12,869\ntoggled on and\noff. But the button that I'm referring\n\n182\n00:04:12,869 --> 00:04:12,879\noff. But the button that I'm referring\n \n\n183\n00:04:12,879 --> 00:04:14,789\noff. But the button that I'm referring\nto, the hammer button that I'm getting\n\n184\n00:04:14,789 --> 00:04:14,799\nto, the hammer button that I'm getting\n \n\n185\n00:04:14,799 --> 00:04:17,830\nto, the hammer button that I'm getting\nexcited about perhaps is no longer part\n\n186\n00:04:17,830 --> 00:04:17,840\nexcited about perhaps is no longer part\n \n\n187\n00:04:17,840 --> 00:04:22,069\nexcited about perhaps is no longer part\nof the interface. So as of May 25, uh\n\n188\n00:04:22,069 --> 00:04:22,079\nof the interface. So as of May 25, uh\n \n\n189\n00:04:22,079 --> 00:04:23,670\nof the interface. So as of May 25, uh\nthis is how it works. things are in\n\n190\n00:04:23,670 --> 00:04:23,680\nthis is how it works. things are in\n \n\n191\n00:04:23,680 --> 00:04:26,150\nthis is how it works. things are in\nflux, so it could change again. But, uh,\n\n192\n00:04:26,150 --> 00:04:26,160\nflux, so it could change again. But, uh,\n \n\n193\n00:04:26,160 --> 00:04:28,270\nflux, so it could change again. But, uh,\njust wanted to give you that heads\n\n194\n00:04:28,270 --> 00:04:28,280\njust wanted to give you that heads\n \n\n195\n00:04:28,280 --> 00:04:30,469\njust wanted to give you that heads\nup. So, the first thing I'm going to do\n\n196\n00:04:30,469 --> 00:04:30,479\nup. So, the first thing I'm going to do\n \n\n197\n00:04:30,479 --> 00:04:47,390\nup. So, the first thing I'm going to do\nis ask Claude if he can list web\n\n198\n00:04:47,390 --> 00:04:47,400\n\n \n\n199\n00:04:47,400 --> 00:04:49,990\n\ncrawls. That looks good. Those are my\n\n200\n00:04:49,990 --> 00:04:50,000\ncrawls. That looks good. Those are my\n \n\n201\n00:04:50,000 --> 00:04:53,430\ncrawls. That looks good. Those are my\ntwo web crawls. So the next thing I want\n\n202\n00:04:53,430 --> 00:04:53,440\ntwo web crawls. So the next thing I want\n \n\n203\n00:04:53,440 --> 00:04:56,550\ntwo web crawls. So the next thing I want\nto do is let's make a query uh so that\n\n204\n00:04:56,550 --> 00:04:56,560\nto do is let's make a query uh so that\n \n\n205\n00:04:56,560 --> 00:04:58,230\nto do is let's make a query uh so that\nwe're searching. So there's two actions.\n\n206\n00:04:58,230 --> 00:04:58,240\nwe're searching. So there's two actions.\n \n\n207\n00:04:58,240 --> 00:05:00,230\nwe're searching. So there's two actions.\nWe just did the list. So let's filter\n\n208\n00:05:00,230 --> 00:05:00,240\nWe just did the list. So let's filter\n \n\n209\n00:05:00,240 --> 00:05:02,030\nWe just did the list. So let's filter\nsome content.\n\n210\n00:05:02,030 --> 00:05:02,040\nsome content.\n \n\n211\n00:05:02,040 --> 00:05:08,430\nsome content.\nSay, can you give me a file type\n\n212\n00:05:08,430 --> 00:05:08,440\n\n \n\n213\n00:05:08,440 --> 00:05:14,590\n\nsummary\n\n214\n00:05:14,590 --> 00:05:14,600\n\n \n\n215\n00:05:14,600 --> 00:05:16,670\n\npragmar.com image\n\n216\n00:05:16,670 --> 00:05:16,680\npragmar.com image\n \n\n217\n00:05:16,680 --> 00:05:21,870\npragmar.com image\ntypes HTML page count\n\n218\n00:05:21,870 --> 00:05:21,880\ntypes HTML page count\n \n\n219\n00:05:21,880 --> 00:05:26,110\ntypes HTML page count\neterc let's see what cloud comes back\n\n220\n00:05:26,110 --> 00:05:26,120\neterc let's see what cloud comes back\n \n\n221\n00:05:26,120 --> 00:05:28,870\neterc let's see what cloud comes back\nwith okay so the first permission I gave\n\n222\n00:05:28,870 --> 00:05:28,880\nwith okay so the first permission I gave\n \n\n223\n00:05:28,880 --> 00:05:30,950\nwith okay so the first permission I gave\nwas to list sites this time I'm giving\n\n224\n00:05:30,950 --> 00:05:30,960\nwas to list sites this time I'm giving\n \n\n225\n00:05:30,960 --> 00:05:41,950\nwas to list sites this time I'm giving\npermission to search the web\n\n226\n00:05:41,950 --> 00:05:41,960\n\n \n\n227\n00:05:41,960 --> 00:05:44,950\n\ncrawl. You can see the content being\n\n228\n00:05:44,950 --> 00:05:44,960\ncrawl. You can see the content being\n \n\n229\n00:05:44,960 --> 00:05:46,510\ncrawl. You can see the content being\ngenerated from the\n\n230\n00:05:46,510 --> 00:05:46,520\ngenerated from the\n \n\n231\n00:05:46,520 --> 00:05:48,610\ngenerated from the\nAPI. Taking a while to\n\n232\n00:05:48,610 --> 00:05:48,620\nAPI. Taking a while to\n \n\n233\n00:05:48,620 --> 00:05:55,950\nAPI. Taking a while to\n[Music]\n\n234\n00:05:55,950 --> 00:05:55,960\n\n \n\n235\n00:05:55,960 --> 00:05:59,029\n\nanalyze. Still going though. See what\n\n236\n00:05:59,029 --> 00:05:59,039\nanalyze. Still going though. See what\n \n\n237\n00:05:59,039 --> 00:06:06,710\nanalyze. Still going though. See what\nhappens.\n\n238\n00:06:06,710 --> 00:06:06,720\n\n \n\n239\n00:06:06,720 --> 00:06:10,350\n\nOh, cool. It's going to generate a\n\n240\n00:06:10,350 --> 00:06:10,360\nOh, cool. It's going to generate a\n \n\n241\n00:06:10,360 --> 00:06:12,909\nOh, cool. It's going to generate a\nreport. So, this is just the tip of the\n\n242\n00:06:12,909 --> 00:06:12,919\nreport. So, this is just the tip of the\n \n\n243\n00:06:12,919 --> 00:06:15,270\nreport. So, this is just the tip of the\niceberg. In my mind, it's a great tool\n\n244\n00:06:15,270 --> 00:06:15,280\niceberg. In my mind, it's a great tool\n \n\n245\n00:06:15,280 --> 00:06:18,150\niceberg. In my mind, it's a great tool\nfor managing a website or administering\n\n246\n00:06:18,150 --> 00:06:18,160\nfor managing a website or administering\n \n\n247\n00:06:18,160 --> 00:06:21,749\nfor managing a website or administering\ncontent on a website. It'd be great tool\n\n248\n00:06:21,749 --> 00:06:21,759\ncontent on a website. It'd be great tool\n \n\n249\n00:06:21,759 --> 00:06:23,990\ncontent on a website. It'd be great tool\nfor marketers if they want a sort of\n\n250\n00:06:23,990 --> 00:06:24,000\nfor marketers if they want a sort of\n \n\n251\n00:06:24,000 --> 00:06:27,350\nfor marketers if they want a sort of\nbrand library that they can just query.\n\n252\n00:06:27,350 --> 00:06:27,360\nbrand library that they can just query.\n \n\n253\n00:06:27,360 --> 00:06:30,270\nbrand library that they can just query.\nuh they could use their website as\n\n254\n00:06:30,270 --> 00:06:30,280\nuh they could use their website as\n \n\n255\n00:06:30,280 --> 00:06:33,670\nuh they could use their website as\nthat. I hope you give it a shot and this\n\n256\n00:06:33,670 --> 00:06:33,680\nthat. I hope you give it a shot and this\n \n\n257\n00:06:33,680 --> 00:06:35,590\nthat. I hope you give it a shot and this\nis multic crawler so it does work with\n\n258\n00:06:35,590 --> 00:06:35,600\nis multic crawler so it does work with\n \n\n259\n00:06:35,600 --> 00:06:39,430\nis multic crawler so it does work with\nkatana site one wget also works with\n\n260\n00:06:39,430 --> 00:06:39,440\nkatana site one wget also works with\n \n\n261\n00:06:39,440 --> 00:06:41,469\nkatana site one wget also works with\nwork files if that's your\n\n262\n00:06:41,469 --> 00:06:41,479\nwork files if that's your\n \n\n263\n00:06:41,479 --> 00:06:44,909\nwork files if that's your\nthing. Yeah, this thing is multic\n\n264\n00:06:44,909 --> 00:06:44,919\nthing. Yeah, this thing is multic\n \n\n265\n00:06:44,919 --> 00:06:47,070\nthing. Yeah, this thing is multic\ncrawler ready to\n\n266\n00:06:47,070 --> 00:06:47,080\ncrawler ready to\n \n\n267\n00:06:47,080 --> 00:06:49,270\ncrawler ready to\ngo. So if you find any of this\n\n268\n00:06:49,270 --> 00:06:49,280\ngo. So if you find any of this\n \n\n269\n00:06:49,280 --> 00:06:51,270\ngo. So if you find any of this\ninteresting uh go ahead and grab\n\n270\n00:06:51,270 --> 00:06:51,280\ninteresting uh go ahead and grab\n \n\n271\n00:06:51,280 --> 00:06:54,309\ninteresting uh go ahead and grab\nyourself MCP server webcrawl and give it\n\n272\n00:06:54,309 --> 00:06:54,319\nyourself MCP server webcrawl and give it\n \n\n273\n00:06:54,319 --> 00:06:57,029\nyourself MCP server webcrawl and give it\na shot. See what you can do.\n\n274\n00:06:57,029 --> 00:06:57,039\na shot. See what you can do.\n \n\n275\n00:06:57,039 --> 00:07:01,160\na shot. See what you can do.\nUh, sky's is the limit.", "1\n00:00:00,700 --> 00:00:06,470\n\n[Music]\n\n2\n00:00:06,470 --> 00:00:06,480\n[Music]\n \n\n3\n00:00:06,480 --> 00:00:08,470\n[Music]\nThis is Ben from Interabot and today I'm\n\n4\n00:00:08,470 --> 00:00:08,480\nThis is Ben from Interabot and today I'm\n \n\n5\n00:00:08,480 --> 00:00:09,990\nThis is Ben from Interabot and today I'm\ngoing to take you through connecting an\n\n6\n00:00:09,990 --> 00:00:10,000\ngoing to take you through connecting an\n \n\n7\n00:00:10,000 --> 00:00:13,830\ngoing to take you through connecting an\nLLM cloud desktop to a web crawl\n\n8\n00:00:13,830 --> 00:00:13,840\nLLM cloud desktop to a web crawl\n \n\n9\n00:00:13,840 --> 00:00:16,710\nLLM cloud desktop to a web crawl\ngenerated by Katana webcwler. And we're\n\n10\n00:00:16,710 --> 00:00:16,720\ngenerated by Katana webcwler. And we're\n \n\n11\n00:00:16,720 --> 00:00:19,429\ngenerated by Katana webcwler. And we're\ngoing to do that with MCP server webcwl\n\n12\n00:00:19,429 --> 00:00:19,439\ngoing to do that with MCP server webcwl\n \n\n13\n00:00:19,439 --> 00:00:21,349\ngoing to do that with MCP server webcwl\nand that's going to act as a sort of\n\n14\n00:00:21,349 --> 00:00:21,359\nand that's going to act as a sort of\n \n\n15\n00:00:21,359 --> 00:00:23,269\nand that's going to act as a sort of\nmiddle layer that can fire off search\n\n16\n00:00:23,269 --> 00:00:23,279\nmiddle layer that can fire off search\n \n\n17\n00:00:23,279 --> 00:00:25,109\nmiddle layer that can fire off search\nand retrieval requests and hand them\n\n18\n00:00:25,109 --> 00:00:25,119\nand retrieval requests and hand them\n \n\n19\n00:00:25,119 --> 00:00:27,910\nand retrieval requests and hand them\nback to the LLM. Uh so it's pretty\n\n20\n00:00:27,910 --> 00:00:27,920\nback to the LLM. Uh so it's pretty\n \n\n21\n00:00:27,920 --> 00:00:29,910\nback to the LLM. Uh so it's pretty\npretty powerful stuff.\n\n22\n00:00:29,910 --> 00:00:29,920\npretty powerful stuff.\n \n\n23\n00:00:29,920 --> 00:00:32,110\npretty powerful stuff.\nAnd uh let's just get into\n\n24\n00:00:32,110 --> 00:00:32,120\nAnd uh let's just get into\n \n\n25\n00:00:32,120 --> 00:00:34,630\nAnd uh let's just get into\nit. So in this case, we're going to set\n\n26\n00:00:34,630 --> 00:00:34,640\nit. So in this case, we're going to set\n \n\n27\n00:00:34,640 --> 00:00:37,510\nit. So in this case, we're going to set\nup the MCP server webcwl. And I'm going\n\n28\n00:00:37,510 --> 00:00:37,520\nup the MCP server webcwl. And I'm going\n \n\n29\n00:00:37,520 --> 00:00:40,069\nup the MCP server webcwl. And I'm going\nto briefly go over Katana crawler and\n\n30\n00:00:40,069 --> 00:00:40,079\nto briefly go over Katana crawler and\n \n\n31\n00:00:40,079 --> 00:00:41,709\nto briefly go over Katana crawler and\nthen we'll connect\n\n32\n00:00:41,709 --> 00:00:41,719\nthen we'll connect\n \n\n33\n00:00:41,719 --> 00:00:44,709\nthen we'll connect\nClaude. Uh in order to install MCP\n\n34\n00:00:44,709 --> 00:00:44,719\nClaude. Uh in order to install MCP\n \n\n35\n00:00:44,719 --> 00:00:46,470\nClaude. Uh in order to install MCP\nserver webcrawl, we need the runtime.\n\n36\n00:00:46,470 --> 00:00:46,480\nserver webcrawl, we need the runtime.\n \n\n37\n00:00:46,480 --> 00:00:49,190\nserver webcrawl, we need the runtime.\nAnd the runtime is Python. You can get\n\n38\n00:00:49,190 --> 00:00:49,200\nAnd the runtime is Python. You can get\n \n\n39\n00:00:49,200 --> 00:00:51,510\nAnd the runtime is Python. You can get\nPython at python.org. Just head over\n\n40\n00:00:51,510 --> 00:00:51,520\nPython at python.org. Just head over\n \n\n41\n00:00:51,520 --> 00:00:54,389\nPython at python.org. Just head over\nthere and click on the downloads. Once\n\n42\n00:00:54,389 --> 00:00:54,399\nthere and click on the downloads. Once\n \n\n43\n00:00:54,399 --> 00:00:55,910\nthere and click on the downloads. Once\nyou get that, you're going to want to\n\n44\n00:00:55,910 --> 00:00:55,920\nyou get that, you're going to want to\n \n\n45\n00:00:55,920 --> 00:00:57,990\nyou get that, you're going to want to\nopen up a shell. And you can just test\n\n46\n00:00:57,990 --> 00:00:58,000\nopen up a shell. And you can just test\n \n\n47\n00:00:58,000 --> 00:01:01,029\nopen up a shell. And you can just test\nthat Python is on your path with a uh\n\n48\n00:01:01,029 --> 00:01:01,039\nthat Python is on your path with a uh\n \n\n49\n00:01:01,039 --> 00:01:02,349\nthat Python is on your path with a uh\nPython\n\n50\n00:01:02,349 --> 00:01:02,359\nPython\n \n\n51\n00:01:02,359 --> 00:01:04,710\nPython\nversion. And it is on my path, which\n\n52\n00:01:04,710 --> 00:01:04,720\nversion. And it is on my path, which\n \n\n53\n00:01:04,720 --> 00:01:06,469\nversion. And it is on my path, which\nmeans pip, which is the package\n\n54\n00:01:06,469 --> 00:01:06,479\nmeans pip, which is the package\n \n\n55\n00:01:06,479 --> 00:01:08,870\nmeans pip, which is the package\ninstaller for Python, is also going to\n\n56\n00:01:08,870 --> 00:01:08,880\ninstaller for Python, is also going to\n \n\n57\n00:01:08,880 --> 00:01:12,350\ninstaller for Python, is also going to\nbe on my\n\n58\n00:01:12,350 --> 00:01:12,360\n\n \n\n59\n00:01:12,360 --> 00:01:14,630\n\npath. If it's not on your path, you're\n\n60\n00:01:14,630 --> 00:01:14,640\npath. If it's not on your path, you're\n \n\n61\n00:01:14,640 --> 00:01:16,469\npath. If it's not on your path, you're\njust going to have to use Absolute Paths\n\n62\n00:01:16,469 --> 00:01:16,479\njust going to have to use Absolute Paths\n \n\n63\n00:01:16,479 --> 00:01:17,630\njust going to have to use Absolute Paths\nfor this to\n\n64\n00:01:17,630 --> 00:01:17,640\nfor this to\n \n\n65\n00:01:17,640 --> 00:01:19,990\nfor this to\nwork. So, at this point, I'm going to\n\n66\n00:01:19,990 --> 00:01:20,000\nwork. So, at this point, I'm going to\n \n\n67\n00:01:20,000 --> 00:01:22,870\nwork. So, at this point, I'm going to\ninstall MCP server webcrawl. And the uh\n\n68\n00:01:22,870 --> 00:01:22,880\ninstall MCP server webcrawl. And the uh\n \n\n69\n00:01:22,880 --> 00:01:25,390\ninstall MCP server webcrawl. And the uh\ncommand to do that is pip\n\n70\n00:01:25,390 --> 00:01:25,400\ncommand to do that is pip\n \n\n71\n00:01:25,400 --> 00:01:28,109\ncommand to do that is pip\ninstall mcp\n\n72\n00:01:28,109 --> 00:01:28,119\ninstall mcp\n \n\n73\n00:01:28,119 --> 00:01:30,030\ninstall mcp\nserver\n\n74\n00:01:30,030 --> 00:01:30,040\nserver\n \n\n75\n00:01:30,040 --> 00:01:32,310\nserver\nwebcrow. And I already have it. So it's\n\n76\n00:01:32,310 --> 00:01:32,320\nwebcrow. And I already have it. So it's\n \n\n77\n00:01:32,320 --> 00:01:34,789\nwebcrow. And I already have it. So it's\ngoing to say requirements satisfied. But\n\n78\n00:01:34,789 --> 00:01:34,799\ngoing to say requirements satisfied. But\n \n\n79\n00:01:34,799 --> 00:01:37,230\ngoing to say requirements satisfied. But\nfor you presumably it's going to\n\n80\n00:01:37,230 --> 00:01:37,240\nfor you presumably it's going to\n \n\n81\n00:01:37,240 --> 00:01:39,429\nfor you presumably it's going to\ninstall. That's going to be exactly what\n\n82\n00:01:39,429 --> 00:01:39,439\ninstall. That's going to be exactly what\n \n\n83\n00:01:39,439 --> 00:01:42,630\ninstall. That's going to be exactly what\nyou need. So we have our runtime. We\n\n84\n00:01:42,630 --> 00:01:42,640\nyou need. So we have our runtime. We\n \n\n85\n00:01:42,640 --> 00:01:46,350\nyou need. So we have our runtime. We\nhave our MCP server. Uh let's look at\n\n86\n00:01:46,350 --> 00:01:46,360\nhave our MCP server. Uh let's look at\n \n\n87\n00:01:46,360 --> 00:01:49,749\nhave our MCP server. Uh let's look at\nKatana. So Katana is over here in\n\n88\n00:01:49,749 --> 00:01:49,759\nKatana. So Katana is over here in\n \n\n89\n00:01:49,759 --> 00:01:52,550\nKatana. So Katana is over here in\nproject discovery. Katana and GitHub.\n\n90\n00:01:52,550 --> 00:01:52,560\nproject discovery. Katana and GitHub.\n \n\n91\n00:01:52,560 --> 00:01:55,270\nproject discovery. Katana and GitHub.\nI'm going to use uh Ubuntu OS\n\n92\n00:01:55,270 --> 00:01:55,280\nI'm going to use uh Ubuntu OS\n \n\n93\n00:01:55,280 --> 00:01:57,270\nI'm going to use uh Ubuntu OS\nenvironment to run my crawls just\n\n94\n00:01:57,270 --> 00:01:57,280\nenvironment to run my crawls just\n \n\n95\n00:01:57,280 --> 00:02:00,429\nenvironment to run my crawls just\nbecause I already have Golang set up\n\n96\n00:02:00,429 --> 00:02:00,439\nbecause I already have Golang set up\n \n\n97\n00:02:00,439 --> 00:02:02,749\nbecause I already have Golang set up\nthere. You might need to install the Go\n\n98\n00:02:02,749 --> 00:02:02,759\nthere. You might need to install the Go\n \n\n99\n00:02:02,759 --> 00:02:06,310\nthere. You might need to install the Go\nlanguage if you haven't already. I'm\n\n100\n00:02:06,310 --> 00:02:06,320\nlanguage if you haven't already. I'm\n \n\n101\n00:02:06,320 --> 00:02:07,749\nlanguage if you haven't already. I'm\ngoing to presume that most of you\n\n102\n00:02:07,749 --> 00:02:07,759\ngoing to presume that most of you\n \n\n103\n00:02:07,759 --> 00:02:09,749\ngoing to presume that most of you\nalready have Katana installed because\n\n104\n00:02:09,749 --> 00:02:09,759\nalready have Katana installed because\n \n\n105\n00:02:09,759 --> 00:02:12,150\nalready have Katana installed because\nyou found this video. But let's quickly\n\n106\n00:02:12,150 --> 00:02:12,160\nyou found this video. But let's quickly\n \n\n107\n00:02:12,160 --> 00:02:14,630\nyou found this video. But let's quickly\ngo over it. So you install Go download\n\n108\n00:02:14,630 --> 00:02:14,640\ngo over it. So you install Go download\n \n\n109\n00:02:14,640 --> 00:02:16,390\ngo over it. So you install Go download\nand install.\n\n110\n00:02:16,390 --> 00:02:16,400\nand install.\n \n\n111\n00:02:16,400 --> 00:02:18,710\nand install.\nIf you want to check that Go is on path,\n\n112\n00:02:18,710 --> 00:02:18,720\nIf you want to check that Go is on path,\n \n\n113\n00:02:18,720 --> 00:02:20,110\nIf you want to check that Go is on path,\nyou can say Go\n\n114\n00:02:20,110 --> 00:02:20,120\nyou can say Go\n \n\n115\n00:02:20,120 --> 00:02:22,229\nyou can say Go\nversion. It should be. I think by\n\n116\n00:02:22,229 --> 00:02:22,239\nversion. It should be. I think by\n \n\n117\n00:02:22,239 --> 00:02:26,190\nversion. It should be. I think by\ndefault it is. And then to install\n\n118\n00:02:26,190 --> 00:02:26,200\ndefault it is. And then to install\n \n\n119\n00:02:26,200 --> 00:02:29,030\ndefault it is. And then to install\nKatana, you head over to this GitHub\n\n120\n00:02:29,030 --> 00:02:29,040\nKatana, you head over to this GitHub\n \n\n121\n00:02:29,040 --> 00:02:31,910\nKatana, you head over to this GitHub\npage and it's right up at the top. Just\n\n122\n00:02:31,910 --> 00:02:31,920\npage and it's right up at the top. Just\n \n\n123\n00:02:31,920 --> 00:02:33,710\npage and it's right up at the top. Just\ngrab\n\n124\n00:02:33,710 --> 00:02:33,720\ngrab\n \n\n125\n00:02:33,720 --> 00:02:37,470\ngrab\nthis, paste it\n\n126\n00:02:37,470 --> 00:02:37,480\n\n \n\n127\n00:02:37,480 --> 00:02:40,070\n\nin. It's going to take a second to\n\n128\n00:02:40,070 --> 00:02:40,080\nin. It's going to take a second to\n \n\n129\n00:02:40,080 --> 00:02:41,550\nin. It's going to take a second to\ndownload\n\n130\n00:02:41,550 --> 00:02:41,560\ndownload\n \n\n131\n00:02:41,560 --> 00:02:43,990\ndownload\nand should be done pretty soon. Here.\n\n132\n00:02:43,990 --> 00:02:44,000\nand should be done pretty soon. Here.\n \n\n133\n00:02:44,000 --> 00:02:47,350\nand should be done pretty soon. Here.\nThere it is. So, at this point, I know\n\n134\n00:02:47,350 --> 00:02:47,360\nThere it is. So, at this point, I know\n \n\n135\n00:02:47,360 --> 00:02:49,670\nThere it is. So, at this point, I know\nI've done a uh quite a few things in\n\n136\n00:02:49,670 --> 00:02:49,680\nI've done a uh quite a few things in\n \n\n137\n00:02:49,680 --> 00:02:51,910\nI've done a uh quite a few things in\nrapid succession. So, let me just go\n\n138\n00:02:51,910 --> 00:02:51,920\nrapid succession. So, let me just go\n \n\n139\n00:02:51,920 --> 00:02:54,070\nrapid succession. So, let me just go\nover what we've done. We uh we've\n\n140\n00:02:54,070 --> 00:02:54,080\nover what we've done. We uh we've\n \n\n141\n00:02:54,080 --> 00:02:57,030\nover what we've done. We uh we've\ngathered our server runtime, Python.\n\n142\n00:02:57,030 --> 00:02:57,040\ngathered our server runtime, Python.\n \n\n143\n00:02:57,040 --> 00:02:59,509\ngathered our server runtime, Python.\nWe've installed our MCP server, MCP\n\n144\n00:02:59,509 --> 00:02:59,519\nWe've installed our MCP server, MCP\n \n\n145\n00:02:59,519 --> 00:03:03,030\nWe've installed our MCP server, MCP\nserver webcrawl, and we've got Katana\n\n146\n00:03:03,030 --> 00:03:03,040\nserver webcrawl, and we've got Katana\n \n\n147\n00:03:03,040 --> 00:03:04,270\nserver webcrawl, and we've got Katana\nready to\n\n148\n00:03:04,270 --> 00:03:04,280\nready to\n \n\n149\n00:03:04,280 --> 00:03:06,390\nready to\ngo. So, now that we have everything\n\n150\n00:03:06,390 --> 00:03:06,400\ngo. So, now that we have everything\n \n\n151\n00:03:06,400 --> 00:03:07,750\ngo. So, now that we have everything\ninstalled, we're going to head back to\n\n152\n00:03:07,750 --> 00:03:07,760\ninstalled, we're going to head back to\n \n\n153\n00:03:07,760 --> 00:03:11,309\ninstalled, we're going to head back to\nMCP server webcrawl and head down to MCP\n\n154\n00:03:11,309 --> 00:03:11,319\nMCP server webcrawl and head down to MCP\n \n\n155\n00:03:11,319 --> 00:03:15,350\nMCP server webcrawl and head down to MCP\nconfiguration and select Katana.\n\n156\n00:03:15,350 --> 00:03:15,360\nconfiguration and select Katana.\n \n\n157\n00:03:15,360 --> 00:03:16,630\nconfiguration and select Katana.\nAt the top there's going to be an\n\n158\n00:03:16,630 --> 00:03:16,640\nAt the top there's going to be an\n \n\n159\n00:03:16,640 --> 00:03:18,630\nAt the top there's going to be an\nexample configuration. Uh we're going to\n\n160\n00:03:18,630 --> 00:03:18,640\nexample configuration. Uh we're going to\n \n\n161\n00:03:18,640 --> 00:03:20,229\nexample configuration. Uh we're going to\nget there but we're not there yet. Right\n\n162\n00:03:20,229 --> 00:03:20,239\nget there but we're not there yet. Right\n \n\n163\n00:03:20,239 --> 00:03:21,949\nget there but we're not there yet. Right\nnow we're just collecting\n\n164\n00:03:21,949 --> 00:03:21,959\nnow we're just collecting\n \n\n165\n00:03:21,959 --> 00:03:24,990\nnow we're just collecting\ncrawls and uh I'm going to grab\n\n166\n00:03:24,990 --> 00:03:25,000\ncrawls and uh I'm going to grab\n \n\n167\n00:03:25,000 --> 00:03:28,070\ncrawls and uh I'm going to grab\nthis. I'll just explain these arguments.\n\n168\n00:03:28,070 --> 00:03:28,080\nthis. I'll just explain these arguments.\n \n\n169\n00:03:28,080 --> 00:03:30,070\nthis. I'll just explain these arguments.\nStore response means it's going to store\n\n170\n00:03:30,070 --> 00:03:30,080\nStore response means it's going to store\n \n\n171\n00:03:30,080 --> 00:03:32,550\nStore response means it's going to store\nthe files that it collects and store\n\n172\n00:03:32,550 --> 00:03:32,560\nthe files that it collects and store\n \n\n173\n00:03:32,560 --> 00:03:34,630\nthe files that it collects and store\nresponse dur is going to set where it's\n\n174\n00:03:34,630 --> 00:03:34,640\nresponse dur is going to set where it's\n \n\n175\n00:03:34,640 --> 00:03:37,509\nresponse dur is going to set where it's\ngoing to save those files. If I don't\n\n176\n00:03:37,509 --> 00:03:37,519\ngoing to save those files. If I don't\n \n\n177\n00:03:37,519 --> 00:03:41,670\ngoing to save those files. If I don't\nexplicitly state store response dur\n\n178\n00:03:41,670 --> 00:03:41,680\nexplicitly state store response dur\n \n\n179\n00:03:41,680 --> 00:03:43,350\nexplicitly state store response dur\nuh write to a default directory, which\n\n180\n00:03:43,350 --> 00:03:43,360\nuh write to a default directory, which\n \n\n181\n00:03:43,360 --> 00:03:46,149\nuh write to a default directory, which\nmeans if I crawl example.com and then\n\n182\n00:03:46,149 --> 00:03:46,159\nmeans if I crawl example.com and then\n \n\n183\n00:03:46,159 --> 00:03:49,190\nmeans if I crawl example.com and then\ncrawl pragmar.com, it's just going to uh\n\n184\n00:03:49,190 --> 00:03:49,200\ncrawl pragmar.com, it's just going to uh\n \n\n185\n00:03:49,200 --> 00:03:50,869\ncrawl pragmar.com, it's just going to uh\noverwrite everything into the same\n\n186\n00:03:50,869 --> 00:03:50,879\noverwrite everything into the same\n \n\n187\n00:03:50,879 --> 00:03:54,070\noverwrite everything into the same\ndefault directory. Whereas if I specify\n\n188\n00:03:54,070 --> 00:03:54,080\ndefault directory. Whereas if I specify\n \n\n189\n00:03:54,080 --> 00:03:56,070\ndefault directory. Whereas if I specify\nthe crawls directory, it's going to\n\n190\n00:03:56,070 --> 00:03:56,080\nthe crawls directory, it's going to\n \n\n191\n00:03:56,080 --> 00:03:58,630\nthe crawls directory, it's going to\ncreate a crawls directory and then store\n\n192\n00:03:58,630 --> 00:03:58,640\ncreate a crawls directory and then store\n \n\n193\n00:03:58,640 --> 00:04:00,910\ncreate a crawls directory and then store\neach crawl in a different host\n\n194\n00:04:00,910 --> 00:04:00,920\neach crawl in a different host\n \n\n195\n00:04:00,920 --> 00:04:03,509\neach crawl in a different host\ndirectory. That's exactly what I want\n\n196\n00:04:03,509 --> 00:04:03,519\ndirectory. That's exactly what I want\n \n\n197\n00:04:03,519 --> 00:04:04,710\ndirectory. That's exactly what I want\nbecause I want to be able to work with\n\n198\n00:04:04,710 --> 00:04:04,720\nbecause I want to be able to work with\n \n\n199\n00:04:04,720 --> 00:04:06,710\nbecause I want to be able to work with\nmultiple hosts.\n\n200\n00:04:06,710 --> 00:04:06,720\nmultiple hosts.\n \n\n201\n00:04:06,720 --> 00:04:10,309\nmultiple hosts.\nSo, all that said, let's grab this and\n\n202\n00:04:10,309 --> 00:04:10,319\nSo, all that said, let's grab this and\n \n\n203\n00:04:10,319 --> 00:04:13,910\nSo, all that said, let's grab this and\npaste it in here. And you can take a\n\n204\n00:04:13,910 --> 00:04:13,920\npaste it in here. And you can take a\n \n\n205\n00:04:13,920 --> 00:04:17,189\npaste it in here. And you can take a\nlook at where I am saving these files.\n\n206\n00:04:17,189 --> 00:04:17,199\nlook at where I am saving these files.\n \n\n207\n00:04:17,199 --> 00:04:20,270\nlook at where I am saving these files.\nSo, right now, I'm working in MCP demo\n\n208\n00:04:20,270 --> 00:04:20,280\nSo, right now, I'm working in MCP demo\n \n\n209\n00:04:20,280 --> 00:04:23,110\nSo, right now, I'm working in MCP demo\nkatana. So, let's run this. And you\n\n210\n00:04:23,110 --> 00:04:23,120\nkatana. So, let's run this. And you\n \n\n211\n00:04:23,120 --> 00:04:25,270\nkatana. So, let's run this. And you\nshould see a crawls directory pop up\n\n212\n00:04:25,270 --> 00:04:25,280\nshould see a crawls directory pop up\n \n\n213\n00:04:25,280 --> 00:04:27,390\nshould see a crawls directory pop up\nafter it's\n\n214\n00:04:27,390 --> 00:04:27,400\nafter it's\n \n\n215\n00:04:27,400 --> 00:04:32,590\nafter it's\ndone. It's going to take a\n\n216\n00:04:32,590 --> 00:04:32,600\n\n \n\n217\n00:04:32,600 --> 00:04:35,990\n\nsecond. There it goes. And I'm not even\n\n218\n00:04:35,990 --> 00:04:36,000\nsecond. There it goes. And I'm not even\n \n\n219\n00:04:36,000 --> 00:04:37,670\nsecond. There it goes. And I'm not even\ngoing to really look. I'm just going to\n\n220\n00:04:37,670 --> 00:04:37,680\ngoing to really look. I'm just going to\n \n\n221\n00:04:37,680 --> 00:04:40,310\ngoing to really look. I'm just going to\ngrab another host here. I'm going to\n\n222\n00:04:40,310 --> 00:04:40,320\ngrab another host here. I'm going to\n \n\n223\n00:04:40,320 --> 00:04:44,270\ngrab another host here. I'm going to\ngrab\n\n224\n00:04:44,270 --> 00:04:44,280\n\n \n\n225\n00:04:44,280 --> 00:04:54,670\n\nPragmar. We'll take a look after it's\n\n226\n00:04:54,670 --> 00:04:54,680\n\n \n\n227\n00:04:54,680 --> 00:04:57,590\n\ndone. Okay. And we're done. So, we've\n\n228\n00:04:57,590 --> 00:04:57,600\ndone. Okay. And we're done. So, we've\n \n\n229\n00:04:57,600 --> 00:04:59,430\ndone. Okay. And we're done. So, we've\ngot two crawls now. And we can verify\n\n230\n00:04:59,430 --> 00:04:59,440\ngot two crawls now. And we can verify\n \n\n231\n00:04:59,440 --> 00:05:01,629\ngot two crawls now. And we can verify\nthat by heading into the crawls\n\n232\n00:05:01,629 --> 00:05:01,639\nthat by heading into the crawls\n \n\n233\n00:05:01,639 --> 00:05:04,070\nthat by heading into the crawls\ndirectory. An example will just have one\n\n234\n00:05:04,070 --> 00:05:04,080\ndirectory. An example will just have one\n \n\n235\n00:05:04,080 --> 00:05:06,950\ndirectory. An example will just have one\nfile. It's the index and Pragmar will\n\n236\n00:05:06,950 --> 00:05:06,960\nfile. It's the index and Pragmar will\n \n\n237\n00:05:06,960 --> 00:05:08,270\nfile. It's the index and Pragmar will\nhave quite a\n\n238\n00:05:08,270 --> 00:05:08,280\nhave quite a\n \n\n239\n00:05:08,280 --> 00:05:12,189\nhave quite a\nfew and that's exactly what we expected.\n\n240\n00:05:12,189 --> 00:05:12,199\nfew and that's exactly what we expected.\n \n\n241\n00:05:12,199 --> 00:05:15,390\nfew and that's exactly what we expected.\nSo this is all looking very very\n\n242\n00:05:15,390 --> 00:05:15,400\nSo this is all looking very very\n \n\n243\n00:05:15,400 --> 00:05:17,350\nSo this is all looking very very\ngood. And the next thing we're going to\n\n244\n00:05:17,350 --> 00:05:17,360\ngood. And the next thing we're going to\n \n\n245\n00:05:17,360 --> 00:05:19,189\ngood. And the next thing we're going to\ndo is configure cloud so that we can\n\n246\n00:05:19,189 --> 00:05:19,199\ndo is configure cloud so that we can\n \n\n247\n00:05:19,199 --> 00:05:21,230\ndo is configure cloud so that we can\naccess all this\n\n248\n00:05:21,230 --> 00:05:21,240\naccess all this\n \n\n249\n00:05:21,240 --> 00:05:24,070\naccess all this\ndata. So we're coming into claude cold.\n\n250\n00:05:24,070 --> 00:05:24,080\ndata. So we're coming into claude cold.\n \n\n251\n00:05:24,080 --> 00:05:25,710\ndata. So we're coming into claude cold.\nWe don't have any MCP servers\n\n252\n00:05:25,710 --> 00:05:25,720\nWe don't have any MCP servers\n \n\n253\n00:05:25,720 --> 00:05:27,749\nWe don't have any MCP servers\nconfigured. And the way you can tell\n\n254\n00:05:27,749 --> 00:05:27,759\nconfigured. And the way you can tell\n \n\n255\n00:05:27,759 --> 00:05:29,510\nconfigured. And the way you can tell\nthat is that you'll only have two\n\n256\n00:05:29,510 --> 00:05:29,520\nthat is that you'll only have two\n \n\n257\n00:05:29,520 --> 00:05:31,749\nthat is that you'll only have two\nbuttons here. You have search and tools\n\n258\n00:05:31,749 --> 00:05:31,759\nbuttons here. You have search and tools\n \n\n259\n00:05:31,759 --> 00:05:34,230\nbuttons here. You have search and tools\nand uh this uploads button. If we had\n\n260\n00:05:34,230 --> 00:05:34,240\nand uh this uploads button. If we had\n \n\n261\n00:05:34,240 --> 00:05:36,629\nand uh this uploads button. If we had\nMCP servers, we'd have a button with a\n\n262\n00:05:36,629 --> 00:05:36,639\nMCP servers, we'd have a button with a\n \n\n263\n00:05:36,639 --> 00:05:38,629\nMCP servers, we'd have a button with a\nhammer on it. And we're going to get\n\n264\n00:05:38,629 --> 00:05:38,639\nhammer on it. And we're going to get\n \n\n265\n00:05:38,639 --> 00:05:40,629\nhammer on it. And we're going to get\nthere. Believe me, we're going to get\n\n266\n00:05:40,629 --> 00:05:40,639\nthere. Believe me, we're going to get\n \n\n267\n00:05:40,639 --> 00:05:45,790\nthere. Believe me, we're going to get\nthere. So, uh, let's go to file,\n\n268\n00:05:45,790 --> 00:05:45,800\nthere. So, uh, let's go to file,\n \n\n269\n00:05:45,800 --> 00:05:48,790\nthere. So, uh, let's go to file,\nsettings, developer, and edit config.\n\n270\n00:05:48,790 --> 00:05:48,800\nsettings, developer, and edit config.\n \n\n271\n00:05:48,800 --> 00:05:51,629\nsettings, developer, and edit config.\nThis is going to be where we add our MCP\n\n272\n00:05:51,629 --> 00:05:51,639\nThis is going to be where we add our MCP\n \n\n273\n00:05:51,639 --> 00:05:54,189\nThis is going to be where we add our MCP\nserver. So, I'm going to open with\n\n274\n00:05:54,189 --> 00:05:54,199\nserver. So, I'm going to open with\n \n\n275\n00:05:54,199 --> 00:05:56,550\nserver. So, I'm going to open with\ncode. So, I'm going to head back to our\n\n276\n00:05:56,550 --> 00:05:56,560\ncode. So, I'm going to head back to our\n \n\n277\n00:05:56,560 --> 00:05:59,590\ncode. So, I'm going to head back to our\nMCP configuration uh, with our Katana\n\n278\n00:05:59,590 --> 00:05:59,600\nMCP configuration uh, with our Katana\n \n\n279\n00:05:59,600 --> 00:06:01,830\nMCP configuration uh, with our Katana\ntab, and we're just going to look at\n\n280\n00:06:01,830 --> 00:06:01,840\ntab, and we're just going to look at\n \n\n281\n00:06:01,840 --> 00:06:03,909\ntab, and we're just going to look at\nthis for a second. So, we're going to\n\n282\n00:06:03,909 --> 00:06:03,919\nthis for a second. So, we're going to\n \n\n283\n00:06:03,919 --> 00:06:06,309\nthis for a second. So, we're going to\nuse it. Uh, we're going to change the\n\n284\n00:06:06,309 --> 00:06:06,319\nuse it. Uh, we're going to change the\n \n\n285\n00:06:06,319 --> 00:06:09,350\nuse it. Uh, we're going to change the\ncommand. For Windows, all we need is MCP\n\n286\n00:06:09,350 --> 00:06:09,360\ncommand. For Windows, all we need is MCP\n \n\n287\n00:06:09,360 --> 00:06:12,150\ncommand. For Windows, all we need is MCP\nserver webcrawl. Uh, for Mac OS, you're\n\n288\n00:06:12,150 --> 00:06:12,160\nserver webcrawl. Uh, for Mac OS, you're\n \n\n289\n00:06:12,160 --> 00:06:14,870\nserver webcrawl. Uh, for Mac OS, you're\ngoing to need the full path to MCP\n\n290\n00:06:14,870 --> 00:06:14,880\ngoing to need the full path to MCP\n \n\n291\n00:06:14,880 --> 00:06:15,790\ngoing to need the full path to MCP\nserver\n\n292\n00:06:15,790 --> 00:06:15,800\nserver\n \n\n293\n00:06:15,800 --> 00:06:18,390\nserver\nwebcrawl. And for the data source arg,\n\n294\n00:06:18,390 --> 00:06:18,400\nwebcrawl. And for the data source arg,\n \n\n295\n00:06:18,400 --> 00:06:22,029\nwebcrawl. And for the data source arg,\nI'm going to use this directory right\n\n296\n00:06:22,029 --> 00:06:22,039\nI'm going to use this directory right\n \n\n297\n00:06:22,039 --> 00:06:25,870\nI'm going to use this directory right\nhere. So, let me assemble\n\n298\n00:06:25,870 --> 00:06:25,880\nhere. So, let me assemble\n \n\n299\n00:06:25,880 --> 00:06:29,590\nhere. So, let me assemble\nthat. Put that in there. Save it. Now,\n\n300\n00:06:29,590 --> 00:06:29,600\nthat. Put that in there. Save it. Now,\n \n\n301\n00:06:29,600 --> 00:06:31,670\nthat. Put that in there. Save it. Now,\nwe got to head back to Claude. We got to\n\n302\n00:06:31,670 --> 00:06:31,680\nwe got to head back to Claude. We got to\n \n\n303\n00:06:31,680 --> 00:06:33,110\nwe got to head back to Claude. We got to\nclose it up.\n\n304\n00:06:33,110 --> 00:06:33,120\nclose it up.\n \n\n305\n00:06:33,120 --> 00:06:36,510\nclose it up.\nSo, in order to reload the\n\n306\n00:06:36,510 --> 00:06:36,520\nSo, in order to reload the\n \n\n307\n00:06:36,520 --> 00:06:39,189\nSo, in order to reload the\nconf, and this is really important for\n\n308\n00:06:39,189 --> 00:06:39,199\nconf, and this is really important for\n \n\n309\n00:06:39,199 --> 00:06:41,029\nconf, and this is really important for\nWindows at least, you need to exit.\n\n310\n00:06:41,029 --> 00:06:41,039\nWindows at least, you need to exit.\n \n\n311\n00:06:41,039 --> 00:06:42,830\nWindows at least, you need to exit.\nThere's two options, close and\n\n312\n00:06:42,830 --> 00:06:42,840\nThere's two options, close and\n \n\n313\n00:06:42,840 --> 00:06:45,270\nThere's two options, close and\nexit. If you just close, it's not going\n\n314\n00:06:45,270 --> 00:06:45,280\nexit. If you just close, it's not going\n \n\n315\n00:06:45,280 --> 00:06:48,430\nexit. If you just close, it's not going\nto reload the conf. So, make sure you\n\n316\n00:06:48,430 --> 00:06:48,440\nto reload the conf. So, make sure you\n \n\n317\n00:06:48,440 --> 00:06:50,749\nto reload the conf. So, make sure you\nexit, come\n\n318\n00:06:50,749 --> 00:06:50,759\nexit, come\n \n\n319\n00:06:50,759 --> 00:06:52,990\nexit, come\nback, and look at\n\n320\n00:06:52,990 --> 00:06:53,000\nback, and look at\n \n\n321\n00:06:53,000 --> 00:06:55,749\nback, and look at\nhere. Two MCP tools available. What\n\n322\n00:06:55,749 --> 00:06:55,759\nhere. Two MCP tools available. What\n \n\n323\n00:06:55,759 --> 00:06:57,670\nhere. Two MCP tools available. What\ncould those be? Just a heads up, the\n\n324\n00:06:57,670 --> 00:06:57,680\ncould those be? Just a heads up, the\n \n\n325\n00:06:57,680 --> 00:07:00,309\ncould those be? Just a heads up, the\ncloud desktop uh interface has changed\n\n326\n00:07:00,309 --> 00:07:00,319\ncloud desktop uh interface has changed\n \n\n327\n00:07:00,319 --> 00:07:04,390\ncloud desktop uh interface has changed\nsince I recorded the uh demo. And MCP\n\n328\n00:07:04,390 --> 00:07:04,400\nsince I recorded the uh demo. And MCP\n \n\n329\n00:07:04,400 --> 00:07:07,189\nsince I recorded the uh demo. And MCP\nservers are now under search and tools.\n\n330\n00:07:07,189 --> 00:07:07,199\nservers are now under search and tools.\n \n\n331\n00:07:07,199 --> 00:07:09,270\nservers are now under search and tools.\nSo like right here, and they can be\n\n332\n00:07:09,270 --> 00:07:09,280\nSo like right here, and they can be\n \n\n333\n00:07:09,280 --> 00:07:12,790\nSo like right here, and they can be\ntoggled on and off, but the button that\n\n334\n00:07:12,790 --> 00:07:12,800\ntoggled on and off, but the button that\n \n\n335\n00:07:12,800 --> 00:07:14,790\ntoggled on and off, but the button that\nI'm referring to, the hammer button that\n\n336\n00:07:14,790 --> 00:07:14,800\nI'm referring to, the hammer button that\n \n\n337\n00:07:14,800 --> 00:07:17,749\nI'm referring to, the hammer button that\nI'm getting excited about perhaps is no\n\n338\n00:07:17,749 --> 00:07:17,759\nI'm getting excited about perhaps is no\n \n\n339\n00:07:17,759 --> 00:07:20,710\nI'm getting excited about perhaps is no\nlonger part of the interface. So as of\n\n340\n00:07:20,710 --> 00:07:20,720\nlonger part of the interface. So as of\n \n\n341\n00:07:20,720 --> 00:07:23,830\nlonger part of the interface. So as of\nMay 25, uh this is how it works. things\n\n342\n00:07:23,830 --> 00:07:23,840\nMay 25, uh this is how it works. things\n \n\n343\n00:07:23,840 --> 00:07:25,990\nMay 25, uh this is how it works. things\nare in flux, so it could change again.\n\n344\n00:07:25,990 --> 00:07:26,000\nare in flux, so it could change again.\n \n\n345\n00:07:26,000 --> 00:07:27,510\nare in flux, so it could change again.\nBut, uh, just wanted to give you that\n\n346\n00:07:27,510 --> 00:07:27,520\nBut, uh, just wanted to give you that\n \n\n347\n00:07:27,520 --> 00:07:28,990\nBut, uh, just wanted to give you that\nheads\n\n348\n00:07:28,990 --> 00:07:29,000\nheads\n \n\n349\n00:07:29,000 --> 00:07:31,189\nheads\nup. The first thing I'm going to do is\n\n350\n00:07:31,189 --> 00:07:31,199\nup. The first thing I'm going to do is\n \n\n351\n00:07:31,199 --> 00:07:38,350\nup. The first thing I'm going to do is\njust ask Claude to list crawled\n\n352\n00:07:38,350 --> 00:07:38,360\n\n \n\n353\n00:07:38,360 --> 00:07:41,350\n\nsites. And we should get back Pragmar\n\n354\n00:07:41,350 --> 00:07:41,360\nsites. And we should get back Pragmar\n \n\n355\n00:07:41,360 --> 00:07:50,990\nsites. And we should get back Pragmar\nand example if everything's\n\n356\n00:07:50,990 --> 00:07:51,000\n\n \n\n357\n00:07:51,000 --> 00:07:53,510\n\nworking. Now, listing sites is pretty\n\n358\n00:07:53,510 --> 00:07:53,520\nworking. Now, listing sites is pretty\n \n\n359\n00:07:53,520 --> 00:07:55,350\nworking. Now, listing sites is pretty\nuseful and it's certainly useful for the\n\n360\n00:07:55,350 --> 00:07:55,360\nuseful and it's certainly useful for the\n \n\n361\n00:07:55,360 --> 00:07:58,550\nuseful and it's certainly useful for the\nLLM in terms of what it has access to,\n\n362\n00:07:58,550 --> 00:07:58,560\nLLM in terms of what it has access to,\n \n\n363\n00:07:58,560 --> 00:08:00,950\nLLM in terms of what it has access to,\nbut for our purposes,\n\n364\n00:08:00,950 --> 00:08:00,960\nbut for our purposes,\n \n\n365\n00:08:00,960 --> 00:08:03,510\nbut for our purposes,\nuh, search is really going to be the the\n\n366\n00:08:03,510 --> 00:08:03,520\nuh, search is really going to be the the\n \n\n367\n00:08:03,520 --> 00:08:07,029\nuh, search is really going to be the the\nworkhorse of this server. And, um, it\n\n368\n00:08:07,029 --> 00:08:07,039\nworkhorse of this server. And, um, it\n \n\n369\n00:08:07,039 --> 00:08:09,189\nworkhorse of this server. And, um, it\nsays Claude to find all the help pages\n\n370\n00:08:09,189 --> 00:08:09,199\nsays Claude to find all the help pages\n \n\n371\n00:08:09,199 --> 00:08:11,909\nsays Claude to find all the help pages\non the site and tell me how they're\n\n372\n00:08:11,909 --> 00:08:11,919\non the site and tell me how they're\n \n\n373\n00:08:11,919 --> 00:08:22,710\non the site and tell me how they're\ndifferent.\n\n374\n00:08:22,710 --> 00:08:22,720\n\n \n\n375\n00:08:22,720 --> 00:08:24,070\n\nAnd this is going to pop up a\n\n376\n00:08:24,070 --> 00:08:24,080\nAnd this is going to pop up a\n \n\n377\n00:08:24,080 --> 00:08:25,670\nAnd this is going to pop up a\npermission. So the first permission I\n\n378\n00:08:25,670 --> 00:08:25,680\npermission. So the first permission I\n \n\n379\n00:08:25,680 --> 00:08:27,510\npermission. So the first permission I\ngave was to list sites. This permission\n\n380\n00:08:27,510 --> 00:08:27,520\ngave was to list sites. This permission\n \n\n381\n00:08:27,520 --> 00:08:30,150\ngave was to list sites. This permission\nis to search. We're just going to allow\n\n382\n00:08:30,150 --> 00:08:30,160\nis to search. We're just going to allow\n \n\n383\n00:08:30,160 --> 00:08:45,590\nis to search. We're just going to allow\nit.\n\n384\n00:08:45,590 --> 00:08:45,600\n\n \n\n385\n00:08:45,600 --> 00:08:57,630\n\nSo you can see the gears\n\n386\n00:08:57,630 --> 00:08:57,640\n\n \n\n387\n00:08:57,640 --> 00:08:59,990\n\nturning. Found the support page even\n\n388\n00:08:59,990 --> 00:09:00,000\nturning. Found the support page even\n \n\n389\n00:09:00,000 --> 00:09:01,430\nturning. Found the support page even\nthough it's not help. The other ones\n\n390\n00:09:01,430 --> 00:09:01,440\nthough it's not help. The other ones\n \n\n391\n00:09:01,440 --> 00:09:04,949\nthough it's not help. The other ones\nwere help.\n\n392\n00:09:04,949 --> 00:09:04,959\n\n \n\n393\n00:09:04,959 --> 00:09:07,670\n\nSo just to give you another example, uh\n\n394\n00:09:07,670 --> 00:09:07,680\nSo just to give you another example, uh\n \n\n395\n00:09:07,680 --> 00:09:11,310\nSo just to give you another example, uh\nI could say take the\n\n396\n00:09:11,310 --> 00:09:11,320\nI could say take the\n \n\n397\n00:09:11,320 --> 00:09:13,550\nI could say take the\nappstat help\n\n398\n00:09:13,550 --> 00:09:13,560\nappstat help\n \n\n399\n00:09:13,560 --> 00:09:17,710\nappstat help\npage and\n\n400\n00:09:17,710 --> 00:09:17,720\npage and\n \n\n401\n00:09:17,720 --> 00:09:19,230\npage and\nanalyze\n\n402\n00:09:19,230 --> 00:09:19,240\nanalyze\n \n\n403\n00:09:19,240 --> 00:09:23,949\nanalyze\nheadings for\n\n404\n00:09:23,949 --> 00:09:23,959\n\n \n\n405\n00:09:23,959 --> 00:09:26,470\n\nSEO. Claude's going to take that back.\n\n406\n00:09:26,470 --> 00:09:26,480\nSEO. Claude's going to take that back.\n \n\n407\n00:09:26,480 --> 00:09:28,870\nSEO. Claude's going to take that back.\nClaude already has the content. So\n\n408\n00:09:28,870 --> 00:09:28,880\nClaude already has the content. So\n \n\n409\n00:09:28,880 --> 00:09:30,710\nClaude already has the content. So\ntechnically I don't think Claude needs\n\n410\n00:09:30,710 --> 00:09:30,720\ntechnically I don't think Claude needs\n \n\n411\n00:09:30,720 --> 00:09:32,590\ntechnically I don't think Claude needs\nto go back,\n\n412\n00:09:32,590 --> 00:09:32,600\nto go back,\n \n\n413\n00:09:32,600 --> 00:09:36,430\nto go back,\nbut did\n\n414\n00:09:36,430 --> 00:09:36,440\n\n \n\n415\n00:09:36,440 --> 00:09:39,790\n\nanyway. So now you got your little SEO\n\n416\n00:09:39,790 --> 00:09:39,800\nanyway. So now you got your little SEO\n \n\n417\n00:09:39,800 --> 00:09:41,910\nanyway. So now you got your little SEO\nreport. Can really do anything you want\n\n418\n00:09:41,910 --> 00:09:41,920\nreport. Can really do anything you want\n \n\n419\n00:09:41,920 --> 00:09:44,350\nreport. Can really do anything you want\nwith this. It can be technical, it can\n\n420\n00:09:44,350 --> 00:09:44,360\nwith this. It can be technical, it can\n \n\n421\n00:09:44,360 --> 00:09:48,070\nwith this. It can be technical, it can\nbe more markety like\n\n422\n00:09:48,070 --> 00:09:48,080\nbe more markety like\n \n\n423\n00:09:48,080 --> 00:09:49,990\nbe more markety like\nuh the website is obviously useful to\n\n424\n00:09:49,990 --> 00:09:50,000\nuh the website is obviously useful to\n \n\n425\n00:09:50,000 --> 00:09:51,670\nuh the website is obviously useful to\nmany many people for many different\n\n426\n00:09:51,670 --> 00:09:51,680\nmany many people for many different\n \n\n427\n00:09:51,680 --> 00:09:54,790\nmany many people for many different\nreasons. So having this level of access,\n\n428\n00:09:54,790 --> 00:09:54,800\nreasons. So having this level of access,\n \n\n429\n00:09:54,800 --> 00:09:57,829\nreasons. So having this level of access,\nI think I think it's useful. So that's\n\n430\n00:09:57,829 --> 00:09:57,839\nI think I think it's useful. So that's\n \n\n431\n00:09:57,839 --> 00:10:01,150\nI think I think it's useful. So that's\nthat's an overview of MCP server web\n\n432\n00:10:01,150 --> 00:10:01,160\nthat's an overview of MCP server web\n \n\n433\n00:10:01,160 --> 00:10:04,470\nthat's an overview of MCP server web\ncrawl. Give it a shot and hopefully this\n\n434\n00:10:04,470 --> 00:10:04,480\ncrawl. Give it a shot and hopefully this\n \n\n435\n00:10:04,480 --> 00:10:06,710\ncrawl. Give it a shot and hopefully this\nis exactly what you're looking for.\n\n436\n00:10:06,710 --> 00:10:06,720\nis exactly what you're looking for.\n \n\n437\n00:10:06,720 --> 00:10:09,630\nis exactly what you're looking for.\nAnyways, thanks for watching and uh\n\n438\n00:10:09,630 --> 00:10:09,640\nAnyways, thanks for watching and uh\n \n\n439\n00:10:09,640 --> 00:10:13,880\nAnyways, thanks for watching and uh\nyeah, later on.", "1\n00:00:00,680 --> 00:00:06,869\n\n[Music]\n\n2\n00:00:06,869 --> 00:00:06,879\n\n \n\n3\n00:00:06,879 --> 00:00:10,070\n\nOkay, so Ben from Interobot here and\n\n4\n00:00:10,070 --> 00:00:10,080\nOkay, so Ben from Interobot here and\n \n\n5\n00:00:10,080 --> 00:00:13,830\nOkay, so Ben from Interobot here and\ntoday I am going over MCP server webcwl\n\n6\n00:00:13,830 --> 00:00:13,840\ntoday I am going over MCP server webcwl\n \n\n7\n00:00:13,840 --> 00:00:17,189\ntoday I am going over MCP server webcwl\nwhich lives between your LLM and your\n\n8\n00:00:17,189 --> 00:00:17,199\nwhich lives between your LLM and your\n \n\n9\n00:00:17,199 --> 00:00:19,670\nwhich lives between your LLM and your\nweb crawls or your web archives. So your\n\n10\n00:00:19,670 --> 00:00:19,680\nweb crawls or your web archives. So your\n \n\n11\n00:00:19,680 --> 00:00:22,310\nweb crawls or your web archives. So your\nLLM can either under your direction or\n\n12\n00:00:22,310 --> 00:00:22,320\nLLM can either under your direction or\n \n\n13\n00:00:22,320 --> 00:00:23,910\nLLM can either under your direction or\nautonomously\n\n14\n00:00:23,910 --> 00:00:23,920\nautonomously\n \n\n15\n00:00:23,920 --> 00:00:25,230\nautonomously\nuh do its\n\n16\n00:00:25,230 --> 00:00:25,240\nuh do its\n \n\n17\n00:00:25,240 --> 00:00:29,109\nuh do its\nresearch. So it's like search and doc.\n\n18\n00:00:29,109 --> 00:00:29,119\nresearch. So it's like search and doc.\n \n\n19\n00:00:29,119 --> 00:00:31,830\nresearch. So it's like search and doc.\nIt's got both. So, if we come over to\n\n20\n00:00:31,830 --> 00:00:31,840\nIt's got both. So, if we come over to\n \n\n21\n00:00:31,840 --> 00:00:34,069\nIt's got both. So, if we come over to\nClaude, I don't have any MCP servers set\n\n22\n00:00:34,069 --> 00:00:34,079\nClaude, I don't have any MCP servers set\n \n\n23\n00:00:34,079 --> 00:00:36,950\nClaude, I don't have any MCP servers set\nup right now. And the way that you can\n\n24\n00:00:36,950 --> 00:00:36,960\nup right now. And the way that you can\n \n\n25\n00:00:36,960 --> 00:00:39,110\nup right now. And the way that you can\ndetermine that is you can just look in\n\n26\n00:00:39,110 --> 00:00:39,120\ndetermine that is you can just look in\n \n\n27\n00:00:39,120 --> 00:00:41,750\ndetermine that is you can just look in\nthis area and you won't see a hammer\n\n28\n00:00:41,750 --> 00:00:41,760\nthis area and you won't see a hammer\n \n\n29\n00:00:41,760 --> 00:00:44,190\nthis area and you won't see a hammer\nbutton. You'll just see this\n\n30\n00:00:44,190 --> 00:00:44,200\nbutton. You'll just see this\n \n\n31\n00:00:44,200 --> 00:00:48,310\nbutton. You'll just see this\num these options here, these options,\n\n32\n00:00:48,310 --> 00:00:48,320\num these options here, these options,\n \n\n33\n00:00:48,320 --> 00:00:50,110\num these options here, these options,\nbut nothing\n\n34\n00:00:50,110 --> 00:00:50,120\nbut nothing\n \n\n35\n00:00:50,120 --> 00:00:53,350\nbut nothing\nmore. Just a heads up, the Cloud Desktop\n\n36\n00:00:53,350 --> 00:00:53,360\nmore. Just a heads up, the Cloud Desktop\n \n\n37\n00:00:53,360 --> 00:00:55,510\nmore. Just a heads up, the Cloud Desktop\nuh interface has changed since I\n\n38\n00:00:55,510 --> 00:00:55,520\nuh interface has changed since I\n \n\n39\n00:00:55,520 --> 00:00:59,830\nuh interface has changed since I\nrecorded the uh demo. and MCP servers\n\n40\n00:00:59,830 --> 00:00:59,840\nrecorded the uh demo. and MCP servers\n \n\n41\n00:00:59,840 --> 00:01:02,630\nrecorded the uh demo. and MCP servers\nare now under search and tools. So like\n\n42\n00:01:02,630 --> 00:01:02,640\nare now under search and tools. So like\n \n\n43\n00:01:02,640 --> 00:01:05,550\nare now under search and tools. So like\nright here, they can be toggled on and\n\n44\n00:01:05,550 --> 00:01:05,560\nright here, they can be toggled on and\n \n\n45\n00:01:05,560 --> 00:01:08,390\nright here, they can be toggled on and\noff, but the button that I'm referring\n\n46\n00:01:08,390 --> 00:01:08,400\noff, but the button that I'm referring\n \n\n47\n00:01:08,400 --> 00:01:10,310\noff, but the button that I'm referring\nto, the hammer button that I'm getting\n\n48\n00:01:10,310 --> 00:01:10,320\nto, the hammer button that I'm getting\n \n\n49\n00:01:10,320 --> 00:01:13,270\nto, the hammer button that I'm getting\nexcited about perhaps is no longer part\n\n50\n00:01:13,270 --> 00:01:13,280\nexcited about perhaps is no longer part\n \n\n51\n00:01:13,280 --> 00:01:17,510\nexcited about perhaps is no longer part\nof the interface. So as of May 25, uh\n\n52\n00:01:17,510 --> 00:01:17,520\nof the interface. So as of May 25, uh\n \n\n53\n00:01:17,520 --> 00:01:19,190\nof the interface. So as of May 25, uh\nthis is how it works. Things are in\n\n54\n00:01:19,190 --> 00:01:19,200\nthis is how it works. Things are in\n \n\n55\n00:01:19,200 --> 00:01:21,590\nthis is how it works. Things are in\nflux, so it could change again, but uh\n\n56\n00:01:21,590 --> 00:01:21,600\nflux, so it could change again, but uh\n \n\n57\n00:01:21,600 --> 00:01:24,230\nflux, so it could change again, but uh\njust wanted to give you that heads up. I\n\n58\n00:01:24,230 --> 00:01:24,240\njust wanted to give you that heads up. I\n \n\n59\n00:01:24,240 --> 00:01:26,870\njust wanted to give you that heads up. I\ndo not have any MCP servers yet and I'm\n\n60\n00:01:26,870 --> 00:01:26,880\ndo not have any MCP servers yet and I'm\n \n\n61\n00:01:26,880 --> 00:01:29,429\ndo not have any MCP servers yet and I'm\ngoing to show you how to set that up.\n\n62\n00:01:29,429 --> 00:01:29,439\ngoing to show you how to set that up.\n \n\n63\n00:01:29,439 --> 00:01:31,590\ngoing to show you how to set that up.\nSo, if you come over to the website,\n\n64\n00:01:31,590 --> 00:01:31,600\nSo, if you come over to the website,\n \n\n65\n00:01:31,600 --> 00:01:34,069\nSo, if you come over to the website,\nI've got my MCP configuration listed\n\n66\n00:01:34,069 --> 00:01:34,079\nI've got my MCP configuration listed\n \n\n67\n00:01:34,079 --> 00:01:36,910\nI've got my MCP configuration listed\nhere and this is wget and we're doing\n\n68\n00:01:36,910 --> 00:01:36,920\nhere and this is wget and we're doing\n \n\n69\n00:01:36,920 --> 00:01:41,350\nhere and this is wget and we're doing\nwget. So, we start with this as our base\n\n70\n00:01:41,350 --> 00:01:41,360\nwget. So, we start with this as our base\n \n\n71\n00:01:41,360 --> 00:01:43,830\nwget. So, we start with this as our base\nuh configuration and then you're going\n\n72\n00:01:43,830 --> 00:01:43,840\nuh configuration and then you're going\n \n\n73\n00:01:43,840 --> 00:01:45,550\nuh configuration and then you're going\nto want to\n\n74\n00:01:45,550 --> 00:01:45,560\nto want to\n \n\n75\n00:01:45,560 --> 00:01:48,310\nto want to\nset this path to wherever you're using\n\n76\n00:01:48,310 --> 00:01:48,320\nset this path to wherever you're using\n \n\n77\n00:01:48,320 --> 00:01:52,069\nset this path to wherever you're using\nit. me. I am doing this demo from a\n\n78\n00:01:52,069 --> 00:01:52,079\nit. me. I am doing this demo from a\n \n\n79\n00:01:52,079 --> 00:01:57,789\nit. me. I am doing this demo from a\nfolder on the\n\n80\n00:01:57,789 --> 00:01:57,799\n\n \n\n81\n00:01:57,799 --> 00:02:00,870\n\ndesktop. So, let's get this set up. Come\n\n82\n00:02:00,870 --> 00:02:00,880\ndesktop. So, let's get this set up. Come\n \n\n83\n00:02:00,880 --> 00:02:03,709\ndesktop. So, let's get this set up. Come\nover here. You say uh file settings\n\n84\n00:02:03,709 --> 00:02:03,719\nover here. You say uh file settings\n \n\n85\n00:02:03,719 --> 00:02:06,469\nover here. You say uh file settings\ndeveloper edit config. And that's going\n\n86\n00:02:06,469 --> 00:02:06,479\ndeveloper edit config. And that's going\n \n\n87\n00:02:06,479 --> 00:02:08,710\ndeveloper edit config. And that's going\nto select the uh cloud desktop config\n\n88\n00:02:08,710 --> 00:02:08,720\nto select the uh cloud desktop config\n \n\n89\n00:02:08,720 --> 00:02:09,550\nto select the uh cloud desktop config\nfor\n\n90\n00:02:09,550 --> 00:02:09,560\nfor\n \n\n91\n00:02:09,560 --> 00:02:13,550\nfor\nyou. I already have that up in my uh VS\n\n92\n00:02:13,550 --> 00:02:13,560\nyou. I already have that up in my uh VS\n \n\n93\n00:02:13,560 --> 00:02:17,910\nyou. I already have that up in my uh VS\nCode. So, I'm going to come over here.\n\n94\n00:02:17,910 --> 00:02:17,920\nCode. So, I'm going to come over here.\n \n\n95\n00:02:17,920 --> 00:02:20,110\nCode. So, I'm going to come over here.\nThe command is actually mcp server\n\n96\n00:02:20,110 --> 00:02:20,120\nThe command is actually mcp server\n \n\n97\n00:02:20,120 --> 00:02:23,430\nThe command is actually mcp server\nwebcwl args our crawler wget which is\n\n98\n00:02:23,430 --> 00:02:23,440\nwebcwl args our crawler wget which is\n \n\n99\n00:02:23,440 --> 00:02:25,470\nwebcwl args our crawler wget which is\ncorrect and then our data\n\n100\n00:02:25,470 --> 00:02:25,480\ncorrect and then our data\n \n\n101\n00:02:25,480 --> 00:02:29,190\ncorrect and then our data\nsource is this wget folder but this all\n\n102\n00:02:29,190 --> 00:02:29,200\nsource is this wget folder but this all\n \n\n103\n00:02:29,200 --> 00:02:30,190\nsource is this wget folder but this all\nlooks\n\n104\n00:02:30,190 --> 00:02:30,200\nlooks\n \n\n105\n00:02:30,200 --> 00:02:33,110\nlooks\ngood. So let's reflect on what we've\n\n106\n00:02:33,110 --> 00:02:33,120\ngood. So let's reflect on what we've\n \n\n107\n00:02:33,120 --> 00:02:36,790\ngood. So let's reflect on what we've\ndone so far. We've uh added our server\n\n108\n00:02:36,790 --> 00:02:36,800\ndone so far. We've uh added our server\n \n\n109\n00:02:36,800 --> 00:02:40,150\ndone so far. We've uh added our server\nto the cloud config. We have cloud\n\n110\n00:02:40,150 --> 00:02:40,160\nto the cloud config. We have cloud\n \n\n111\n00:02:40,160 --> 00:02:42,869\nto the cloud config. We have cloud\ndesktop running already. Now we got to\n\n112\n00:02:42,869 --> 00:02:42,879\ndesktop running already. Now we got to\n \n\n113\n00:02:42,879 --> 00:02:44,150\ndesktop running already. Now we got to\nmake sure that we have everything we\n\n114\n00:02:44,150 --> 00:02:44,160\nmake sure that we have everything we\n \n\n115\n00:02:44,160 --> 00:02:46,550\nmake sure that we have everything we\nneed to run the MCP server. What you'll\n\n116\n00:02:46,550 --> 00:02:46,560\nneed to run the MCP server. What you'll\n \n\n117\n00:02:46,560 --> 00:02:48,550\nneed to run the MCP server. What you'll\nneed is Python and you'll need the\n\n118\n00:02:48,550 --> 00:02:48,560\nneed is Python and you'll need the\n \n\n119\n00:02:48,560 --> 00:02:54,110\nneed is Python and you'll need the\nserver itself. I've already got\n\n120\n00:02:54,110 --> 00:02:54,120\n\n \n\n121\n00:02:54,120 --> 00:02:58,229\n\nPython. You install uh using this pip\n\n122\n00:02:58,229 --> 00:02:58,239\nPython. You install uh using this pip\n \n\n123\n00:02:58,239 --> 00:03:01,070\nPython. You install uh using this pip\ncommand. pip install MCP server\n\n124\n00:03:01,070 --> 00:03:01,080\ncommand. pip install MCP server\n \n\n125\n00:03:01,080 --> 00:03:03,670\ncommand. pip install MCP server\nwebcrawl. But for me, everything's\n\n126\n00:03:03,670 --> 00:03:03,680\nwebcrawl. But for me, everything's\n \n\n127\n00:03:03,680 --> 00:03:06,110\nwebcrawl. But for me, everything's\nalready satisfied. So once you have\n\n128\n00:03:06,110 --> 00:03:06,120\nalready satisfied. So once you have\n \n\n129\n00:03:06,120 --> 00:03:08,309\nalready satisfied. So once you have\nthat, we've got our requirements. So,\n\n130\n00:03:08,309 --> 00:03:08,319\nthat, we've got our requirements. So,\n \n\n131\n00:03:08,319 --> 00:03:10,390\nthat, we've got our requirements. So,\nwe've got Cloud Desktop, we've got\n\n132\n00:03:10,390 --> 00:03:10,400\nwe've got Cloud Desktop, we've got\n \n\n133\n00:03:10,400 --> 00:03:13,470\nwe've got Cloud Desktop, we've got\nPython, we've got MCP server\n\n134\n00:03:13,470 --> 00:03:13,480\nPython, we've got MCP server\n \n\n135\n00:03:13,480 --> 00:03:16,390\nPython, we've got MCP server\nwebcwl, and we've got our configuration\n\n136\n00:03:16,390 --> 00:03:16,400\nwebcwl, and we've got our configuration\n \n\n137\n00:03:16,400 --> 00:03:19,509\nwebcwl, and we've got our configuration\nin place. The problem is that the\n\n138\n00:03:19,509 --> 00:03:19,519\nin place. The problem is that the\n \n\n139\n00:03:19,519 --> 00:03:22,070\nin place. The problem is that the\nconfiguration is not going to get uh set\n\n140\n00:03:22,070 --> 00:03:22,080\nconfiguration is not going to get uh set\n \n\n141\n00:03:22,080 --> 00:03:24,390\nconfiguration is not going to get uh set\nup until we restart cloud. That means\n\n142\n00:03:24,390 --> 00:03:24,400\nup until we restart cloud. That means\n \n\n143\n00:03:24,400 --> 00:03:28,149\nup until we restart cloud. That means\nexit, not close. If you just close, the\n\n144\n00:03:28,149 --> 00:03:28,159\nexit, not close. If you just close, the\n \n\n145\n00:03:28,159 --> 00:03:31,710\nexit, not close. If you just close, the\nprocess will continue running in the\n\n146\n00:03:31,710 --> 00:03:31,720\nprocess will continue running in the\n \n\n147\n00:03:31,720 --> 00:03:34,149\nprocess will continue running in the\nbackground. We got our hammer. We got a\n\n148\n00:03:34,149 --> 00:03:34,159\nbackground. We got our hammer. We got a\n \n\n149\n00:03:34,159 --> 00:03:36,390\nbackground. We got our hammer. We got a\nnew button here. We've got two MCP tools\n\n150\n00:03:36,390 --> 00:03:36,400\nnew button here. We've got two MCP tools\n \n\n151\n00:03:36,400 --> 00:03:40,869\nnew button here. We've got two MCP tools\navailable.\n\n152\n00:03:40,869 --> 00:03:40,879\n\n \n\n153\n00:03:40,879 --> 00:03:42,380\n\nAt this point, we don't have any\n\n154\n00:03:42,380 --> 00:03:42,390\nAt this point, we don't have any\n \n\n155\n00:03:42,390 --> 00:03:43,710\nAt this point, we don't have any\n[Music]\n\n156\n00:03:43,710 --> 00:03:43,720\n[Music]\n \n\n157\n00:03:43,720 --> 00:03:46,149\n[Music]\ncrawls. I'm going to be putting my\n\n158\n00:03:46,149 --> 00:03:46,159\ncrawls. I'm going to be putting my\n \n\n159\n00:03:46,159 --> 00:03:49,869\ncrawls. I'm going to be putting my\ncrawls into this wget folder\n\n160\n00:03:49,869 --> 00:03:49,879\ncrawls into this wget folder\n \n\n161\n00:03:49,879 --> 00:03:59,309\ncrawls into this wget folder\nhere. So, I'll just ask Claude like\n\n162\n00:03:59,309 --> 00:03:59,319\n\n \n\n163\n00:03:59,319 --> 00:04:02,789\n\nuh if it can list the crawled sites and\n\n164\n00:04:02,789 --> 00:04:02,799\nuh if it can list the crawled sites and\n \n\n165\n00:04:02,799 --> 00:04:04,550\nuh if it can list the crawled sites and\nthere should be no crawled sites\n\n166\n00:04:04,550 --> 00:04:04,560\nthere should be no crawled sites\n \n\n167\n00:04:04,560 --> 00:04:08,710\nthere should be no crawled sites\navailable.\n\n168\n00:04:08,710 --> 00:04:08,720\n\n \n\n169\n00:04:08,720 --> 00:04:09,869\n\nAnd that's\n\n170\n00:04:09,869 --> 00:04:09,879\nAnd that's\n \n\n171\n00:04:09,879 --> 00:04:12,869\nAnd that's\ngood. We don't have them yet. So, at\n\n172\n00:04:12,869 --> 00:04:12,879\ngood. We don't have them yet. So, at\n \n\n173\n00:04:12,879 --> 00:04:14,470\ngood. We don't have them yet. So, at\nthis point, we're going to add our\n\n174\n00:04:14,470 --> 00:04:14,480\nthis point, we're going to add our\n \n\n175\n00:04:14,480 --> 00:04:16,789\nthis point, we're going to add our\ncrawls. And we're going to do that.\n\n176\n00:04:16,789 --> 00:04:16,799\ncrawls. And we're going to do that.\n \n\n177\n00:04:16,799 --> 00:04:19,349\ncrawls. And we're going to do that.\nWell, I'm on Windows, so I'm going to\n\n178\n00:04:19,349 --> 00:04:19,359\nWell, I'm on Windows, so I'm going to\n \n\n179\n00:04:19,359 --> 00:04:21,990\nWell, I'm on Windows, so I'm going to\nuse Ubuntu on Windows. In order to do\n\n180\n00:04:21,990 --> 00:04:22,000\nuse Ubuntu on Windows. In order to do\n \n\n181\n00:04:22,000 --> 00:04:23,350\nuse Ubuntu on Windows. In order to do\nthat, if you're on Mac, you'd use\n\n182\n00:04:23,350 --> 00:04:23,360\nthat, if you're on Mac, you'd use\n \n\n183\n00:04:23,360 --> 00:04:26,469\nthat, if you're on Mac, you'd use\nterminal. Terminal has wget, and Windows\n\n184\n00:04:26,469 --> 00:04:26,479\nterminal. Terminal has wget, and Windows\n \n\n185\n00:04:26,479 --> 00:04:28,230\nterminal. Terminal has wget, and Windows\ndoesn't have that natively. You have to\n\n186\n00:04:28,230 --> 00:04:28,240\ndoesn't have that natively. You have to\n \n\n187\n00:04:28,240 --> 00:04:32,230\ndoesn't have that natively. You have to\nactually add um either Ubuntu or or some\n\n188\n00:04:32,230 --> 00:04:32,240\nactually add um either Ubuntu or or some\n \n\n189\n00:04:32,240 --> 00:04:35,670\nactually add um either Ubuntu or or some\nother Linux distro in order to get it.\n\n190\n00:04:35,670 --> 00:04:35,680\nother Linux distro in order to get it.\n \n\n191\n00:04:35,680 --> 00:04:38,950\nother Linux distro in order to get it.\nBut I'm using Ubuntu and I am just going\n\n192\n00:04:38,950 --> 00:04:38,960\nBut I'm using Ubuntu and I am just going\n \n\n193\n00:04:38,960 --> 00:04:40,110\nBut I'm using Ubuntu and I am just going\nto\n\n194\n00:04:40,110 --> 00:04:40,120\nto\n \n\n195\n00:04:40,120 --> 00:04:46,590\nto\nuse the example right off the web for\n\n196\n00:04:46,590 --> 00:04:46,600\n\n \n\n197\n00:04:46,600 --> 00:04:48,590\n\nexample.com. There we\n\n198\n00:04:48,590 --> 00:04:48,600\nexample.com. There we\n \n\n199\n00:04:48,600 --> 00:04:52,350\nexample.com. There we\ngo. You can see it shows up in that\n\n200\n00:04:52,350 --> 00:04:52,360\ngo. You can see it shows up in that\n \n\n201\n00:04:52,360 --> 00:04:54,790\ngo. You can see it shows up in that\ndirectory. And while we're here, I'm\n\n202\n00:04:54,790 --> 00:04:54,800\ndirectory. And while we're here, I'm\n \n\n203\n00:04:54,800 --> 00:04:58,030\ndirectory. And while we're here, I'm\nalso going to get\n\n204\n00:04:58,030 --> 00:04:58,040\nalso going to get\n \n\n205\n00:04:58,040 --> 00:05:07,510\nalso going to get\num my site pragmar.com.\n\n206\n00:05:07,510 --> 00:05:07,520\n\n \n\n207\n00:05:07,520 --> 00:05:11,310\n\nSo now we have a couple sites in this\n\n208\n00:05:11,310 --> 00:05:11,320\nSo now we have a couple sites in this\n \n\n209\n00:05:11,320 --> 00:05:13,909\nSo now we have a couple sites in this\ndirectory and this directory is what is\n\n210\n00:05:13,909 --> 00:05:13,919\ndirectory and this directory is what is\n \n\n211\n00:05:13,919 --> 00:05:18,350\ndirectory and this directory is what is\nconfigured in our cloud desktop\n\n212\n00:05:18,350 --> 00:05:18,360\nconfigured in our cloud desktop\n \n\n213\n00:05:18,360 --> 00:05:21,390\nconfigured in our cloud desktop\nconfig.json and we're going to ask\n\n214\n00:05:21,390 --> 00:05:21,400\nconfig.json and we're going to ask\n \n\n215\n00:05:21,400 --> 00:05:29,790\nconfig.json and we're going to ask\nclaude can you check sites\n\n216\n00:05:29,790 --> 00:05:29,800\n\n \n\n217\n00:05:29,800 --> 00:05:32,790\n\nagain? We have our crawled sites.\n\n218\n00:05:32,790 --> 00:05:32,800\nagain? We have our crawled sites.\n \n\n219\n00:05:32,800 --> 00:05:35,029\nagain? We have our crawled sites.\nThey're in our demo directory and Claude\n\n220\n00:05:35,029 --> 00:05:35,039\nThey're in our demo directory and Claude\n \n\n221\n00:05:35,039 --> 00:05:37,270\nThey're in our demo directory and Claude\nis aware of them. So like uh with\n\n222\n00:05:37,270 --> 00:05:37,280\nis aware of them. So like uh with\n \n\n223\n00:05:37,280 --> 00:05:39,110\nis aware of them. So like uh with\npragmar.com, I know there's a page on\n\n224\n00:05:39,110 --> 00:05:39,120\npragmar.com, I know there's a page on\n \n\n225\n00:05:39,120 --> 00:05:40,390\npragmar.com, I know there's a page on\nthere called cubit. Well, there's\n\n226\n00:05:40,390 --> 00:05:40,400\nthere called cubit. Well, there's\n \n\n227\n00:05:40,400 --> 00:05:42,029\nthere called cubit. Well, there's\nactually three cubit\n\n228\n00:05:42,029 --> 00:05:42,039\nactually three cubit\n \n\n229\n00:05:42,039 --> 00:05:45,189\nactually three cubit\npages. Let's try and retrieve copy for\n\n230\n00:05:45,189 --> 00:05:45,199\npages. Let's try and retrieve copy for\n \n\n231\n00:05:45,199 --> 00:05:47,510\npages. Let's try and retrieve copy for\nthe primary cubit page. See what\n\n232\n00:05:47,510 --> 00:05:47,520\nthe primary cubit page. See what\n \n\n233\n00:05:47,520 --> 00:06:25,749\nthe primary cubit page. See what\nhappens.\n\n234\n00:06:25,749 --> 00:06:25,759\n\n \n\n235\n00:06:25,759 --> 00:06:27,830\n\nSo that's not so bad. Found it pretty\n\n236\n00:06:27,830 --> 00:06:27,840\nSo that's not so bad. Found it pretty\n \n\n237\n00:06:27,840 --> 00:06:30,390\nSo that's not so bad. Found it pretty\nquick. Sometimes it needs to do a few\n\n238\n00:06:30,390 --> 00:06:30,400\nquick. Sometimes it needs to do a few\n \n\n239\n00:06:30,400 --> 00:06:33,550\nquick. Sometimes it needs to do a few\nqueries to get where it needs to\n\n240\n00:06:33,550 --> 00:06:33,560\nqueries to get where it needs to\n \n\n241\n00:06:33,560 --> 00:06:37,270\nqueries to get where it needs to\ngo. If it starts going off, sometimes\n\n242\n00:06:37,270 --> 00:06:37,280\ngo. If it starts going off, sometimes\n \n\n243\n00:06:37,280 --> 00:06:41,110\ngo. If it starts going off, sometimes\nyou got to reel it in, stop it. Um, but\n\n244\n00:06:41,110 --> 00:06:41,120\nyou got to reel it in, stop it. Um, but\n \n\n245\n00:06:41,120 --> 00:06:45,150\nyou got to reel it in, stop it. Um, but\nhere's um here's a\n\n246\n00:06:45,150 --> 00:06:45,160\n\n \n\n247\n00:06:45,160 --> 00:06:49,629\n\nsummary, which is nice. Let's say\n\n248\n00:06:49,629 --> 00:06:49,639\nsummary, which is nice. Let's say\n \n\n249\n00:06:49,639 --> 00:06:54,749\nsummary, which is nice. Let's say\nuh summary is\n\n250\n00:06:54,749 --> 00:06:54,759\n\n \n\n251\n00:06:54,759 --> 00:07:04,670\n\nnice. What should I\n\n252\n00:07:04,670 --> 00:07:04,680\n\n \n\n253\n00:07:04,680 --> 00:07:08,230\n\ndo? So, yeah, you can you can uh use it\n\n254\n00:07:08,230 --> 00:07:08,240\ndo? So, yeah, you can you can uh use it\n \n\n255\n00:07:08,240 --> 00:07:10,230\ndo? So, yeah, you can you can uh use it\nfor SEO, you can use it for content\n\n256\n00:07:10,230 --> 00:07:10,240\nfor SEO, you can use it for content\n \n\n257\n00:07:10,240 --> 00:07:12,870\nfor SEO, you can use it for content\nstrategy, you can use it for marketing.\n\n258\n00:07:12,870 --> 00:07:12,880\nstrategy, you can use it for marketing.\n \n\n259\n00:07:12,880 --> 00:07:15,110\nstrategy, you can use it for marketing.\nSky's the limit here. Do whatever you\n\n260\n00:07:15,110 --> 00:07:15,120\nSky's the limit here. Do whatever you\n \n\n261\n00:07:15,120 --> 00:07:17,749\nSky's the limit here. Do whatever you\nwant with it. It's open source. You can\n\n262\n00:07:17,749 --> 00:07:17,759\nwant with it. It's open source. You can\n \n\n263\n00:07:17,759 --> 00:07:20,629\nwant with it. It's open source. You can\nuh modify it if you want. Put your own\n\n264\n00:07:20,629 --> 00:07:20,639\nuh modify it if you want. Put your own\n \n\n265\n00:07:20,639 --> 00:07:23,749\nuh modify it if you want. Put your own\ncrawler on it if you're a developer.\n\n266\n00:07:23,749 --> 00:07:23,759\ncrawler on it if you're a developer.\n \n\n267\n00:07:23,759 --> 00:07:26,950\ncrawler on it if you're a developer.\nIt's up to you. But um I hope you can\n\n268\n00:07:26,950 --> 00:07:26,960\nIt's up to you. But um I hope you can\n \n\n269\n00:07:26,960 --> 00:07:29,029\nIt's up to you. But um I hope you can\nget some use out of this thing. Take\n\n270\n00:07:29,029 --> 00:07:29,039\nget some use out of this thing. Take\n \n\n271\n00:07:29,039 --> 00:07:32,039\nget some use out of this thing. Take\ncare.", "1\n00:00:00,700 --> 00:00:06,950\n\n[Music]\n\n2\n00:00:06,950 --> 00:00:06,960\n[Music]\n \n\n3\n00:00:06,960 --> 00:00:09,110\n[Music]\nHey, this is Ben from Interabot and\n\n4\n00:00:09,110 --> 00:00:09,120\nHey, this is Ben from Interabot and\n \n\n5\n00:00:09,120 --> 00:00:11,190\nHey, this is Ben from Interabot and\ntoday I'm going to go over connecting\n\n6\n00:00:11,190 --> 00:00:11,200\ntoday I'm going to go over connecting\n \n\n7\n00:00:11,200 --> 00:00:13,589\ntoday I'm going to go over connecting\nyour work archives, which are single\n\n8\n00:00:13,589 --> 00:00:13,599\nyour work archives, which are single\n \n\n9\n00:00:13,599 --> 00:00:17,990\nyour work archives, which are single\nfile web archives to claude desktop uh\n\n10\n00:00:17,990 --> 00:00:18,000\nfile web archives to claude desktop uh\n \n\n11\n00:00:18,000 --> 00:00:21,109\nfile web archives to claude desktop uh\nvia MCP server webcrawl. And what MCP\n\n12\n00:00:21,109 --> 00:00:21,119\nvia MCP server webcrawl. And what MCP\n \n\n13\n00:00:21,119 --> 00:00:23,429\nvia MCP server webcrawl. And what MCP\nserver webcwl does is it acts as like a\n\n14\n00:00:23,429 --> 00:00:23,439\nserver webcwl does is it acts as like a\n \n\n15\n00:00:23,439 --> 00:00:25,509\nserver webcwl does is it acts as like a\nsearch and retrieval interface to your\n\n16\n00:00:25,509 --> 00:00:25,519\nsearch and retrieval interface to your\n \n\n17\n00:00:25,519 --> 00:00:27,990\nsearch and retrieval interface to your\nwork files so that claude can query\n\n18\n00:00:27,990 --> 00:00:28,000\nwork files so that claude can query\n \n\n19\n00:00:28,000 --> 00:00:30,109\nwork files so that claude can query\nagainst that. opens up a lot of\n\n20\n00:00:30,109 --> 00:00:30,119\nagainst that. opens up a lot of\n \n\n21\n00:00:30,119 --> 00:00:32,590\nagainst that. opens up a lot of\npossibilities, but let's just get into\n\n22\n00:00:32,590 --> 00:00:32,600\npossibilities, but let's just get into\n \n\n23\n00:00:32,600 --> 00:00:35,670\npossibilities, but let's just get into\nit. The warp file format, you can see\n\n24\n00:00:35,670 --> 00:00:35,680\nit. The warp file format, you can see\n \n\n25\n00:00:35,680 --> 00:00:37,270\nit. The warp file format, you can see\nthat there's a bunch of software that\n\n26\n00:00:37,270 --> 00:00:37,280\nthat there's a bunch of software that\n \n\n27\n00:00:37,280 --> 00:00:39,190\nthat there's a bunch of software that\ncan generate these. So, for this\n\n28\n00:00:39,190 --> 00:00:39,200\ncan generate these. So, for this\n \n\n29\n00:00:39,200 --> 00:00:41,389\ncan generate these. So, for this\ntutorial, we're going to be using uh\n\n30\n00:00:41,389 --> 00:00:41,399\ntutorial, we're going to be using uh\n \n\n31\n00:00:41,399 --> 00:00:43,670\ntutorial, we're going to be using uh\nwget, but I want to go over some of the\n\n32\n00:00:43,670 --> 00:00:43,680\nwget, but I want to go over some of the\n \n\n33\n00:00:43,680 --> 00:00:46,630\nwget, but I want to go over some of the\npros and cons of using warp versus using\n\n34\n00:00:46,630 --> 00:00:46,640\npros and cons of using warp versus using\n \n\n35\n00:00:46,640 --> 00:00:49,590\npros and cons of using warp versus using\na wget mirror because I also have\n\n36\n00:00:49,590 --> 00:00:49,600\na wget mirror because I also have\n \n\n37\n00:00:49,600 --> 00:00:51,990\na wget mirror because I also have\nanother tutorial about using wget in\n\n38\n00:00:51,990 --> 00:00:52,000\nanother tutorial about using wget in\n \n\n39\n00:00:52,000 --> 00:00:54,630\nanother tutorial about using wget in\nmirror mode. The difference is that\n\n40\n00:00:54,630 --> 00:00:54,640\nmirror mode. The difference is that\n \n\n41\n00:00:54,640 --> 00:00:57,910\nmirror mode. The difference is that\nmirror mode is a lot faster at archiving\n\n42\n00:00:57,910 --> 00:00:57,920\nmirror mode is a lot faster at archiving\n \n\n43\n00:00:57,920 --> 00:01:00,869\nmirror mode is a lot faster at archiving\na website. However, on the plus side,\n\n44\n00:01:00,869 --> 00:01:00,879\na website. However, on the plus side,\n \n\n45\n00:01:00,879 --> 00:01:03,430\na website. However, on the plus side,\nwhat warp does get is it gets the status\n\n46\n00:01:03,430 --> 00:01:03,440\nwhat warp does get is it gets the status\n \n\n47\n00:01:03,440 --> 00:01:06,630\nwhat warp does get is it gets the status\ncodes and it gets the headers where wget\n\n48\n00:01:06,630 --> 00:01:06,640\ncodes and it gets the headers where wget\n \n\n49\n00:01:06,640 --> 00:01:09,030\ncodes and it gets the headers where wget\nget mirror will not produce those. It\n\n50\n00:01:09,030 --> 00:01:09,040\nget mirror will not produce those. It\n \n\n51\n00:01:09,040 --> 00:01:11,670\nget mirror will not produce those. It\nwill just save files to the to the disk\n\n52\n00:01:11,670 --> 00:01:11,680\nwill just save files to the to the disk\n \n\n53\n00:01:11,680 --> 00:01:13,710\nwill just save files to the to the disk\nand then the indexer goes through the\n\n54\n00:01:13,710 --> 00:01:13,720\nand then the indexer goes through the\n \n\n55\n00:01:13,720 --> 00:01:15,990\nand then the indexer goes through the\nfiles. At that point, it's already lost\n\n56\n00:01:15,990 --> 00:01:16,000\nfiles. At that point, it's already lost\n \n\n57\n00:01:16,000 --> 00:01:19,670\nfiles. At that point, it's already lost\nthe status codes and the headers. To\n\n58\n00:01:19,670 --> 00:01:19,680\nthe status codes and the headers. To\n \n\n59\n00:01:19,680 --> 00:01:22,789\nthe status codes and the headers. To\nsummarize, wget warp is going to be more\n\n60\n00:01:22,789 --> 00:01:22,799\nsummarize, wget warp is going to be more\n \n\n61\n00:01:22,799 --> 00:01:25,429\nsummarize, wget warp is going to be more\ncomprehensive and wget mirror is going\n\n62\n00:01:25,429 --> 00:01:25,439\ncomprehensive and wget mirror is going\n \n\n63\n00:01:25,439 --> 00:01:29,190\ncomprehensive and wget mirror is going\nto be a faster crawl. So, pick your\n\n64\n00:01:29,190 --> 00:01:29,200\nto be a faster crawl. So, pick your\n \n\n65\n00:01:29,200 --> 00:01:32,789\nto be a faster crawl. So, pick your\npoison uh or use a different crawler.\n\n66\n00:01:32,789 --> 00:01:32,799\npoison uh or use a different crawler.\n \n\n67\n00:01:32,799 --> 00:01:34,550\npoison uh or use a different crawler.\nThe first thing you're going to need is\n\n68\n00:01:34,550 --> 00:01:34,560\nThe first thing you're going to need is\n \n\n69\n00:01:34,560 --> 00:01:37,030\nThe first thing you're going to need is\nuh the server runtime, which is Python.\n\n70\n00:01:37,030 --> 00:01:37,040\nuh the server runtime, which is Python.\n \n\n71\n00:01:37,040 --> 00:01:39,350\nuh the server runtime, which is Python.\nYou may already have this. If you don't,\n\n72\n00:01:39,350 --> 00:01:39,360\nYou may already have this. If you don't,\n \n\n73\n00:01:39,360 --> 00:01:41,590\nYou may already have this. If you don't,\njust head over to python.org. Run\n\n74\n00:01:41,590 --> 00:01:41,600\njust head over to python.org. Run\n \n\n75\n00:01:41,600 --> 00:01:43,030\njust head over to python.org. Run\nthrough the installer. If you're using\n\n76\n00:01:43,030 --> 00:01:43,040\nthrough the installer. If you're using\n \n\n77\n00:01:43,040 --> 00:01:45,190\nthrough the installer. If you're using\nWindows, there's a check box to add\n\n78\n00:01:45,190 --> 00:01:45,200\nWindows, there's a check box to add\n \n\n79\n00:01:45,200 --> 00:01:47,350\nWindows, there's a check box to add\nPython to path. You're going to want to\n\n80\n00:01:47,350 --> 00:01:47,360\nPython to path. You're going to want to\n \n\n81\n00:01:47,360 --> 00:01:49,030\nPython to path. You're going to want to\ncheck that.\n\n82\n00:01:49,030 --> 00:01:49,040\ncheck that.\n \n\n83\n00:01:49,040 --> 00:01:50,630\ncheck that.\nAnd once you've got that set up, we're\n\n84\n00:01:50,630 --> 00:01:50,640\nAnd once you've got that set up, we're\n \n\n85\n00:01:50,640 --> 00:01:52,149\nAnd once you've got that set up, we're\ngoing to head over to a shell. So for\n\n86\n00:01:52,149 --> 00:01:52,159\ngoing to head over to a shell. So for\n \n\n87\n00:01:52,159 --> 00:01:54,230\ngoing to head over to a shell. So for\nMac OS, this would be terminal. Uh for\n\n88\n00:01:54,230 --> 00:01:54,240\nMac OS, this would be terminal. Uh for\n \n\n89\n00:01:54,240 --> 00:01:57,230\nMac OS, this would be terminal. Uh for\nWindows, I'm using\n\n90\n00:01:57,230 --> 00:01:57,240\nWindows, I'm using\n \n\n91\n00:01:57,240 --> 00:01:59,190\nWindows, I'm using\nPowerShell. And from here, I'm just\n\n92\n00:01:59,190 --> 00:01:59,200\nPowerShell. And from here, I'm just\n \n\n93\n00:01:59,200 --> 00:02:03,870\nPowerShell. And from here, I'm just\ngoing to check that Python is on my\n\n94\n00:02:03,870 --> 00:02:03,880\n\n \n\n95\n00:02:03,880 --> 00:02:07,109\n\npath. And it is. And if we have Python,\n\n96\n00:02:07,109 --> 00:02:07,119\npath. And it is. And if we have Python,\n \n\n97\n00:02:07,119 --> 00:02:08,869\npath. And it is. And if we have Python,\nwe have pip. So from here, we're going\n\n98\n00:02:08,869 --> 00:02:08,879\nwe have pip. So from here, we're going\n \n\n99\n00:02:08,879 --> 00:02:11,430\nwe have pip. So from here, we're going\nto install MCP server webcrawl, which is\n\n100\n00:02:11,430 --> 00:02:11,440\nto install MCP server webcrawl, which is\n \n\n101\n00:02:11,440 --> 00:02:14,670\nto install MCP server webcrawl, which is\nthe uh MCP server. The way to do that is\n\n102\n00:02:14,670 --> 00:02:14,680\nthe uh MCP server. The way to do that is\n \n\n103\n00:02:14,680 --> 00:02:22,830\nthe uh MCP server. The way to do that is\npip install mcp server\n\n104\n00:02:22,830 --> 00:02:22,840\n\n \n\n105\n00:02:22,840 --> 00:02:24,790\n\nwebcwl and I've already got it\n\n106\n00:02:24,790 --> 00:02:24,800\nwebcwl and I've already got it\n \n\n107\n00:02:24,800 --> 00:02:26,670\nwebcwl and I've already got it\ninstalled. So it says requirements\n\n108\n00:02:26,670 --> 00:02:26,680\ninstalled. So it says requirements\n \n\n109\n00:02:26,680 --> 00:02:29,350\ninstalled. So it says requirements\nsatisfied. Hopefully this installs it\n\n110\n00:02:29,350 --> 00:02:29,360\nsatisfied. Hopefully this installs it\n \n\n111\n00:02:29,360 --> 00:02:32,830\nsatisfied. Hopefully this installs it\nfor you. The way to check is to say\n\n112\n00:02:32,830 --> 00:02:32,840\nfor you. The way to check is to say\n \n\n113\n00:02:32,840 --> 00:02:34,430\nfor you. The way to check is to say\nmcp\n\n114\n00:02:34,430 --> 00:02:34,440\nmcp\n \n\n115\n00:02:34,440 --> 00:02:39,390\nmcp\nserver\n\n116\n00:02:39,390 --> 00:02:39,400\n\n \n\n117\n00:02:39,400 --> 00:02:42,790\n\nwebcwl and it's there. So, we're good.\n\n118\n00:02:42,790 --> 00:02:42,800\nwebcwl and it's there. So, we're good.\n \n\n119\n00:02:42,800 --> 00:02:44,630\nwebcwl and it's there. So, we're good.\nSo, what does this mean? This means that\n\n120\n00:02:44,630 --> 00:02:44,640\nSo, what does this mean? This means that\n \n\n121\n00:02:44,640 --> 00:02:46,550\nSo, what does this mean? This means that\nour MCP server, the thing that sits\n\n122\n00:02:46,550 --> 00:02:46,560\nour MCP server, the thing that sits\n \n\n123\n00:02:46,560 --> 00:02:49,670\nour MCP server, the thing that sits\nbetween the web crawl and cloud desktop,\n\n124\n00:02:49,670 --> 00:02:49,680\nbetween the web crawl and cloud desktop,\n \n\n125\n00:02:49,680 --> 00:02:52,470\nbetween the web crawl and cloud desktop,\nis ready to serve. So, we've got a few\n\n126\n00:02:52,470 --> 00:02:52,480\nis ready to serve. So, we've got a few\n \n\n127\n00:02:52,480 --> 00:02:54,630\nis ready to serve. So, we've got a few\nmore steps. Uh we're going to need to\n\n128\n00:02:54,630 --> 00:02:54,640\nmore steps. Uh we're going to need to\n \n\n129\n00:02:54,640 --> 00:02:56,949\nmore steps. Uh we're going to need to\ngather some crawls and then we're going\n\n130\n00:02:56,949 --> 00:02:56,959\ngather some crawls and then we're going\n \n\n131\n00:02:56,959 --> 00:02:59,830\ngather some crawls and then we're going\nto need to configure cloud desktop to uh\n\n132\n00:02:59,830 --> 00:02:59,840\nto need to configure cloud desktop to uh\n \n\n133\n00:02:59,840 --> 00:03:04,190\nto need to configure cloud desktop to uh\nsearch and filter these\n\n134\n00:03:04,190 --> 00:03:04,200\n\n \n\n135\n00:03:04,200 --> 00:03:06,869\n\ncrawls. So, I'm on Windows. So my\n\n136\n00:03:06,869 --> 00:03:06,879\ncrawls. So, I'm on Windows. So my\n \n\n137\n00:03:06,879 --> 00:03:08,710\ncrawls. So, I'm on Windows. So my\nversion of wget is actually running on\n\n138\n00:03:08,710 --> 00:03:08,720\nversion of wget is actually running on\n \n\n139\n00:03:08,720 --> 00:03:10,949\nversion of wget is actually running on\nUbuntu on Windows, but if you're using\n\n140\n00:03:10,949 --> 00:03:10,959\nUbuntu on Windows, but if you're using\n \n\n141\n00:03:10,959 --> 00:03:12,869\nUbuntu on Windows, but if you're using\nMac OS, you're probably going to install\n\n142\n00:03:12,869 --> 00:03:12,879\nMac OS, you're probably going to install\n \n\n143\n00:03:12,879 --> 00:03:14,670\nMac OS, you're probably going to install\nWget through Brew if you haven't\n\n144\n00:03:14,670 --> 00:03:14,680\nWget through Brew if you haven't\n \n\n145\n00:03:14,680 --> 00:03:17,190\nWget through Brew if you haven't\nalready. You can check if Wget is\n\n146\n00:03:17,190 --> 00:03:17,200\nalready. You can check if Wget is\n \n\n147\n00:03:17,200 --> 00:03:18,910\nalready. You can check if Wget is\navailable with a\n\n148\n00:03:18,910 --> 00:03:18,920\navailable with a\n \n\n149\n00:03:18,920 --> 00:03:22,430\navailable with a\nuh wget\n\n150\n00:03:22,430 --> 00:03:22,440\nuh wget\n \n\n151\n00:03:22,440 --> 00:03:25,110\nuh wget\nversion. So at this point, we have all\n\n152\n00:03:25,110 --> 00:03:25,120\nversion. So at this point, we have all\n \n\n153\n00:03:25,120 --> 00:03:26,470\nversion. So at this point, we have all\nof our software that we're going to\n\n154\n00:03:26,470 --> 00:03:26,480\nof our software that we're going to\n \n\n155\n00:03:26,480 --> 00:03:28,790\nof our software that we're going to\nneed. We have our server runtime, we\n\n156\n00:03:28,790 --> 00:03:28,800\nneed. We have our server runtime, we\n \n\n157\n00:03:28,800 --> 00:03:31,990\nneed. We have our server runtime, we\nhave our server, we have Cloud Desktop,\n\n158\n00:03:31,990 --> 00:03:32,000\nhave our server, we have Cloud Desktop,\n \n\n159\n00:03:32,000 --> 00:03:34,550\nhave our server, we have Cloud Desktop,\nand we have Wgetit. So, we're just going\n\n160\n00:03:34,550 --> 00:03:34,560\nand we have Wgetit. So, we're just going\n \n\n161\n00:03:34,560 --> 00:03:37,030\nand we have Wgetit. So, we're just going\nto grab some crawls to work with for the\n\n162\n00:03:37,030 --> 00:03:37,040\nto grab some crawls to work with for the\n \n\n163\n00:03:37,040 --> 00:03:39,030\nto grab some crawls to work with for the\ndemo. And to do that, I'm going to head\n\n164\n00:03:39,030 --> 00:03:39,040\ndemo. And to do that, I'm going to head\n \n\n165\n00:03:39,040 --> 00:03:42,390\ndemo. And to do that, I'm going to head\nback to the MCP server webcwl page, head\n\n166\n00:03:42,390 --> 00:03:42,400\nback to the MCP server webcwl page, head\n \n\n167\n00:03:42,400 --> 00:03:45,430\nback to the MCP server webcwl page, head\ndown to the configuration, select warp,\n\n168\n00:03:45,430 --> 00:03:45,440\ndown to the configuration, select warp,\n \n\n169\n00:03:45,440 --> 00:03:47,190\ndown to the configuration, select warp,\nand uh you can see that there's a\n\n170\n00:03:47,190 --> 00:03:47,200\nand uh you can see that there's a\n \n\n171\n00:03:47,200 --> 00:03:48,670\nand uh you can see that there's a\nconfiguration that we're going to add to\n\n172\n00:03:48,670 --> 00:03:48,680\nconfiguration that we're going to add to\n \n\n173\n00:03:48,680 --> 00:03:51,190\nconfiguration that we're going to add to\nClaude. But before we do that, let's get\n\n174\n00:03:51,190 --> 00:03:51,200\nClaude. But before we do that, let's get\n \n\n175\n00:03:51,200 --> 00:03:52,990\nClaude. But before we do that, let's get\nsome data.\n\n176\n00:03:52,990 --> 00:03:53,000\nsome data.\n \n\n177\n00:03:53,000 --> 00:03:56,710\nsome data.\nSo, let's head back here.\n\n178\n00:03:56,710 --> 00:03:56,720\nSo, let's head back here.\n \n\n179\n00:03:56,720 --> 00:04:13,710\nSo, let's head back here.\nSo we'll grab example and we'll\n\n180\n00:04:13,710 --> 00:04:13,720\n\n \n\n181\n00:04:13,720 --> 00:04:17,909\n\ngrab. So at this point we have two work\n\n182\n00:04:17,909 --> 00:04:17,919\ngrab. So at this point we have two work\n \n\n183\n00:04:17,919 --> 00:04:20,949\ngrab. So at this point we have two work\nfiles. They're work.gz which is gzip.\n\n184\n00:04:20,949 --> 00:04:20,959\nfiles. They're work.gz which is gzip.\n \n\n185\n00:04:20,959 --> 00:04:23,749\nfiles. They're work.gz which is gzip.\nThat's fine. And in Windows terms that\n\n186\n00:04:23,749 --> 00:04:23,759\nThat's fine. And in Windows terms that\n \n\n187\n00:04:23,759 --> 00:04:27,270\nThat's fine. And in Windows terms that\nis on the desktop. So we can just grab\n\n188\n00:04:27,270 --> 00:04:27,280\nis on the desktop. So we can just grab\n \n\n189\n00:04:27,280 --> 00:04:29,150\nis on the desktop. So we can just grab\nthat. We'll work with that\n\n190\n00:04:29,150 --> 00:04:29,160\nthat. We'll work with that\n \n\n191\n00:04:29,160 --> 00:04:32,070\nthat. We'll work with that\nlater. Okay. So we have our data now.\n\n192\n00:04:32,070 --> 00:04:32,080\nlater. Okay. So we have our data now.\n \n\n193\n00:04:32,080 --> 00:04:33,550\nlater. Okay. So we have our data now.\nNow we just need to connect it to\n\n194\n00:04:33,550 --> 00:04:33,560\nNow we just need to connect it to\n \n\n195\n00:04:33,560 --> 00:04:36,629\nNow we just need to connect it to\nclaude. To do that, we're going to take\n\n196\n00:04:36,629 --> 00:04:36,639\nclaude. To do that, we're going to take\n \n\n197\n00:04:36,639 --> 00:04:37,990\nclaude. To do that, we're going to take\nthis configuration. We're going to\n\n198\n00:04:37,990 --> 00:04:38,000\nthis configuration. We're going to\n \n\n199\n00:04:38,000 --> 00:04:41,030\nthis configuration. We're going to\nmodify it to our own environment. So for\n\n200\n00:04:41,030 --> 00:04:41,040\nmodify it to our own environment. So for\n \n\n201\n00:04:41,040 --> 00:04:43,270\nmodify it to our own environment. So for\nme, that means my command will be just\n\n202\n00:04:43,270 --> 00:04:43,280\nme, that means my command will be just\n \n\n203\n00:04:43,280 --> 00:04:45,830\nme, that means my command will be just\nMCP server webcrawl because it's Windows\n\n204\n00:04:45,830 --> 00:04:45,840\nMCP server webcrawl because it's Windows\n \n\n205\n00:04:45,840 --> 00:04:47,070\nMCP server webcrawl because it's Windows\nand it's on\n\n206\n00:04:47,070 --> 00:04:47,080\nand it's on\n \n\n207\n00:04:47,080 --> 00:04:50,950\nand it's on\npath. And my data source will be that\n\n208\n00:04:50,950 --> 00:04:50,960\npath. And my data source will be that\n \n\n209\n00:04:50,960 --> 00:04:54,790\npath. And my data source will be that\ndirectory that I showed you on desktop.\n\n210\n00:04:54,790 --> 00:04:54,800\ndirectory that I showed you on desktop.\n \n\n211\n00:04:54,800 --> 00:04:57,790\ndirectory that I showed you on desktop.\nSo, let's head back to Claude\n\n212\n00:04:57,790 --> 00:04:57,800\nSo, let's head back to Claude\n \n\n213\n00:04:57,800 --> 00:05:00,550\nSo, let's head back to Claude\nand just want to show you there's no MCP\n\n214\n00:05:00,550 --> 00:05:00,560\nand just want to show you there's no MCP\n \n\n215\n00:05:00,560 --> 00:05:04,430\nand just want to show you there's no MCP\nservers right now. I'm going to head to\n\n216\n00:05:04,430 --> 00:05:04,440\nservers right now. I'm going to head to\n \n\n217\n00:05:04,440 --> 00:05:08,830\nservers right now. I'm going to head to\nsettings, developer, edit,\n\n218\n00:05:08,830 --> 00:05:08,840\n\n \n\n219\n00:05:08,840 --> 00:05:11,550\n\nconfig, and\n\n220\n00:05:11,550 --> 00:05:11,560\nconfig, and\n \n\n221\n00:05:11,560 --> 00:05:14,670\nconfig, and\nthen I'm going to add my new\n\n222\n00:05:14,670 --> 00:05:14,680\nthen I'm going to add my new\n \n\n223\n00:05:14,680 --> 00:05:17,909\nthen I'm going to add my new\nconfiguration. Okay, so let's save that.\n\n224\n00:05:17,909 --> 00:05:17,919\nconfiguration. Okay, so let's save that.\n \n\n225\n00:05:17,919 --> 00:05:21,070\nconfiguration. Okay, so let's save that.\nHead back to Claude. we need to fully\n\n226\n00:05:21,070 --> 00:05:21,080\nHead back to Claude. we need to fully\n \n\n227\n00:05:21,080 --> 00:05:26,189\nHead back to Claude. we need to fully\nclose in order to reload the uh\n\n228\n00:05:26,189 --> 00:05:26,199\n\n \n\n229\n00:05:26,199 --> 00:05:29,029\n\nconfiguration. So when we come back,\n\n230\n00:05:29,029 --> 00:05:29,039\nconfiguration. So when we come back,\n \n\n231\n00:05:29,039 --> 00:05:31,430\nconfiguration. So when we come back,\nwe'll have our MCP server and they are\n\n232\n00:05:31,430 --> 00:05:31,440\nwe'll have our MCP server and they are\n \n\n233\n00:05:31,440 --> 00:05:34,230\nwe'll have our MCP server and they are\nboth active. So everything looks good.\n\n234\n00:05:34,230 --> 00:05:34,240\nboth active. So everything looks good.\n \n\n235\n00:05:34,240 --> 00:05:36,070\nboth active. So everything looks good.\nWe can actually start interacting with\n\n236\n00:05:36,070 --> 00:05:36,080\nWe can actually start interacting with\n \n\n237\n00:05:36,080 --> 00:05:38,070\nWe can actually start interacting with\nour data. Right now we can ask uh can\n\n238\n00:05:38,070 --> 00:05:38,080\nour data. Right now we can ask uh can\n \n\n239\n00:05:38,080 --> 00:05:46,670\nour data. Right now we can ask uh can\nyou list crawled\n\n240\n00:05:46,670 --> 00:05:46,680\n\n \n\n241\n00:05:46,680 --> 00:05:49,830\n\nsites? There we go. Uh if it's your\n\n242\n00:05:49,830 --> 00:05:49,840\nsites? There we go. Uh if it's your\n \n\n243\n00:05:49,840 --> 00:05:50,950\nsites? There we go. Uh if it's your\nfirst time through, you're going to get\n\n244\n00:05:50,950 --> 00:05:50,960\nfirst time through, you're going to get\n \n\n245\n00:05:50,960 --> 00:05:54,070\nfirst time through, you're going to get\nprompted for permission to use the tool.\n\n246\n00:05:54,070 --> 00:05:54,080\nprompted for permission to use the tool.\n \n\n247\n00:05:54,080 --> 00:05:56,950\nprompted for permission to use the tool.\nI've already given that access and it's\n\n248\n00:05:56,950 --> 00:05:56,960\nI've already given that access and it's\n \n\n249\n00:05:56,960 --> 00:05:58,629\nI've already given that access and it's\ngoing to come back with the uh two work\n\n250\n00:05:58,629 --> 00:05:58,639\ngoing to come back with the uh two work\n \n\n251\n00:05:58,639 --> 00:06:01,990\ngoing to come back with the uh two work\nfiles, which is exactly what we want. I\n\n252\n00:06:01,990 --> 00:06:02,000\nfiles, which is exactly what we want. I\n \n\n253\n00:06:02,000 --> 00:06:11,230\nfiles, which is exactly what we want. I\ncan ask like are are there any\n\n254\n00:06:11,230 --> 00:06:11,240\n\n \n\n255\n00:06:11,240 --> 00:06:14,469\n\n404s and uh it's going to search for 404\n\n256\n00:06:14,469 --> 00:06:14,479\n404s and uh it's going to search for 404\n \n\n257\n00:06:14,479 --> 00:06:16,550\n404s and uh it's going to search for 404\nstatus and it's going to it's going to\n\n258\n00:06:16,550 --> 00:06:16,560\nstatus and it's going to it's going to\n \n\n259\n00:06:16,560 --> 00:06:19,189\nstatus and it's going to it's going to\nfind one. And sure, I mean, it's a\n\n260\n00:06:19,189 --> 00:06:19,199\nfind one. And sure, I mean, it's a\n \n\n261\n00:06:19,199 --> 00:06:21,990\nfind one. And sure, I mean, it's a\ncontrived 404, but it is a 404, so it\n\n262\n00:06:21,990 --> 00:06:22,000\ncontrived 404, but it is a 404, so it\n \n\n263\n00:06:22,000 --> 00:06:23,990\ncontrived 404, but it is a 404, so it\ncaught the status. And that's one thing\n\n264\n00:06:23,990 --> 00:06:24,000\ncaught the status. And that's one thing\n \n\n265\n00:06:24,000 --> 00:06:26,870\ncaught the status. And that's one thing\nthat work can do that a wget get mirror\n\n266\n00:06:26,870 --> 00:06:26,880\nthat work can do that a wget get mirror\n \n\n267\n00:06:26,880 --> 00:06:29,670\nthat work can do that a wget get mirror\nwon't do for you is collect status codes\n\n268\n00:06:29,670 --> 00:06:29,680\nwon't do for you is collect status codes\n \n\n269\n00:06:29,680 --> 00:06:32,870\nwon't do for you is collect status codes\nand headers. By the way, there's a give\n\n270\n00:06:32,870 --> 00:06:32,880\nand headers. By the way, there's a give\n \n\n271\n00:06:32,880 --> 00:06:34,390\nand headers. By the way, there's a give\nand take. You just got to figure out\n\n272\n00:06:34,390 --> 00:06:34,400\nand take. You just got to figure out\n \n\n273\n00:06:34,400 --> 00:06:36,790\nand take. You just got to figure out\nwhat's best for you. But with wget, you\n\n274\n00:06:36,790 --> 00:06:36,800\nwhat's best for you. But with wget, you\n \n\n275\n00:06:36,800 --> 00:06:39,590\nwhat's best for you. But with wget, you\ndo have access to both uh the work type\n\n276\n00:06:39,590 --> 00:06:39,600\ndo have access to both uh the work type\n \n\n277\n00:06:39,600 --> 00:06:41,990\ndo have access to both uh the work type\nand the simple file archive. They're\n\n278\n00:06:41,990 --> 00:06:42,000\nand the simple file archive. They're\n \n\n279\n00:06:42,000 --> 00:06:43,749\nand the simple file archive. They're\nboth options. They both have strengths.\n\n280\n00:06:43,749 --> 00:06:43,759\nboth options. They both have strengths.\n \n\n281\n00:06:43,759 --> 00:06:46,870\nboth options. They both have strengths.\nThey both have weaknesses.\n\n282\n00:06:46,870 --> 00:06:46,880\nThey both have weaknesses.\n \n\n283\n00:06:46,880 --> 00:06:48,950\nThey both have weaknesses.\nAnyway, hopefully that gives you an idea\n\n284\n00:06:48,950 --> 00:06:48,960\nAnyway, hopefully that gives you an idea\n \n\n285\n00:06:48,960 --> 00:06:51,790\nAnyway, hopefully that gives you an idea\nfor what you can do with MCP server web\n\n286\n00:06:51,790 --> 00:06:51,800\nfor what you can do with MCP server web\n \n\n287\n00:06:51,800 --> 00:06:54,790\nfor what you can do with MCP server web\ncrawl. It's all I got for today. Have a\n\n288\n00:06:54,790 --> 00:06:54,800\ncrawl. It's all I got for today. Have a\n \n\n289\n00:06:54,800 --> 00:06:59,080\ncrawl. It's all I got for today. Have a\ngood one and uh later on." ], "output": "media", "pwd": "/home/archivebox-data/pragmar/archive/1756357684.130171", "schema": "ArchiveResult", "start_ts": "2025-08-28T05:13:47.459296+00:00", "status": "succeeded" } ], "mercury": [ { "cmd": [ "postlight-parser", "https://pragmar.com/mcp-server-webcrawl/", "--format=text" ], "cmd_version": null, "end_ts": "2025-08-28T05:13:47.448817+00:00", "index_texts": null, "output": "FileNotFoundError: [Errno 2] No such file or directory: 'postlight-parser'", "pwd": "/home/archivebox-data/pragmar/archive/1756357684.130171", "schema": "ArchiveResult", "start_ts": "2025-08-28T05:13:47.447384+00:00", "status": "failed" } ], "pdf": [], "readability": [ { "cmd": [ "readability-extractor", "./{singlefile,dom}.html" ], "cmd_version": null, "end_ts": "2025-08-28T05:13:47.437150+00:00", "index_texts": [], "output": "FileNotFoundError: [Errno 2] No such file or directory: 'readability-extractor'", "pwd": "/home/archivebox-data/pragmar/archive/1756357684.130171", "schema": "ArchiveResult", "start_ts": "2025-08-28T05:13:47.434513+00:00", "status": "failed" } ], "screenshot": [], "singlefile": [], "title": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.1 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.5.0 (x86_64-pc-linux-gnu)", "https://pragmar.com/mcp-server-webcrawl/" ], "cmd_version": "curl 8.5.0 (x86_64-pc-linux-gnu)", "end_ts": "2025-08-28T05:13:47.423619+00:00", "index_texts": null, "output": "mcp-server-webcrawl | MCP server for web crawlers", "pwd": "/home/archivebox-data/pragmar/archive/1756357684.130171", "schema": "ArchiveResult", "start_ts": "2025-08-28T05:13:47.416981+00:00", "status": "succeeded" } ], "wget": [ { "cmd": [ "wget", "--no-verbose", "--adjust-extension", "--convert-links", "--force-directories", "--backup-converted", "--span-hosts", "--no-parent", "-e", "robots=off", "--timeout=60", "--restrict-file-names=windows", "--warc-file=/home/archivebox-data/pragmar/archive/1756357684.130171/warc/1756358021", "--page-requisites", "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.1 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.21.4", "--compression=auto", "https://pragmar.com/mcp-server-webcrawl/" ], "cmd_version": "GNU Wget 1.21.4", "end_ts": "2025-08-28T05:13:47.404218+00:00", "index_texts": null, "output": "pragmar.com/mcp-server-webcrawl/index.html", "pwd": "/home/archivebox-data/pragmar/archive/1756357684.130171", "schema": "ArchiveResult", "start_ts": "2025-08-28T05:13:41.479007+00:00", "status": "succeeded" } ] }, "icons": null, "is_archived": true, "is_static": false, "latest": { "archive_org": "TimeoutExpired: Command '['curl', '--silent', '--location', '--compressed', '--head', '--max-time', '60', '--user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.1 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.5.0 (x86_64-pc-linux-gnu)', 'https://web.archive.org/save/https://pragmar.com/mcp-server-webcrawl/']' timed out after 60 seconds", "dom": null, "favicon": "favicon.ico", "git": null, "media": "media", "pdf": null, "screenshot": null, "singlefile": null, "title": "mcp-server-webcrawl | MCP server for web crawlers", "warc": null, "wget": "pragmar.com/mcp-server-webcrawl/index.html" }, "link_dir": "/home/archivebox-data/pragmar/archive/1756357684.130171", "newest_archive_date": "2025-08-28T05:15:12.894318+00:00", "num_failures": 3, "num_outputs": 5, "oldest_archive_date": "2025-08-28T05:13:41.152004+00:00", "path": "/mcp-server-webcrawl/", "schema": "Link", "scheme": "https", "snapshot_id": "4134dce2-21ab-4958-b41b-0141f243e707", "sources": [ "/home/archivebox-data/pragmar/sources/1756357684-import.txt" ], "tags": null, "tags_str": "", "timestamp": "1756357684.130171", "title": "mcp-server-webcrawl | MCP server for web crawlers", "updated": "2025-08-28T05:13:41.149648+00:00", "updated_date": "2025-08-28 05:13", "url": "https://pragmar.com/mcp-server-webcrawl/" }