mcp-server-webcrawl

Search.setIndex({"docnames": ["guides", "guides/archivebox", "guides/httrack", "guides/interrobot", "guides/katana", "guides/siteone", "guides/warc", "guides/wget", "index", "installation", "interactive", "mcp_server_webcrawl", "mcp_server_webcrawl.crawlers", "mcp_server_webcrawl.crawlers.archivebox", "mcp_server_webcrawl.crawlers.base", "mcp_server_webcrawl.crawlers.httrack", "mcp_server_webcrawl.crawlers.interrobot", "mcp_server_webcrawl.crawlers.katana", "mcp_server_webcrawl.crawlers.siteone", "mcp_server_webcrawl.crawlers.warc", "mcp_server_webcrawl.crawlers.wget", "mcp_server_webcrawl.extras", "mcp_server_webcrawl.interactive", "mcp_server_webcrawl.models", "mcp_server_webcrawl.templates", "mcp_server_webcrawl.utils", "modules", "prompts", "usage"], "filenames": ["guides.rst", "guides/archivebox.rst", "guides/httrack.rst", "guides/interrobot.rst", "guides/katana.rst", "guides/siteone.rst", "guides/warc.rst", "guides/wget.rst", "index.rst", "installation.rst", "interactive.rst", "mcp_server_webcrawl.rst", "mcp_server_webcrawl.crawlers.rst", "mcp_server_webcrawl.crawlers.archivebox.rst", "mcp_server_webcrawl.crawlers.base.rst", "mcp_server_webcrawl.crawlers.httrack.rst", "mcp_server_webcrawl.crawlers.interrobot.rst", "mcp_server_webcrawl.crawlers.katana.rst", "mcp_server_webcrawl.crawlers.siteone.rst", "mcp_server_webcrawl.crawlers.warc.rst", "mcp_server_webcrawl.crawlers.wget.rst", "mcp_server_webcrawl.extras.rst", "mcp_server_webcrawl.interactive.rst", "mcp_server_webcrawl.models.rst", "mcp_server_webcrawl.templates.rst", "mcp_server_webcrawl.utils.rst", "modules.rst", "prompts.rst", "usage.rst"], "titles": ["Setup Guides", "ArchiveBox MCP Setup Guide", "HTTrack MCP Setup Guide", "InterroBot MCP Setup Guide", "Katana MCP Setup Guide", "SiteOne MCP Setup Guide", "WARC MCP Setup Guide", "wget MCP Setup Guide", "mcp-server-webcrawl", "Installation", "Interactive Mode", "mcp_server_webcrawl package", "mcp_server_webcrawl.crawlers package", "mcp_server_webcrawl.crawlers.archivebox package", "mcp_server_webcrawl.crawlers.base package", "mcp_server_webcrawl.crawlers.httrack package", "mcp_server_webcrawl.crawlers.interrobot package", "mcp_server_webcrawl.crawlers.katana package", "mcp_server_webcrawl.crawlers.siteone package", "mcp_server_webcrawl.crawlers.warc package", "mcp_server_webcrawl.crawlers.wget package", "mcp_server_webcrawl.extras package", "mcp_server_webcrawl.interactive package", "mcp_server_webcrawl.models package", "mcp_server_webcrawl.templates package", "mcp_server_webcrawl.utils package", "mcp_server_webcrawl", "Prompt Routines", "Usage"], "terms": {"websit": [1, 3, 4, 5, 6, 8, 15, 23, 27, 28], "github": [4, 8], "doc": [8, 23, 28], "bridg": [], "gap": [], "between": [14, 22], "your": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 27, 28], "web": [1, 2, 3, 4, 8, 9, 10, 13, 14, 15, 16, 17, 18, 19, 20, 23, 27, 28], "crawl": [1, 2, 4, 6, 9, 10, 23, 27, 28], "ai": [8, 9, 10, 28], "languag": 4, "model": [11, 26, 28], "us": [9, 10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 27, 28], "context": [21, 27, 28], "protocol": [], "With": [8, 10], "client": [8, 27, 28], "filter": [8, 13, 14, 15, 16, 17, 18, 19, 20, 22, 28], "analyz": [2, 3, 8, 28], "under": [1, 2, 3, 4, 5, 6, 7, 8, 9], "direct": 8, "autonom": [8, 28], "The": [1, 2, 4, 8, 13, 14, 15, 17, 18, 21, 22, 28], "includ": [1, 2, 3, 4, 5, 6, 7, 8, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25, 28], "full": [8, 14, 28], "text": [8, 14, 16, 17, 21, 22, 23, 28], "search": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 26, 27], "interfac": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 22, 27, 28], "boolean": [8, 10, 13, 14, 15, 16, 17, 18, 19, 20, 27], "support": [1, 2, 9, 10, 14, 19, 22], "resourc": [8, 11, 13, 14, 15, 16, 17, 18, 19, 20, 25, 26, 28], "type": [3, 8, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25, 27], "http": [1, 2, 4, 6, 7, 8, 9, 10, 14, 16, 17, 23, 28], "statu": [6, 8, 14, 17, 22, 23, 28], "more": [1, 2, 3, 4, 5, 6, 7, 8, 10, 14, 18, 28], "provid": [1, 2, 3, 8, 13, 14, 15, 16, 17, 18, 19, 20, 27, 28], "llm": [1, 2, 3, 4, 5, 6, 7, 8, 9, 14, 24, 27, 28], "complet": [1, 2, 3, 4, 5, 6, 7, 8, 14, 28], "menu": 8, "which": [1, 2, 3, 4, 5, 6, 7, 8, 14, 28], "work": [1, 3, 8, 13, 22, 28], "varieti": 8, "crawler": [1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 22, 25, 26], "warc": [0, 1, 8, 9, 12, 28], "wget": [0, 1, 8, 9, 10, 12, 16, 17, 18, 19, 28], "interrobot": [0, 8, 9, 12, 13, 14, 17, 18, 28], "katana": [0, 8, 9, 12, 28], "siteon": [0, 8, 9, 12, 28], "i": [6, 7, 8, 13, 14, 15, 18, 19, 20, 21, 22, 24, 27, 28], "free": 8, "open": [1, 2, 3, 4, 5, 6, 7, 8], "sourc": [1, 2, 4, 8, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 28], "requir": [8, 10, 14, 16, 22, 25, 28], "claud": [8, 9], "desktop": [8, 9], "python": [1, 2, 3, 4, 5, 6, 7, 8, 9, 25], "3": [8, 22, 25, 28], "10": [1, 2, 3, 4, 5, 6, 7, 8, 14, 22, 25], "It": [3, 8, 14, 24, 27, 28], "instal": [8, 28], "command": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], "line": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 22], "via": [6, 7, 8, 9], "pip": [1, 2, 3, 4, 5, 6, 7, 8, 9], "mcp_server_webcrawl": 8, "usag": 8, "index": [1, 2, 3, 6, 7, 8, 12, 13, 22, 28], "modul": [8, 26], "page": [1, 3, 4, 6, 8, 22, 23, 28], "packag": [1, 2, 3, 4, 5, 6, 7, 9, 26], "To": [9, 14], "effect": 9, "you": [1, 2, 3, 4, 5, 6, 7, 9, 10, 14, 27, 28], "need": [1, 2, 3, 4, 5, 6, 7, 9, 14, 16, 22, 28], "an": [1, 3, 4, 7, 9, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 27, 28], "capabl": [1, 3, 4, 5, 9, 22, 27, 28], "basic": [1, 2, 3, 4, 5, 6, 7, 9, 13, 14, 15, 16, 17, 18, 19, 20, 27], "familiar": [1, 2, 3, 4, 5, 6, 7, 9, 28], "run": [6, 7, 9, 10, 13, 14, 15, 16, 17, 18, 19, 20, 22, 27], "after": [1, 2, 3, 4, 5, 6, 7, 9], "ensur": [1, 2, 3, 4, 5, 6, 7, 9, 14, 23], "prerequisit": 9, "ar": [1, 2, 3, 4, 5, 7, 9, 10, 13, 14, 15, 22, 27, 28], "met": 9, "simpli": 14, "abov": 9, "add": [1, 2, 3, 4, 5, 6, 7, 9, 14, 22], "environ": [0, 9, 13, 15, 16, 17, 18, 19, 20, 24], "handl": [13, 14, 15, 16, 17, 18, 22], "its": [1, 23], "own": 1, "depend": [3, 4, 7, 12], "dure": [1, 2, 4, 14], "enabl": [9, 27, 28], "access": [9, 13, 14, 15, 16, 17, 18, 19, 20], "data": [1, 3, 4, 6, 8, 9, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24, 27, 28], "ll": [3, 9], "server": [0, 9, 10, 11, 14, 26, 27, 28], "from": [1, 2, 3, 4, 5, 6, 7, 9, 10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 28], "": [1, 2, 4, 9, 13, 22, 24, 28], "develop": [1, 2, 3, 4, 5, 6, 7, 9, 28], "set": [2, 3, 4, 5, 6, 7, 9, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24, 25, 26, 27], "locat": [2, 3, 4, 5, 9], "section": [0, 3, 9], "appropri": 9, "below": [1, 2, 3, 4, 5, 6, 28], "each": [1, 2, 4, 5, 6, 9, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 27], "choos": 28, "one": [13, 14, 15, 17, 18, 19, 20, 22, 27, 28], "match": [21, 28], "modifi": [1, 2, 3, 4, 5, 6, 7, 14, 23, 28], "datasrc": [1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23], "path": [1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25], "point": [1, 2], "specif": [3, 5, 13, 14, 15, 16, 17, 18, 22, 28], "mcpserver": [1, 2, 3, 4, 5, 6, 7, 9], "webcrawl": [0, 9, 10, 27, 28], "arg": [1, 2, 3, 4, 5, 6, 7, 9, 14], "archiv": [1, 2, 4, 5, 6, 7, 8, 9, 10, 13, 18, 19], "test": [11, 12, 26, 27], "maco": [1, 2, 3, 4, 5, 6, 7], "termin": [1, 2, 3, 4, 5, 6, 7, 10, 22], "window": [1, 2, 3, 4, 5, 6, 7, 22, 25, 28], "wsl": [6, 7], "adjust": 22, "extens": [6, 14, 28], "file": [1, 2, 3, 4, 5, 7, 9, 13, 14, 15, 16, 17, 19, 21, 27, 28], "e": [1, 2, 3, 4, 5, 6, 7, 21, 22], "g": [1, 2, 3, 4, 5, 6, 7, 21, 22], "html": [1, 2, 13, 14, 21, 23, 24, 28], "mirror": [6, 7, 8, 15, 28], "exampl": [1, 2, 3, 4, 5, 6, 7, 28], "com": [1, 2, 3, 4, 5, 6, 7, 28], "recurs": [6, 14], "requisit": 6, "homedir": 3, "document": [1, 2, 3, 4, 5, 6, 7, 10, 22, 28], "v2": 3, "db": 3, "note": [3, 5, 14, 17, 18, 20], "must": [13, 14, 15, 17, 18, 19, 20, 23], "execut": [13, 14, 15, 16, 17, 18, 19, 20, 22, 24], "applic": [2, 5, 22], "On": [1, 2, 3, 4, 5, 6, 7], "replac": [3, 14], "user": [2, 3, 5, 6, 7], "powershel": [], "store": [4, 5, 6, 7, 13], "respons": [4, 6, 13, 14, 15, 16, 17, 18, 19, 20, 23, 28], "save": [1, 2, 3, 4, 5, 6, 7, 9, 15, 16, 17, 18, 19, 20, 28], "content": [1, 2, 3, 4, 5, 6, 7, 26], "dir": 4, "allow": [1, 2, 3, 4, 5, 6, 7, 22, 28], "mani": 3, "site": [1, 2, 3, 4, 5, 6, 7, 11, 13, 14, 15, 16, 17, 18, 19, 20, 22, 25, 26, 27, 28], "u": 4, "gener": [3, 5, 14, 15, 16, 18, 21, 25, 28], "offlin": [5, 15], "check": [1, 2, 3, 4, 5, 6, 7, 14, 22, 27], "can": [1, 2, 3, 4, 5, 6, 7, 9, 10, 14, 18, 21, 27, 28], "up": [2, 3, 4, 5, 6, 7, 9, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25], "connect": [9, 14, 16, 17, 18, 19, 20, 28], "want": [2, 9, 27], "differ": [1, 4, 9, 21, 23, 24], "simultan": 9, "webcrawl_warc": 9, "webcrawl_wget": 9, "ad": 9, "restart": [1, 2, 3, 4, 5, 6, 7, 9], "appli": [9, 22], "chang": [4, 5, 6, 7, 9], "util": [11, 22, 26], "blob": [], "cli": [1, 8, 11, 26], "logger": [11, 26], "querycach": [], "tool": [1, 2, 3, 4, 5, 6, 7, 8, 11, 14, 16, 26], "async": [11, 14, 16, 22], "paramet": [4, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25, 28], "basecrawl": [11, 14, 16, 22], "entri": 13, "mcp": [0, 10, 14, 16, 25, 27, 28], "should": [1, 2, 3, 4, 5, 6, 7, 14, 22, 23, 28], "return": [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25, 28], "none": [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25], "base": [1, 7, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 28], "submodul": [12, 26], "adapt": 12, "api": [1, 2, 3, 4, 5, 6, 7, 12, 13, 15, 16, 17, 18, 19, 20, 28], "get_fixture_directori": 12, "class": [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24], "sitesgroup": [14, 16], "object": [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25], "simpl": 28, "contain": [0, 1, 2, 13, 14, 15, 16, 17, 18, 19, 20, 23, 25, 28], "being": 14, "onc": [14, 28], "site_id": 14, "id": [13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 28], "site_path": 14, "directori": [1, 2, 4, 5, 6, 7, 13, 14, 15, 16, 17, 18, 19, 20, 23, 28], "__init__": [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23], "list": [1, 2, 3, 4, 5, 6, 7, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25, 28], "int": [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25, 28], "get_sit": [13, 14, 15, 16, 17, 18, 19, 20], "dict": [14, 21, 23], "str": [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25, 28], "sitesstat": 14, "some": [1, 2, 6, 12, 14], "bookeep": 14, "troubleshoot": 14, "group": [1, 14, 16, 21, 22, 28], "cach": [13, 14, 15, 16, 17, 18, 19, 20, 21], "bool": [14, 22, 23], "basemanag": [14, 16], "manag": [1, 2, 3, 5, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 28], "memori": [13, 14, 15, 16, 17, 18, 19, 20], "sqlite": [13, 14, 15, 16, 17, 18, 19, 20], "databas": [1, 3, 13, 14, 15, 16, 17, 18, 19, 20, 28], "pool": [14, 16, 17, 18, 19, 20], "effici": [2, 4, 14, 16, 17, 18, 19, 20, 28], "initi": [1, 2, 3, 6, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23], "empti": [3, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 28], "statist": [13, 14, 15, 16, 17, 18, 19, 20, 28], "static": [14, 22], "string_to_id": 14, "dirnam": [], "convert": [14, 21, 22, 23, 24, 25], "string": [13, 14, 15, 16, 17, 18, 19, 20, 21, 25, 28], "name": [1, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24, 25, 27], "number": [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 28], "suitabl": [14, 23], "usual": 25, "get_connect": [14, 16], "get": [10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 25], "creat": [1, 4, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 28], "sites_group": 14, "load": [12, 14, 16, 24], "current": [6, 14, 22], "built": 14, "get_stat": 14, "basejsonapiencod": 14, "jsonencod": 14, "custom": [14, 18, 24], "json": [13, 14, 23], "encod": [14, 21, 23, 28], "basejsonapi": 14, "resourceresulttyp": [14, 17, 23], "enum": [14, 22, 23], "constructor": 14, "sensibl": 14, "default": [2, 14, 21, 22, 24, 25, 28], "If": [1, 2, 3, 4, 5, 6, 7, 13, 14, 15, 18, 19, 20, 22, 27, 28], "skipkei": 14, "fals": [14, 22], "typeerror": 14, "attempt": 14, "kei": [14, 22, 23, 25], "float": [14, 21, 23], "true": [14, 22], "item": 14, "skip": [13, 14, 15, 16, 17, 18, 19, 20, 28], "ensure_ascii": 14, "output": [1, 2, 3, 4, 5, 6, 7, 14, 23, 28], "guarante": 14, "all": [1, 2, 4, 10, 14, 15, 16, 18, 19, 20, 22, 23, 28], "incom": 14, "non": [5, 14, 23], "ascii": 14, "charact": [14, 21, 22], "escap": 14, "check_circular": 14, "circular": 14, "refer": [14, 28], "prevent": 14, "infinit": 14, "would": 14, "caus": 14, "overflowerror": 14, "otherwis": [14, 22], "take": [1, 2, 3, 6, 14, 21, 28], "place": [14, 28], "allow_nan": 14, "nan": 14, "infin": 14, "thi": [0, 1, 2, 3, 4, 5, 7, 14, 16, 22, 24, 27, 28], "behavior": [4, 14], "compliant": 14, "consist": [4, 14], "most": [14, 28], "javascript": [14, 28], "decod": 14, "valueerror": [13, 14, 15, 16, 17, 18, 19, 20, 24], "sort_kei": 14, "dictionari": [14, 21, 23], "sort": [13, 14, 15, 16, 17, 18, 19, 20, 22, 28], "regress": 14, "serial": [14, 23], "compar": [6, 14], "dai": 14, "basi": 14, "indent": 14, "neg": 14, "integ": [14, 23, 28], "arrai": [14, 28], "element": [2, 14, 28], "member": 14, "pretti": 14, "print": 14, "level": [13, 14, 25, 28], "0": [13, 14, 15, 16, 17, 18, 19, 20, 22, 28], "onli": [1, 14, 21, 22, 24, 27, 28], "insert": [14, 22], "newlin": 14, "compact": 14, "represent": [14, 21], "specifi": [2, 4, 13, 14, 15, 16, 17, 18, 19, 20, 23, 24, 28], "separ": [1, 4, 13, 14, 21], "item_separ": 14, "key_separ": 14, "tupl": [13, 14, 15, 16, 17, 18, 19, 20, 22, 25], "elimin": 14, "whitespac": 14, "function": [1, 2, 3, 4, 5, 6, 7, 13, 14, 15, 16, 17, 18, 19, 20, 22], "call": 14, "t": [1, 2, 3, 4, 5, 6, 7, 14, 17, 18, 20, 25, 28], "version": [4, 7, 14, 25], "rais": [13, 14, 15, 16, 17, 18, 19, 20, 24], "obj": 14, "overrid": 14, "serializ": 14, "ani": [10, 14, 22, 28], "standard": [8, 14], "structur": [1, 2, 13, 14, 15], "metadata": [1, 2, 3, 4, 5, 6, 7, 13, 14, 15, 23], "result": [10, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24, 25, 28], "error": [14, 21, 22, 25, 28], "construct": 14, "argument": [10, 13, 14, 15, 17, 18, 19, 20, 28], "creation": [14, 23], "aoc": 14, "echo": 14, "back": 14, "collaps": [14, 28], "dump": 14, "done": 14, "everyth": 14, "within": [4, 14, 21, 22, 28], "implement": [13, 14, 15, 16, 17, 18, 19, 20, 28], "to_dict": [14, 23], "method": [13, 14, 15, 16, 17, 18, 19, 20, 24], "properti": [14, 22], "total": [13, 14, 15, 16, 17, 18, 19, 20, 22], "count": [3, 13, 14, 15, 16, 17, 18, 19, 20, 28], "get_result": 14, "siteresult": [13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 25], "resourceresult": [13, 14, 15, 16, 17, 18, 19, 20, 23], "set_result": 14, "offset": [13, 14, 15, 16, 17, 18, 19, 20, 28], "limit": [13, 14, 15, 16, 17, 18, 19, 20, 28], "those": 14, "beyond": [14, 28], "start": [2, 5, 14, 22, 28], "posit": [14, 21, 22, 23], "maximum": [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 28], "append_error": 14, "messag": 14, "visibl": 14, "endpoint": 14, "to_json": 14, "foundat": 14, "special": [4, 14, 28], "interact": [8, 11, 14, 24, 26], "oper": [1, 14, 22, 23, 28], "serv": 14, "stdin": 14, "stdout": 14, "launch": [10, 14, 22], "await": 14, "asyncfil": 14, "input": [1, 13, 14, 15, 17, 18, 19, 20, 21, 22, 27], "stream": [14, 25], "over": [14, 21], "stdio": [14, 25], "get_initialization_opt": 14, "project": [1, 2, 3, 4, 14, 15, 28], "inform": [1, 2, 3, 4, 5, 6, 7, 14], "initializationopt": 14, "get_sites_api_json": 14, "kwarg": 14, "get_resources_api_json": 14, "get_sites_api": 14, "field": [10, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23], "option": [1, 2, 3, 4, 5, 6, 7, 13, 14, 15, 16, 17, 18, 19, 20, 22, 25, 27, 28], "notimplementederror": 14, "subclass": 14, "get_resources_api": 14, "querystr": 14, "queri": [3, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 27, 28], "mcp_list_tool": [14, 16], "avail": [1, 2, 3, 4, 5, 6, 7, 9, 10, 14, 16, 22, 25], "mcp_call_tool": 14, "request": [14, 27, 28], "super": 14, "tweak": 14, "passthrough": 14, "pass": 14, "doe": [13, 14, 15, 16, 17, 18, 19, 20, 24, 28], "exist": [1, 14, 17, 18, 20, 22, 23], "textcont": 14, "imagecont": 14, "embeddedresourc": 14, "get_thumbnail": [14, 21], "indexedcrawl": [13, 14, 15, 17, 18, 19, 20], "A": [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 27, 28], "share": [14, 18, 22], "common": [10, 14, 22], "given": [21, 22], "status": [14, 17], "20": [13, 14, 15, 16, 17, 18, 19, 20, 24, 28], "basecrawlertest": [13, 14, 15, 16, 17, 18, 19, 20], "testcas": [14, 24], "instanc": [1, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24], "when": [2, 5, 10, 13, 14, 15, 16, 17, 18, 19, 20, 24, 27, 28], "have": [1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 16, 17, 18, 19, 20, 24, 28], "setup": [8, 9, 13, 14, 15, 16, 17, 18, 19, 20, 24], "hook": 14, "fixtur": [13, 14, 15, 16, 17, 18, 19, 20, 24], "befor": [1, 2, 3, 4, 5, 6, 7, 14, 22], "exercis": 14, "iso_to_datetim": [], "dt_string": 25, "iso": [25, 28], "datetim": [14, 23, 25], "struggl": 25, "zulu": [], "fraction": 25, "second": [3, 25], "throw": 25, "smooth": 25, "out": [4, 25, 27, 28], "precis": [25, 28], "isn": 25, "here": 25, "get_resourc": [13, 15, 16, 17, 18, 19, 20], "where": [1, 2, 4, 5, 6], "ANDed": [], "fts5": 27, "order": [13, 14, 15, 16, 17, 18, 19, 20, 25, 28], "pagin": [13, 14, 15, 16, 17, 18, 19, 20, 28], "interrobotcrawl": 16, "retriev": [8, 13, 14, 15, 16, 17, 18, 19, 20, 28], "format": [1, 2, 3, 4, 5, 6, 7, 8, 14, 18, 28], "directli": [22, 28], "addit": [2, 4, 5, 12, 23, 27, 28], "code": [6, 17, 22, 23, 28], "interrobottest": 16, "suit": [13, 15, 16, 17, 18, 19, 20, 24], "test_interrobot_puls": 16, "test_interrobot_mcp": [], "test_interrobot_thumbnail": [], "thumbnail": [11, 13, 14, 15, 16, 17, 18, 23, 26, 28], "imag": [2, 6, 13, 14, 15, 16, 17, 18, 21, 23, 24, 28], "test_interrobot_sit": 16, "test_interrobot_resourc": 16, "variou": [0, 1, 13, 15, 16, 17, 18, 19, 20], "test_interrobot_random_sort": [], "random": [13, 15, 16, 17, 18, 19, 20, 28], "katanamanag": 17, "subdirectori": [13, 14, 17, 18, 20], "katanacrawl": 17, "captur": [2, 6, 15, 16, 17, 18, 19, 20], "exchang": 17, "katanatest": 17, "pars": [13, 14, 15, 16, 17, 18, 19, 20], "test_katana_puls": 17, "test_katana_sit": 17, "test_katana_resourc": 17, "test_katana_random_sort": [], "test_katana_content_pars": 17, "detect": [13, 14, 15, 16, 17, 18, 19, 20, 27], "warcmanag": 19, "warccrawl": 19, "still": [], "awar": [], "warctest": 19, "test_warc_puls": 19, "test_warc_sit": 19, "test_warc_resourc": 19, "wgetmanag": 20, "wgetcrawl": 20, "organ": [1, 2, 18, 20, 27], "wgettest": 20, "test_wget_puls": 20, "test_wget_sit": 20, "test_wget_resourc": 20, "test_wget_random_sort": [], "repres": [22, 23], "undefin": [14, 22, 23], "frame": 23, "ifram": [23, 28], "img": [23, 28], "audio": [23, 28], "video": [1, 2, 3, 4, 5, 6, 7, 9, 23, 28], "font": [23, 28], "css": [2, 6, 23, 28], "style": [2, 21, 22, 23, 28], "script": [23, 28], "feed": [1, 23, 28], "rss": [1, 23, 28], "pdf": [1, 23, 28], "other": [1, 2, 3, 4, 5, 6, 7, 10, 22, 23, 28], "classmethod": [21, 23], "valu": [14, 21, 22, 23, 28], "identifi": [21, 23], "url": [1, 2, 3, 5, 13, 14, 15, 21, 22, 23, 27, 28], "belong": [22, 23], "wa": [1, 2, 3, 4, 5, 6, 7, 14, 22, 23, 27], "found": [3, 21, 23], "header": [6, 10, 14, 21, 22, 23, 28], "timestamp": [1, 13, 23], "last": 23, "modif": 23, "size": [3, 7, 14, 22, 23, 28], "byte": [14, 23, 28], "time": [1, 2, 3, 4, 5, 6, 7, 10, 14, 23], "millisecond": 23, "base64": [21, 23, 28], "experiment": 23, "robot": [16, 23], "txt": [15, 19, 23], "thumbnailmanag": 21, "map": [14, 21, 23], "fail": [14, 21, 25], "get_help_short_messag": 25, "get_help_long_messag": 25, "get_logger_configur": 25, "log": [1, 14, 15, 18, 25], "get_logg": 25, "write": [25, 27], "writabl": 25, "warn": 25, "info": [21, 25], "debug": [22, 25], "initialize_logg": 25, "valid": [14, 25], "querycountcach": [], "reusabl": 22, "light": 14, "max": [14, 24, 28], "ttl": [], "live": [], "250": [], "900": [], "statement": 22, "param": [], "expir": [], "sql": [], "clear": 22, "initialize_mcp_serv": 25, "get_crawler_tool": 25, "descript": [8, 25, 27, 28], "subpackag": 26, "main": [22, 26], "settings_loc": 26, "leverag": 28, "pull": 28, "advanc": [3, 8, 15, 18, 27, 28], "help": [1, 2, 3, 4, 5, 6, 7, 22, 28], "demand": 28, "market": 28, "seo": [2, 27, 28], "etc": [1, 3, 6, 14, 28], "anyth": 28, "els": 28, "truth": 28, "matter": [24, 28], "don": 28, "know": 28, "behind": 28, "design": [27, 28], "consum": 28, "howev": 28, "understand": 28, "case": [4, 21, 27, 28], "two": [], "No": [10, 28], "leav": 28, "mean": 28, "8601": 28, "norobot": 28, "webpag": 28, "across": [1, 13, 22, 28], "fulltext": [14, 28], "better": [14, 28], "AND": [22, 28], "OR": [22, 28], "NOT": [22, 28], "quot": [22, 28], "phrase": [22, 28], "suffix": 28, "wildcard": 28, "prefix": 28, "In": [1, 2, 3, 4, 5, 6, 7, 28], "scenario": 28, "d": 28, "lead": 28, "larg": [1, 2, 3, 6, 28], "200": [14, 28], "success": [1, 2, 3, 4, 5, 6, 7, 22, 28], "404": 27, "500": 28, "Not": [27, 28], "ascend": 28, "descend": 28, "sampl": 28, "100": [24, 28], "small": [], "recognit": [], "while": 28, "keep": 28, "token": [13, 14, 15, 17, 28], "minim": 28, "svg": 28, "numer": 14, "primari": 14, "hash": 14, "space": [14, 22], "collis": 14, "probabl": [14, 28], "8": [14, 22, 25], "32": 14, "bit": [14, 24], "4": [14, 22], "29": 14, "billion": 14, "1": [9, 14, 22, 24, 28], "chanc": 14, "000": 14, "12": 14, "48": 14, "280": 14, "trillion": 14, "0000001": 14, "16": 14, "64": 14, "safe": [14, 22], "zero": 14, "9": [14, 22], "22": 14, "quintillion": 14, "sign": 14, "223": 14, "372": 14, "036": 14, "854": 14, "775": 14, "807": 14, "big": 14, "problem": 14, "larger": 14, "hashspac": 14, "length": [14, 22], "thei": [4, 14, 23, 27, 28], "present": [10, 14, 23], "deriv": 14, "get_basic_head": 14, "file_s": 14, "resource_typ": 14, "read_file_cont": 14, "file_path": 14, "read": 14, "decruft_path": 14, "veri": 14, "touch": [14, 22], "cleanup": [14, 22], "tmp": 14, "nois": 14, "mcp_list_prompt": 14, "prompt": [8, 14], "mcp_list_resourc": 14, "get_sites_func": 14, "get_resources_func": 14, "resource_field_map": 14, "displai": [14, 21, 22], "callabl": 14, "doesn": [1, 2, 3, 4, 5, 6, 7, 14, 17, 18, 20], "siteonemanag": 18, "wrap": [13, 15, 16, 18, 19, 20], "siteonecrawl": 18, "merg": [18, 22], "aquir": 18, "than": [18, 28], "alon": 18, "assertionerror": [13, 15, 18, 19, 20], "siteonetest": 18, "test_siteone_puls": 18, "test_siteone_sit": 18, "test_siteone_resourc": 18, "test_siteone_random_sort": [], "get_resources_with_manag": [], "crawl_manag": [], "to_forcefield_dict": 23, "forcefield": 23, "forc": 23, "even": [23, 27], "null": 23, "appear": [1, 23], "least": 23, "attribut": [21, 22, 23, 28], "plu": [15, 16, 18, 23], "alreadi": 23, "detail": [0, 1, 2, 3, 4, 5, 6, 7], "configur": [0, 14], "guid": [8, 9], "quick": [], "step": [], "concis": 28, "instruct": [1, 2, 3, 4, 5, 6, 7, 27], "ve": [1, 2, 3, 4, 5, 6, 7], "begin": [1, 2, 3, 4, 5, 6, 7, 22], "later": [1, 2, 3, 4, 5, 6, 7], "homebrew": [6, 7], "ubuntu": [6, 7], "go": [1, 2, 3, 4, 5, 6, 7, 27], "edit": [1, 2, 3, 4, 5, 6, 7], "config": [1, 2, 3, 4, 5, 6, 7], "follow": [1, 2, 3, 4, 5, 6, 7, 28], "absolut": [1, 2, 3, 4, 5, 6, 7], "actual": [3, 4, 5, 6, 7], "exit": [1, 2, 3, 4, 5, 6, 7, 22], "just": [1, 2, 3, 4, 5, 6, 7, 10], "close": [1, 2, 3, 4, 5, 6, 7], "navig": [6, 7, 22, 28], "target": [6, 7], "now": [1, 2, 3, 4, 5, 6, 7], "see": [1, 2, 3, 4, 5, 6, 7, 28], "ask": [1, 2, 3, 4, 5, 6, 7], "try": [1, 2, 3, 4, 5, 6, 7], "find": [1, 2, 3, 4, 5, 6, 7, 21, 22, 27, 28], "about": [1, 2, 3, 4, 5, 6, 7, 28], "topic": [1, 2, 3, 4, 5, 6, 7], "show": [1, 2, 3, 4, 5, 6, 7, 10, 21, 22], "correctli": [1, 2, 3, 4, 5, 6, 7], "properli": [1, 2, 3, 4, 5, 6, 7, 15], "correct": [1, 2, 3, 4, 5, 6, 7, 28], "rememb": [1, 2, 3, 4, 5, 6, 7], "first": [1, 2, 3, 4, 5, 6, 7], "permiss": [1, 2, 3, 4, 5, 6, 7], "For": [1, 2, 3, 4, 5, 6, 7, 28], "visit": [1, 2, 3, 4, 5, 6, 7], "pragmar": [1, 2, 3, 4, 5], "sphinxcontrib": [], "youtub": [], "uqeeqvsofhc": [], "width": 22, "640": [], "height": 22, "480": [], "along": [1, 2, 3, 4, 5, 6, 7, 10], "action": [1, 2, 3, 4, 5, 6, 7], "warm": [], "happen": [], "build": [3, 7, 14, 16], "what": 27, "singl": [1, 6, 13, 22, 28], "mode": [6, 8, 28], "comprehens": [1, 2, 3, 4, 5, 6, 14], "preserv": [1, 6, 13, 15], "slower": 6, "faster": [6, 24], "gz": [6, 19], "featur": [15, 16, 18], "like": [16, 22, 24, 28], "make": [1, 2, 3, 4, 5, 6], "sure": [1, 2, 3, 4, 5, 6], "typic": [1, 2, 3, 6], "mai": [1, 2, 6, 28], "commerci": 3, "seamlessli": 3, "sever": 3, "advantag": 3, "friendli": [2, 3, 5, 10], "graphic": 3, "collect": [1, 3, 4, 28], "link": [24, 27], "multipl": [1, 2, 13, 14, 15], "automat": [], "scope": [], "depth": [], "new": [2, 3, 14, 22], "screen": [3, 22], "enter": [3, 5, 10], "wait": [2, 3], "few": [3, 24], "minut": 3, "explor": [3, 5], "report": [3, 5, 13, 14, 15, 16, 17, 18, 19, 20], "give": 3, "me": [3, 4, 5], "summari": 3, "ha": [1, 2, 3, 14, 22], "successfulli": [1, 2, 3, 4, 5], "might": [3, 28], "longer": 3, "process": [2, 3, 14, 22, 28], "program": 4, "discoveri": [1, 4], "offer": [1, 2, 4, 5], "fast": 4, "flexibl": 4, "autom": 4, "highli": 4, "abil": [2, 4], "analysi": [1, 2, 4, 27, 28], "projectdiscoveri": 4, "cmd": 4, "latest": 4, "storag": 4, "mkdir": [1, 4], "repeat": [2, 4, 5], "host": [1, 4, 8, 9, 10], "tell": [4, 5], "how": [4, 28], "re": [4, 22, 23], "flag": 4, "were": [4, 5], "expect": [4, 5], "intuit": 5, "control": [5, 22], "technic": [5, 27], "import": 5, "integr": 5, "click": 5, "button": 5, "shown": 5, "m": [5, 28], "interest": [5, 10], "appstat": [], "cross": [2, 3], "platform": [2, 3], "android": 3, "nativ": 3, "lag": 3, "pypi": 8, "suppos": 28, "stai": 28, "wai": 28, "degre": 28, "sometim": 28, "nudg": 28, "strategi": 28, "currect": 28, "\u2460": 28, "\u2461": 28, "\u2462": 28, "\u2460\u2461\u2462": 28, "ok": 28, "disk": 28, "do": 28, "queryabl": [], "oppos": 28, "strength": 28, "weak": 28, "judg": 28, "them": [14, 28], "well": [2, 14, 28], "fit": 28, "worri": 28, "too": 28, "much": 28, "except": 28, "dev": 28, "honestli": 28, "indexstatu": 14, "enumer": [14, 22, 28], "idl": 14, "partial": 14, "remot": [10, 14], "indexst": [13, 14, 15, 16, 17, 18, 19, 20], "state": [14, 22], "progress": [14, 22], "time_start": 14, "time_end": 14, "set_statu": 14, "increment_process": 14, "durat": 14, "is_timeout": 14, "exceed": 14, "timeout": 14, "threshold": 14, "read_fil": 14, "associ": [14, 16], "index_st": 14, "isoformat_zulu": [], "dt": 25, "get_resources_for_sites_group": 14, "swap_valu": 14, "searchquerypars": 14, "extract": [14, 21, 22, 28], "instead": 14, "accept": 14, "searchsubqueri": 14, "syntax": 14, "per": [14, 22, 24, 28], "parameter": 14, "connection_index_st": 14, "resourcesfulltext": 14, "extra": [11, 14, 26], "indexedmanag": [13, 14, 15, 17, 18, 19, 20], "get_sites_for_directori": 14, "interrobotmanag": 16, "test_interrobot_imag": [13, 16, 17, 18], "to_int_map": 23, "ordin": 23, "utf": 25, "explicitli": 25, "cp1252": 25, "internation": 25, "to_isoformat_zulu": 25, "z": [22, 25], "from_isoformat_zulu": 25, "perform": [14, 27], "markdown": [10, 11, 22, 24, 26, 27, 28], "transform": [21, 24, 28], "reduc": [21, 28], "improv": 28, "readabl": 28, "equival": [], "searchabl": [2, 4, 28], "engin": [27, 28], "complex": 28, "express": 28, "combin": [1, 2, 28], "yourself": [27, 28], "inspect": 28, "ui": [10, 11, 26, 28], "expand": 28, "privaci": 28, "keyword": [1, 5, 22, 28], "polici": 28, "exact": 28, "12345": 28, "equal": 28, "400": 28, "greater": 28, "bodi": 28, "often": 28, "alwai": 28, "xml": 28, "both": 28, "either": 28, "login": [10, 28], "signin": 28, "form": [22, 28], "fullext": 28, "column": [22, 28], "rather": 28, "entir": 28, "restrict": 28, "approach": [1, 28], "look": 28, "pattern": [21, 27, 28], "tabl": [14, 28], "j": 28, "multitud": 28, "broad": 28, "particularli": [14, 28], "without": [22, 27, 28], "system": 28, "embed": 27, "stylesheet": 28, "syndic": 28, "plain": 28, "word": [21, 28], "uncategor": 28, "boundar": 28, "boundari": 28, "h1": 28, "uniqu": [4, 21], "extern": [4, 22, 28], "branch": 4, "origin": 4, "continu": 4, "singular": 4, "expans": [], "nbsp": [], "spread": [], "asset": [2, 27], "test_thumbnails_sync": 16, "synchron": 22, "wrapper": [], "set_extra": 23, "extra_nam": 23, "extra_valu": 23, "br": 24, "snippet": [10, 11, 22, 26, 28], "contextu": 28, "refin": 28, "down": [22, 28], "also": [27, 28], "great": 28, "render": [22, 24, 28], "old": [27, 28], "school": 28, "hit": [21, 28], "highlight": [10, 11, 21, 26, 28], "googl": 28, "1999": 28, "classifi": 14, "run_pragmar_search_test": 14, "batteri": [14, 27], "run_pragmar_image_test": 14, "pragmar_site_id": 14, "run_sites_resources_test": 14, "example_site_id": 14, "run_pragmar_tokenizer_test": 14, "ft": 14, "hyphen": [14, 15], "underscor": 14, "challeng": 14, "thu": 14, "dedic": 14, "parser": [14, 21, 27], "run_pragmar_site_test": 14, "run_pragmar_sort_test": 14, "run_pragmar_content_test": 14, "html_lenienc": 14, "run_pragmar_report": 14, "head": 14, "test_interrobot_search": 16, "test_interrobot_content_pars": 16, "test_interrobot_mcp_featur": 16, "test_interrobot_advanced_site_featur": 16, "test_report": [13, 15, 16, 17, 18, 19, 20], "test_katana_search": 17, "test_pragmar_token": [13, 17], "test_siteone_search": 18, "test_siteone_content_pars": 18, "test_siteone_advanced_featur": 18, "cover": [15, 18, 27], "test_warc_search": 19, "test_warc_random_sort": [], "test_warc_content_pars": 19, "test_wget_search": 20, "test_wget_content_pars": 20, "routin": 8, "track": 14, "whether": [14, 22], "concurr": 14, "binari": 14, "unread": 14, "determin": 14, "clean": [14, 22], "temp": 14, "weird": 14, "normal": 14, "toolkit": 27, "necessari": 27, "freestyl": 27, "figur": 27, "react": 27, "ones": 27, "These": [21, 27, 28], "copi": 27, "past": 27, "raw": [10, 22, 27], "logic": 27, "procedur": 27, "loop": 27, "gopher": 27, "servic": 27, "shortcut": 27, "select": [24, 27], "less": 27, "same": 27, "download": [10, 27], "categori": 27, "audit": [2, 27], "auditseo": 27, "md": 27, "optim": [14, 27], "dive": 27, "deeper": 27, "audit404": 27, "broken": 27, "issu": 27, "suggest": 27, "fix": 27, "auditperf": 27, "speed": [24, 27], "real": 27, "talk": 27, "auditfil": 27, "discov": 27, "composit": 27, "fashion": 27, "inspir": 27, "yesteryear": 27, "testsearch": 27, "self": [1, 8, 27], "inconsist": 27, "subsequ": 27, "convers": 27, "achiev": 28, "desir": 28, "view": [10, 28], "visual": 28, "vari": [], "linux": [1, 2], "gui": [5, 8], "secur": 8, "focus": [8, 22, 28], "extrasxpath": [14, 28], "domain": [5, 15], "xpath": [11, 26, 28], "selector": [21, 28], "scrape": 28, "outerhtml": 28, "ignor": 28, "One": 28, "produc": 28, "roughli": 28, "so": [21, 24, 28], "session": [11, 13, 26, 28], "idea": 28, "cours": 28, "care": 28, "notic": 28, "affin": 28, "chat": 28, "budget": 28, "1the": [], "href": [], "outer": [], "extrasregex": [14, 28], "test_interrobot_sort": 16, "test_katana_sort": 17, "test_siteone_sort": 18, "test_warc_sort": 19, "test_wget_sort": 20, "get_markdown": 21, "get_regex": 21, "snippetcontentextractor": 21, "lxml": 21, "compon": 21, "markup": 21, "comment": 21, "priorit": 21, "noisier": 21, "priority_ord": 21, "document_text": 21, "document_attribut": 21, "document_com": 21, "document_markup": 21, "get_snippet": 21, "excerpt": 21, "around": 21, "term": [1, 15, 21, 22], "find_snippets_in_text": 21, "max_snippet": 21, "15": 21, "group_nam": 21, "whole": 21, "surround": 21, "bold": 21, "insensit": 21, "max_snippets_matched_count": 21, "reserv": 21, "futur": 21, "max_snippets_context_s": 21, "side": 21, "invalid": 21, "get_xpath": 21, "regex": [11, 26, 28], "regular": 28, "phone": [], "part": [], "relationship": [], "subset": 28, "independ": 28, "core": [10, 28], "quickli": 28, "judici": 28, "crunch": 28, "top": 28, "archiveboxmanag": 13, "reus": 13, "folder": 13, "treat": [13, 24], "archiveboxcrawl": 13, "individu": [1, 13, 22], "archiveboxtest": 13, "multi": [13, 15], "test_archivebox_puls": 13, "test_archivebox_sit": 13, "test_archivebox_search": 13, "test_archivebox_resourc": 13, "test_archivebox_sort": 13, "test_archivebox_content_pars": 13, "test_archivebox_url_reconstruct": 13, "reconstruct": [13, 15], "test_archivebox_dedupl": 13, "dedupl": 13, "test_archivebox_metadata_pars": 13, "test_archivebox_timestamped_structur": 13, "test_archivebox_error_resili": 13, "resili": 13, "malform": 13, "miss": 13, "test_archivebox_multi_sit": 13, "get_basic_headers2": [], "mime": 14, "httrackmanag": 15, "httrackcrawl": 15, "ht": 15, "potenti": 15, "httracktest": 15, "test_httrack_puls": 15, "test_httrack_sit": 15, "test_httrack_search": 15, "test_httrack_resourc": 15, "test_httrack_imag": 15, "test_httrack_sort": 15, "test_httrack_content_pars": 15, "test_httrack_token": 15, "test_httrack_log_parsing_featur": 15, "relat": [1, 15], "test_httrack_url_reconstruct": 15, "test_httrack_domain_detect": 15, "test_httrack_file_exclus": 15, "exclud": 15, "test_httrack_advanced_featur": 15, "markdowntransform": 21, "memoiz": 21, "xslt": 21, "get_xslt_transform": 21, "due": 21, "templatetest": 24, "why": 24, "reason": 24, "thing": 24, "tailor": 24, "aggress": 24, "remov": [22, 24], "html2text": 24, "block": 24, "decend": 24, "p": 24, "test_core_html": 24, "httrack": [0, 8, 9, 12], "archivebox": [0, 8, 9, 12], "power": 1, "solut": 1, "screenshot": 1, "brows": 1, "browser": 1, "bookmark": 1, "long": 1, "robust": [], "duplic": [], "offici": [], "my": 2, "cd": 1, "init": 1, "snapshot": [], "ran": [], "rich": 1, "media": [], "export": [], "backup": [], "sqlite3": 1, "research": 1, "copier": 2, "establish": 2, "wizard": 2, "anoth": 2, "c": [], "norm": [], "conduct": 2, "easi": 2, "maintain": [], "unlik": 1, "focu": [1, 22], "togeth": 1, "parent": 1, "webadmin": 1, "templat": [11, 26], "docker": 1, "untest": 1, "brew": 1, "get_crawl": 12, "crawler_nam": 12, "lazi": 12, "highlightspan": 22, "span": 22, "end": 22, "highlightprocessor": 22, "quoted_phrase_pattern": 22, "compil": 22, "word_pattern": 22, "b": 22, "w": 22, "snippet_marker_pattern": 22, "za": 22, "_": 22, "ignore_word": 22, "extract_search_term": 22, "find_highlights_in_text": 22, "search_term": 22, "extract_snippet_highlight": 22, "snippet_text": 22, "marker": 22, "merge_overlapping_highlight": 22, "overlap": 22, "adjac": 22, "render_text_with_highlight": 22, "stdscr": 22, "x": 22, "y": 22, "max_width": 22, "normal_styl": 22, "hit_styl": 22, "searchmanag": 22, "debounc": 22, "never": 22, "privat": 22, "interactivesess": 22, "autosearch": 22, "immedi": 22, "trigger": 22, "cancel_pend": 22, "cancel": 22, "pend": 22, "timer": 22, "check_pend": 22, "updat": 22, "has_pend": 22, "is_search": 22, "coordin": 22, "ui_stat": 22, "uistat": 22, "ui_focus": 22, "uifocus": 22, "searchdocumentview": 22, "searchresultsview": 22, "searchform": 22, "searchformview": 22, "searchman": 22, "debug_add": 22, "msg": 22, "debug_clear": 22, "public": 22, "set_ui_st": 22, "transit": 22, "cleanli": 22, "set_init_input_arg": 22, "set_init_crawl": 22, "set_init_sit": 22, "set_init_searchform": 22, "basecursesview": 22, "get_theme_color_pair": 22, "theme": 22, "themedefinit": 22, "documentmod": 22, "2": [9, 22], "navigationdirect": 22, "left": 22, "right": 22, "scrolldirect": 22, "searchfiltertyp": 22, "document_mod": 22, "51": 22, "header_act": 22, "7": 22, "17": 22, "header_inact": 22, "233": 22, "header_out": 22, "235": 22, "help_link": 22, "5": 22, "27": 22, "http_error": 22, "6": 22, "88": 22, "http_warn": 22, "130": 22, "inactive_queri": 22, "245": 22, "237": 22, "snippet_default": 22, "243": 22, "snippet_highlight": 22, "232": 22, "ui_error": 22, "11": 22, "search_form": 22, "search_result": 22, "search_init": 22, "safe_addstr": 22, "addstr": 22, "edg": 22, "inputradio": 22, "radio": 22, "possibl": 22, "off": 22, "state1": 22, "state2": 22, "inputradiogroup": 22, "label": 22, "inputradiost": 22, "defin": 22, "current_st": 22, "display_label": 22, "next_stat": 22, "cycl": 22, "next": 22, "field_index": 22, "set_stat": 22, "layout": 22, "calculate_group_width": 22, "calcul": 22, "set_layout_constraint": 22, "available_width": 22, "available_height": 22, "is_constrain": 22, "constraint": 22, "grid": 22, "horizont": 22, "vertic": 22, "constrain": 22, "affect": 22, "get_grid_posit": 22, "radio_index": 22, "linear": 22, "row": 22, "get_index_from_grid": 22, "col": 22, "bound": 22, "navigate_left": 22, "current_radio_index": 22, "navigate_right": 22, "navigate_to_row": 22, "target_row": 22, "from_column": 22, "get_row_from_index": 22, "inter": 22, "namedtupl": 22, "alia": 22, "__new__": 22, "_cl": 22, "inputtext": 22, "cursor": 22, "consolid": 22, "initial_valu": 22, "max_length": 22, "unlimit": 22, "backspac": 22, "reset": 22, "delet": 22, "move": 22, "handle_input": 22, "keyboard": 22, "curs": 22, "home": 22, "insert_char": 22, "char": 22, "is_empti": 22, "move_cursor_left": 22, "move_cursor_right": 22, "box": 22, "set_valu": 22, "new_valu": 22, "viewbound": 22, "get_extra": 23, "enhanc": [], "dynam": [], "through": [], "extend": [], "placehold": [], "persist": [], "workflow": [], "experi": [], "classic": 10, "doubl": 10, "archvi": [], "against": 10, "local": 10, "realiz": 10, "ssh": 10, "sit": 10, "sync": 10, "multifactor": 10, "drudgeri": 10, "datasourc": [], "app": [], "bash": [], "manual": 10, "solar": 10, "eclips": 10, "org": 9, "expos": 10, "layer": 10, "tui": 10, "bypass": 10, "altogeth": 10, "human": 10, "aspect": 10, "termial": 10}, "objects": {"": [[11, 0, 0, "-", "mcp_server_webcrawl"]], "mcp_server_webcrawl": [[12, 0, 0, "-", "crawlers"], [21, 0, 0, "-", "extras"], [22, 0, 0, "-", "interactive"], [23, 0, 0, "-", "models"], [11, 0, 0, "-", "settings"], [11, 0, 0, "-", "settings_local"], [24, 0, 0, "-", "templates"], [25, 0, 0, "-", "utils"]], "mcp_server_webcrawl.crawlers": [[13, 0, 0, "-", "archivebox"], [14, 0, 0, "-", "base"], [12, 3, 1, "", "get_crawler"], [12, 3, 1, "", "get_fixture_directory"], [15, 0, 0, "-", "httrack"], [16, 0, 0, "-", "interrobot"], [17, 0, 0, "-", "katana"], [18, 0, 0, "-", "siteone"], [19, 0, 0, "-", "warc"], [20, 0, 0, "-", "wget"]], "mcp_server_webcrawl.crawlers.archivebox": [[13, 0, 0, "-", "adapter"], [13, 0, 0, "-", "crawler"], [13, 0, 0, "-", "tests"]], "mcp_server_webcrawl.crawlers.archivebox.adapter": [[13, 1, 1, "", "ArchiveBoxManager"], [13, 3, 1, "", "get_resources"], [13, 3, 1, "", "get_sites"]], "mcp_server_webcrawl.crawlers.archivebox.adapter.ArchiveBoxManager": [[13, 2, 1, "", "__init__"]], "mcp_server_webcrawl.crawlers.archivebox.crawler": [[13, 1, 1, "", "ArchiveBoxCrawler"]], "mcp_server_webcrawl.crawlers.archivebox.crawler.ArchiveBoxCrawler": [[13, 2, 1, "", "__init__"]], "mcp_server_webcrawl.crawlers.archivebox.tests": [[13, 1, 1, "", "ArchiveBoxTests"]], "mcp_server_webcrawl.crawlers.archivebox.tests.ArchiveBoxTests": [[13, 2, 1, "", "setUp"], [13, 2, 1, "", "test_archivebox_content_parsing"], [13, 2, 1, "", "test_archivebox_deduplication"], [13, 2, 1, "", "test_archivebox_error_resilience"], [13, 2, 1, "", "test_archivebox_metadata_parsing"], [13, 2, 1, "", "test_archivebox_multi_site"], [13, 2, 1, "", "test_archivebox_pulse"], [13, 2, 1, "", "test_archivebox_resources"], [13, 2, 1, "", "test_archivebox_search"], [13, 2, 1, "", "test_archivebox_sites"], [13, 2, 1, "", "test_archivebox_sorts"], [13, 2, 1, "", "test_archivebox_timestamped_structure"], [13, 2, 1, "", "test_archivebox_url_reconstruction"], [13, 2, 1, "", "test_interrobot_images"], [13, 2, 1, "", "test_pragmar_tokenizer"], [13, 2, 1, "", "test_report"]], "mcp_server_webcrawl.crawlers.base": [[14, 0, 0, "-", "adapter"], [14, 0, 0, "-", "api"], [14, 0, 0, "-", "crawler"], [14, 0, 0, "-", "indexed"], [14, 0, 0, "-", "tests"]], "mcp_server_webcrawl.crawlers.base.adapter": [[14, 1, 1, "", "BaseManager"], [14, 1, 1, "", "IndexState"], [14, 1, 1, "", "IndexStatus"], [14, 1, 1, "", "SitesGroup"], [14, 1, 1, "", "SitesStat"]], "mcp_server_webcrawl.crawlers.base.adapter.BaseManager": [[14, 2, 1, "", "__init__"], [14, 2, 1, "", "decruft_path"], [14, 2, 1, "", "get_basic_headers"], [14, 2, 1, "", "get_resources_for_sites_group"], [14, 2, 1, "", "get_stats"], [14, 2, 1, "", "read_file_contents"], [14, 2, 1, "", "read_files"], [14, 2, 1, "", "string_to_id"]], "mcp_server_webcrawl.crawlers.base.adapter.IndexState": [[14, 2, 1, "", "__init__"], [14, 4, 1, "", "duration"], [14, 2, 1, "", "increment_processed"], [14, 2, 1, "", "is_timeout"], [14, 5, 1, "", "processed"], [14, 2, 1, "", "set_status"], [14, 5, 1, "", "status"], [14, 5, 1, "", "time_end"], [14, 5, 1, "", "time_start"], [14, 2, 1, "", "to_dict"]], "mcp_server_webcrawl.crawlers.base.adapter.IndexStatus": [[14, 5, 1, "", "COMPLETE"], [14, 5, 1, "", "FAILED"], [14, 5, 1, "", "IDLE"], [14, 5, 1, "", "INDEXING"], [14, 5, 1, "", "PARTIAL"], [14, 5, 1, "", "REMOTE"], [14, 5, 1, "", "UNDEFINED"]], "mcp_server_webcrawl.crawlers.base.adapter.SitesGroup": [[14, 2, 1, "", "__init__"], [14, 2, 1, "", "get_sites"]], "mcp_server_webcrawl.crawlers.base.adapter.SitesStat": [[14, 2, 1, "", "__init__"]], "mcp_server_webcrawl.crawlers.base.api": [[14, 1, 1, "", "BaseJsonApi"], [14, 1, 1, "", "BaseJsonApiEncoder"]], "mcp_server_webcrawl.crawlers.base.api.BaseJsonApi": [[14, 2, 1, "", "__init__"], [14, 2, 1, "", "append_error"], [14, 2, 1, "", "get_results"], [14, 2, 1, "", "set_results"], [14, 2, 1, "", "to_dict"], [14, 2, 1, "", "to_json"], [14, 4, 1, "", "total"]], "mcp_server_webcrawl.crawlers.base.api.BaseJsonApiEncoder": [[14, 2, 1, "", "default"]], "mcp_server_webcrawl.crawlers.base.crawler": [[14, 1, 1, "", "BaseCrawler"]], "mcp_server_webcrawl.crawlers.base.crawler.BaseCrawler": [[14, 2, 1, "", "__init__"], [14, 4, 1, "", "datasrc"], [14, 2, 1, "", "get_initialization_options"], [14, 2, 1, "", "get_resources_api"], [14, 2, 1, "", "get_resources_api_json"], [14, 2, 1, "", "get_sites_api"], [14, 2, 1, "", "get_sites_api_json"], [14, 2, 1, "", "get_thumbnails"], [14, 2, 1, "", "mcp_call_tool"], [14, 2, 1, "", "mcp_list_prompts"], [14, 2, 1, "", "mcp_list_resources"], [14, 2, 1, "", "mcp_list_tools"], [14, 2, 1, "", "serve"]], "mcp_server_webcrawl.crawlers.base.indexed": [[14, 1, 1, "", "IndexedCrawler"], [14, 1, 1, "", "IndexedManager"]], "mcp_server_webcrawl.crawlers.base.indexed.IndexedCrawler": [[14, 2, 1, "", "__init__"], [14, 2, 1, "", "mcp_list_tools"]], "mcp_server_webcrawl.crawlers.base.indexed.IndexedManager": [[14, 2, 1, "", "__init__"], [14, 2, 1, "", "get_connection"], [14, 2, 1, "", "get_sites_for_directories"]], "mcp_server_webcrawl.crawlers.base.tests": [[14, 1, 1, "", "BaseCrawlerTests"]], "mcp_server_webcrawl.crawlers.base.tests.BaseCrawlerTests": [[14, 2, 1, "", "run_pragmar_content_tests"], [14, 2, 1, "", "run_pragmar_image_tests"], [14, 2, 1, "", "run_pragmar_report"], [14, 2, 1, "", "run_pragmar_search_tests"], [14, 2, 1, "", "run_pragmar_site_tests"], [14, 2, 1, "", "run_pragmar_sort_tests"], [14, 2, 1, "", "run_pragmar_tokenizer_tests"], [14, 2, 1, "", "run_sites_resources_tests"], [14, 2, 1, "", "setUp"]], "mcp_server_webcrawl.crawlers.httrack": [[15, 0, 0, "-", "adapter"], [15, 0, 0, "-", "crawler"], [15, 0, 0, "-", "tests"]], "mcp_server_webcrawl.crawlers.httrack.adapter": [[15, 1, 1, "", "HtTrackManager"], [15, 3, 1, "", "get_resources"], [15, 3, 1, "", "get_sites"]], "mcp_server_webcrawl.crawlers.httrack.adapter.HtTrackManager": [[15, 2, 1, "", "__init__"]], "mcp_server_webcrawl.crawlers.httrack.crawler": [[15, 1, 1, "", "HtTrackCrawler"]], "mcp_server_webcrawl.crawlers.httrack.crawler.HtTrackCrawler": [[15, 2, 1, "", "__init__"]], "mcp_server_webcrawl.crawlers.httrack.tests": [[15, 1, 1, "", "HtTrackTests"]], "mcp_server_webcrawl.crawlers.httrack.tests.HtTrackTests": [[15, 2, 1, "", "setUp"], [15, 2, 1, "", "test_httrack_advanced_features"], [15, 2, 1, "", "test_httrack_content_parsing"], [15, 2, 1, "", "test_httrack_domain_detection"], [15, 2, 1, "", "test_httrack_file_exclusion"], [15, 2, 1, "", "test_httrack_images"], [15, 2, 1, "", "test_httrack_log_parsing_features"], [15, 2, 1, "", "test_httrack_pulse"], [15, 2, 1, "", "test_httrack_resources"], [15, 2, 1, "", "test_httrack_search"], [15, 2, 1, "", "test_httrack_sites"], [15, 2, 1, "", "test_httrack_sorts"], [15, 2, 1, "", "test_httrack_tokenizer"], [15, 2, 1, "", "test_httrack_url_reconstruction"], [15, 2, 1, "", "test_report"]], "mcp_server_webcrawl.crawlers.interrobot": [[16, 0, 0, "-", "adapter"], [16, 0, 0, "-", "crawler"], [16, 0, 0, "-", "tests"]], "mcp_server_webcrawl.crawlers.interrobot.adapter": [[16, 1, 1, "", "InterroBotManager"], [16, 3, 1, "", "get_resources"], [16, 3, 1, "", "get_sites"]], "mcp_server_webcrawl.crawlers.interrobot.adapter.InterroBotManager": [[16, 2, 1, "", "__init__"], [16, 2, 1, "", "get_connection"]], "mcp_server_webcrawl.crawlers.interrobot.crawler": [[16, 1, 1, "", "InterroBotCrawler"]], "mcp_server_webcrawl.crawlers.interrobot.crawler.InterroBotCrawler": [[16, 2, 1, "", "__init__"], [16, 2, 1, "", "mcp_list_tools"]], "mcp_server_webcrawl.crawlers.interrobot.tests": [[16, 1, 1, "", "InterroBotTests"]], "mcp_server_webcrawl.crawlers.interrobot.tests.InterroBotTests": [[16, 2, 1, "", "setUp"], [16, 2, 1, "", "test_interrobot_advanced_site_features"], [16, 2, 1, "", "test_interrobot_content_parsing"], [16, 2, 1, "", "test_interrobot_images"], [16, 2, 1, "", "test_interrobot_mcp_features"], [16, 2, 1, "", "test_interrobot_pulse"], [16, 2, 1, "", "test_interrobot_resources"], [16, 2, 1, "", "test_interrobot_search"], [16, 2, 1, "", "test_interrobot_sites"], [16, 2, 1, "", "test_interrobot_sorts"], [16, 2, 1, "", "test_report"], [16, 2, 1, "", "test_thumbnails_sync"]], "mcp_server_webcrawl.crawlers.katana": [[17, 0, 0, "-", "adapter"], [17, 0, 0, "-", "crawler"], [17, 0, 0, "-", "tests"]], "mcp_server_webcrawl.crawlers.katana.adapter": [[17, 1, 1, "", "KatanaManager"], [17, 3, 1, "", "get_resources"], [17, 3, 1, "", "get_sites"]], "mcp_server_webcrawl.crawlers.katana.adapter.KatanaManager": [[17, 2, 1, "", "__init__"]], "mcp_server_webcrawl.crawlers.katana.crawler": [[17, 1, 1, "", "KatanaCrawler"]], "mcp_server_webcrawl.crawlers.katana.crawler.KatanaCrawler": [[17, 2, 1, "", "__init__"]], "mcp_server_webcrawl.crawlers.katana.tests": [[17, 1, 1, "", "KatanaTests"]], "mcp_server_webcrawl.crawlers.katana.tests.KatanaTests": [[17, 2, 1, "", "setUp"], [17, 2, 1, "", "test_interrobot_images"], [17, 2, 1, "", "test_katana_content_parsing"], [17, 2, 1, "", "test_katana_pulse"], [17, 2, 1, "", "test_katana_resources"], [17, 2, 1, "", "test_katana_search"], [17, 2, 1, "", "test_katana_sites"], [17, 2, 1, "", "test_katana_sorts"], [17, 2, 1, "", "test_pragmar_tokenizer"], [17, 2, 1, "", "test_report"]], "mcp_server_webcrawl.crawlers.siteone": [[18, 0, 0, "-", "adapter"], [18, 0, 0, "-", "crawler"], [18, 0, 0, "-", "tests"]], "mcp_server_webcrawl.crawlers.siteone.adapter": [[18, 1, 1, "", "SiteOneManager"], [18, 3, 1, "", "get_resources"], [18, 3, 1, "", "get_sites"]], "mcp_server_webcrawl.crawlers.siteone.adapter.SiteOneManager": [[18, 2, 1, "", "__init__"]], "mcp_server_webcrawl.crawlers.siteone.crawler": [[18, 1, 1, "", "SiteOneCrawler"]], "mcp_server_webcrawl.crawlers.siteone.crawler.SiteOneCrawler": [[18, 2, 1, "", "__init__"]], "mcp_server_webcrawl.crawlers.siteone.tests": [[18, 1, 1, "", "SiteOneTests"]], "mcp_server_webcrawl.crawlers.siteone.tests.SiteOneTests": [[18, 2, 1, "", "setUp"], [18, 2, 1, "", "test_interrobot_images"], [18, 2, 1, "", "test_report"], [18, 2, 1, "", "test_siteone_advanced_features"], [18, 2, 1, "", "test_siteone_content_parsing"], [18, 2, 1, "", "test_siteone_pulse"], [18, 2, 1, "", "test_siteone_resources"], [18, 2, 1, "", "test_siteone_search"], [18, 2, 1, "", "test_siteone_sites"], [18, 2, 1, "", "test_siteone_sorts"]], "mcp_server_webcrawl.crawlers.warc": [[19, 0, 0, "-", "adapter"], [19, 0, 0, "-", "crawler"], [19, 0, 0, "-", "tests"]], "mcp_server_webcrawl.crawlers.warc.adapter": [[19, 1, 1, "", "WarcManager"], [19, 3, 1, "", "get_resources"], [19, 3, 1, "", "get_sites"]], "mcp_server_webcrawl.crawlers.warc.adapter.WarcManager": [[19, 2, 1, "", "__init__"]], "mcp_server_webcrawl.crawlers.warc.crawler": [[19, 1, 1, "", "WarcCrawler"]], "mcp_server_webcrawl.crawlers.warc.crawler.WarcCrawler": [[19, 2, 1, "", "__init__"]], "mcp_server_webcrawl.crawlers.warc.tests": [[19, 1, 1, "", "WarcTests"]], "mcp_server_webcrawl.crawlers.warc.tests.WarcTests": [[19, 2, 1, "", "setUp"], [19, 2, 1, "", "test_report"], [19, 2, 1, "", "test_warc_content_parsing"], [19, 2, 1, "", "test_warc_pulse"], [19, 2, 1, "", "test_warc_resources"], [19, 2, 1, "", "test_warc_search"], [19, 2, 1, "", "test_warc_sites"], [19, 2, 1, "", "test_warc_sorts"]], "mcp_server_webcrawl.crawlers.wget": [[20, 0, 0, "-", "adapter"], [20, 0, 0, "-", "crawler"], [20, 0, 0, "-", "tests"]], "mcp_server_webcrawl.crawlers.wget.adapter": [[20, 1, 1, "", "WgetManager"], [20, 3, 1, "", "get_resources"], [20, 3, 1, "", "get_sites"]], "mcp_server_webcrawl.crawlers.wget.adapter.WgetManager": [[20, 2, 1, "", "__init__"]], "mcp_server_webcrawl.crawlers.wget.crawler": [[20, 1, 1, "", "WgetCrawler"]], "mcp_server_webcrawl.crawlers.wget.crawler.WgetCrawler": [[20, 2, 1, "", "__init__"]], "mcp_server_webcrawl.crawlers.wget.tests": [[20, 1, 1, "", "WgetTests"]], "mcp_server_webcrawl.crawlers.wget.tests.WgetTests": [[20, 2, 1, "", "setUp"], [20, 2, 1, "", "test_report"], [20, 2, 1, "", "test_wget_content_parsing"], [20, 2, 1, "", "test_wget_pulse"], [20, 2, 1, "", "test_wget_resources"], [20, 2, 1, "", "test_wget_search"], [20, 2, 1, "", "test_wget_sites"], [20, 2, 1, "", "test_wget_sorts"]], "mcp_server_webcrawl.extras": [[21, 0, 0, "-", "markdown"], [21, 0, 0, "-", "regex"], [21, 0, 0, "-", "snippets"], [21, 0, 0, "-", "thumbnails"], [21, 0, 0, "-", "xpath"]], "mcp_server_webcrawl.extras.markdown": [[21, 1, 1, "", "MarkdownTransformer"], [21, 3, 1, "", "get_markdown"]], "mcp_server_webcrawl.extras.markdown.MarkdownTransformer": [[21, 2, 1, "", "get_xslt_transform"]], "mcp_server_webcrawl.extras.regex": [[21, 3, 1, "", "get_regex"]], "mcp_server_webcrawl.extras.snippets": [[21, 1, 1, "", "SnippetContentExtractor"], [21, 3, 1, "", "find_snippets_in_text"], [21, 3, 1, "", "get_snippets"]], "mcp_server_webcrawl.extras.snippets.SnippetContentExtractor": [[21, 5, 1, "", "PRIORITY_ORDER"], [21, 2, 1, "", "__init__"]], "mcp_server_webcrawl.extras.thumbnails": [[21, 1, 1, "", "ThumbnailManager"]], "mcp_server_webcrawl.extras.thumbnails.ThumbnailManager": [[21, 2, 1, "", "__init__"], [21, 2, 1, "", "get_thumbnails"]], "mcp_server_webcrawl.extras.xpath": [[21, 3, 1, "", "get_xpath"]], "mcp_server_webcrawl.interactive": [[22, 0, 0, "-", "highlights"], [22, 0, 0, "-", "search"], [22, 0, 0, "-", "session"], [22, 0, 0, "-", "ui"]], "mcp_server_webcrawl.interactive.highlights": [[22, 1, 1, "", "HighlightProcessor"], [22, 1, 1, "", "HighlightSpan"]], "mcp_server_webcrawl.interactive.highlights.HighlightProcessor": [[22, 5, 1, "", "IGNORE_WORDS"], [22, 5, 1, "", "QUOTED_PHRASE_PATTERN"], [22, 5, 1, "", "SNIPPET_MARKER_PATTERN"], [22, 5, 1, "", "WORD_PATTERN"], [22, 2, 1, "", "extract_search_terms"], [22, 2, 1, "", "extract_snippet_highlights"], [22, 2, 1, "", "find_highlights_in_text"], [22, 2, 1, "", "merge_overlapping_highlights"], [22, 2, 1, "", "render_text_with_highlights"]], "mcp_server_webcrawl.interactive.highlights.HighlightSpan": [[22, 2, 1, "", "__init__"], [22, 5, 1, "", "end"], [22, 5, 1, "", "start"], [22, 5, 1, "", "text"]], "mcp_server_webcrawl.interactive.search": [[22, 1, 1, "", "SearchManager"]], "mcp_server_webcrawl.interactive.search.SearchManager": [[22, 2, 1, "", "__init__"], [22, 2, 1, "", "autosearch"], [22, 2, 1, "", "cancel_pending"], [22, 2, 1, "", "check_pending"], [22, 2, 1, "", "cleanup"], [22, 2, 1, "", "has_pending"], [22, 2, 1, "", "is_searching"]], "mcp_server_webcrawl.interactive.session": [[22, 1, 1, "", "InteractiveSession"]], "mcp_server_webcrawl.interactive.session.InteractiveSession": [[22, 2, 1, "", "__init__"], [22, 4, 1, "", "crawler"], [22, 2, 1, "", "debug_add"], [22, 2, 1, "", "debug_clear"], [22, 4, 1, "", "document"], [22, 2, 1, "", "get_theme_color_pair"], [22, 4, 1, "", "results"], [22, 2, 1, "", "run"], [22, 4, 1, "", "searchform"], [22, 4, 1, "", "searchman"], [22, 2, 1, "", "set_init_crawler"], [22, 2, 1, "", "set_init_input_args"], [22, 2, 1, "", "set_init_searchform"], [22, 2, 1, "", "set_init_sites"], [22, 2, 1, "", "set_ui_state"], [22, 4, 1, "", "sites"], [22, 4, 1, "", "ui_focused"], [22, 4, 1, "", "ui_state"]], "mcp_server_webcrawl.interactive.ui": [[22, 1, 1, "", "DocumentMode"], [22, 1, 1, "", "InputRadio"], [22, 1, 1, "", "InputRadioGroup"], [22, 1, 1, "", "InputRadioState"], [22, 1, 1, "", "InputText"], [22, 1, 1, "", "NavigationDirection"], [22, 1, 1, "", "ScrollDirection"], [22, 1, 1, "", "SearchFilterType"], [22, 1, 1, "", "ThemeDefinition"], [22, 1, 1, "", "UiFocusable"], [22, 1, 1, "", "UiState"], [22, 1, 1, "", "ViewBounds"], [22, 3, 1, "", "safe_addstr"]], "mcp_server_webcrawl.interactive.ui.DocumentMode": [[22, 5, 1, "", "HEADERS"], [22, 5, 1, "", "MARKDOWN"], [22, 5, 1, "", "RAW"]], "mcp_server_webcrawl.interactive.ui.InputRadio": [[22, 2, 1, "", "__init__"], [22, 4, 1, "", "current_state"], [22, 4, 1, "", "display_label"], [22, 2, 1, "", "next_state"], [22, 2, 1, "", "render"], [22, 2, 1, "", "set_state"], [22, 2, 1, "", "set_states"], [22, 4, 1, "", "value"]], "mcp_server_webcrawl.interactive.ui.InputRadioGroup": [[22, 2, 1, "", "__init__"], [22, 2, 1, "", "calculate_group_width"], [22, 2, 1, "", "clear"], [22, 2, 1, "", "get_grid_position"], [22, 2, 1, "", "get_index_from_grid"], [22, 2, 1, "", "get_row_from_index"], [22, 2, 1, "", "navigate_left"], [22, 2, 1, "", "navigate_right"], [22, 2, 1, "", "navigate_to_row"], [22, 2, 1, "", "set_layout_constraints"], [22, 4, 1, "", "value"]], "mcp_server_webcrawl.interactive.ui.InputRadioState": [[22, 2, 1, "", "__new__"], [22, 5, 1, "", "label"], [22, 5, 1, "", "value"]], "mcp_server_webcrawl.interactive.ui.InputText": [[22, 2, 1, "", "__init__"], [22, 2, 1, "", "backspace"], [22, 2, 1, "", "clear"], [22, 2, 1, "", "delete"], [22, 2, 1, "", "end"], [22, 2, 1, "", "handle_input"], [22, 2, 1, "", "home"], [22, 2, 1, "", "insert_char"], [22, 2, 1, "", "is_empty"], [22, 2, 1, "", "move_cursor_left"], [22, 2, 1, "", "move_cursor_right"], [22, 2, 1, "", "render"], [22, 2, 1, "", "set_value"]], "mcp_server_webcrawl.interactive.ui.NavigationDirection": [[22, 5, 1, "", "DOWN"], [22, 5, 1, "", "LEFT"], [22, 5, 1, "", "RIGHT"], [22, 5, 1, "", "UP"]], "mcp_server_webcrawl.interactive.ui.ScrollDirection": [[22, 5, 1, "", "DOWN"], [22, 5, 1, "", "UP"]], "mcp_server_webcrawl.interactive.ui.SearchFilterType": [[22, 5, 1, "", "ANY"], [22, 5, 1, "", "PAGES"]], "mcp_server_webcrawl.interactive.ui.ThemeDefinition": [[22, 5, 1, "", "DOCUMENT_MODE"], [22, 5, 1, "", "HEADER_ACTIVE"], [22, 5, 1, "", "HEADER_INACTIVE"], [22, 5, 1, "", "HEADER_OUTER"], [22, 5, 1, "", "HELP_LINK"], [22, 5, 1, "", "HTTP_ERROR"], [22, 5, 1, "", "HTTP_WARN"], [22, 5, 1, "", "INACTIVE_QUERY"], [22, 5, 1, "", "SNIPPET_DEFAULT"], [22, 5, 1, "", "SNIPPET_HIGHLIGHT"], [22, 5, 1, "", "UI_ERROR"]], "mcp_server_webcrawl.interactive.ui.UiFocusable": [[22, 5, 1, "", "SEARCH_FORM"], [22, 5, 1, "", "SEARCH_RESULTS"], [22, 5, 1, "", "UNDEFINED"]], "mcp_server_webcrawl.interactive.ui.UiState": [[22, 5, 1, "", "DOCUMENT"], [22, 5, 1, "", "HELP"], [22, 5, 1, "", "REQUIREMENTS"], [22, 5, 1, "", "SEARCH_INIT"], [22, 5, 1, "", "SEARCH_RESULTS"], [22, 5, 1, "", "UNDEFINED"]], "mcp_server_webcrawl.interactive.ui.ViewBounds": [[22, 2, 1, "", "__init__"]], "mcp_server_webcrawl.models": [[23, 0, 0, "-", "resources"], [23, 0, 0, "-", "sites"]], "mcp_server_webcrawl.models.resources": [[23, 1, 1, "", "ResourceResult"], [23, 1, 1, "", "ResourceResultType"]], "mcp_server_webcrawl.models.resources.ResourceResult": [[23, 2, 1, "", "__init__"], [23, 2, 1, "", "get_extra"], [23, 2, 1, "", "set_extra"], [23, 2, 1, "", "to_dict"], [23, 2, 1, "", "to_forcefield_dict"]], "mcp_server_webcrawl.models.resources.ResourceResultType": [[23, 5, 1, "", "AUDIO"], [23, 5, 1, "", "CSS"], [23, 5, 1, "", "DOC"], [23, 5, 1, "", "FEED"], [23, 5, 1, "", "FONT"], [23, 5, 1, "", "FRAME"], [23, 5, 1, "", "IMAGE"], [23, 5, 1, "", "OTHER"], [23, 5, 1, "", "PAGE"], [23, 5, 1, "", "PDF"], [23, 5, 1, "", "SCRIPT"], [23, 5, 1, "", "TEXT"], [23, 5, 1, "", "UNDEFINED"], [23, 5, 1, "", "VIDEO"], [23, 2, 1, "", "to_int_map"], [23, 2, 1, "", "values"]], "mcp_server_webcrawl.models.sites": [[23, 1, 1, "", "SiteResult"]], "mcp_server_webcrawl.models.sites.SiteResult": [[23, 2, 1, "", "__init__"], [23, 2, 1, "", "to_dict"], [23, 2, 1, "", "to_forcefield_dict"]], "mcp_server_webcrawl.templates": [[24, 0, 0, "-", "tests"]], "mcp_server_webcrawl.templates.tests": [[24, 1, 1, "", "TemplateTests"]], "mcp_server_webcrawl.templates.tests.TemplateTests": [[24, 2, 1, "", "setUp"], [24, 2, 1, "", "test_core_html"]], "mcp_server_webcrawl.utils": [[25, 0, 0, "-", "cli"], [25, 3, 1, "", "from_isoformat_zulu"], [25, 0, 0, "-", "logger"], [25, 0, 0, "-", "server"], [25, 3, 1, "", "to_isoformat_zulu"], [25, 0, 0, "-", "tools"]], "mcp_server_webcrawl.utils.cli": [[25, 3, 1, "", "get_help_long_message"], [25, 3, 1, "", "get_help_short_message"]], "mcp_server_webcrawl.utils.logger": [[25, 3, 1, "", "get_logger"], [25, 3, 1, "", "get_logger_configuration"], [25, 3, 1, "", "initialize_logger"]], "mcp_server_webcrawl.utils.server": [[25, 3, 1, "", "initialize_mcp_server"]], "mcp_server_webcrawl.utils.tools": [[25, 3, 1, "", "get_crawler_tools"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:method", "3": "py:function", "4": "py:property", "5": "py:attribute"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "method", "Python method"], "3": ["py", "function", "Python function"], "4": ["py", "property", "Python property"], "5": ["py", "attribute", "Python attribute"]}, "titleterms": {"mcp": [1, 2, 3, 4, 5, 6, 7, 8, 9], "server": [1, 2, 3, 4, 5, 6, 7, 8, 25], "webcrawl": [1, 2, 3, 4, 5, 6, 7, 8], "content": [8, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 28], "indic": 8, "tabl": 8, "instal": [1, 2, 3, 4, 5, 6, 7, 9], "requir": [1, 2, 3, 4, 5, 6, 7, 9], "configur": [1, 2, 3, 4, 5, 6, 7, 9], "wget": [6, 7, 20], "warc": [6, 19], "interrobot": [3, 16], "katana": [4, 17], "siteon": [5, 18], "multipl": 9, "mcp_server_webcrawl": [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26], "packag": [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25], "subpackag": [11, 12], "submodul": [11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25], "main": 11, "modul": [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25], "set": [1, 11], "settings_loc": 11, "crawler": [8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 28], "base": 14, "adapt": [13, 14, 15, 16, 17, 18, 19, 20], "api": 14, "index": 14, "test": [13, 14, 15, 16, 17, 18, 19, 20, 24], "model": 23, "resourc": 23, "site": 23, "util": 25, "blob": [], "cli": 25, "logger": 25, "querycach": [], "tool": [25, 28], "usag": [10, 28], "avail": [0, 28], "webcrawl_sit": 28, "webcrawl_search": 28, "setup": [0, 1, 2, 3, 4, 5, 6, 7], "guid": [0, 1, 2, 3, 4, 5, 6, 7], "A": [], "quick": [], "step": [1, 2, 3, 4, 5, 6, 7], "1": [1, 2, 3, 4, 5, 6, 7], "web": [], "crawl": [3, 5, 7], "2": [1, 2, 3, 4, 5, 6, 7], "claud": [1, 2, 3, 4, 5, 6, 7], "desktop": [1, 2, 3, 4, 5, 6, 7], "3": [1, 2, 3, 4, 5, 6, 7], "websit": [2, 7], "4": [1, 2, 3, 4, 5, 6, 7], "verifi": [1, 2, 3, 4, 5, 6, 7], "us": [1, 2, 3, 4, 5, 6, 7], "troubleshoot": [1, 2, 3, 4, 5, 6, 7], "what": [1, 2, 3, 4, 5, 6], "ar": 6, "file": 6, "creat": [2, 3, 5, 6], "i": [1, 2, 3, 4, 5], "run": 4, "featur": 28, "support": [8, 28], "field": 28, "boolean": 28, "search": [22, 28], "syntax": 28, "definit": 28, "type": 28, "prompt": 27, "routin": 27, "extra": [21, 28], "markdown": 21, "regex": 21, "snippet": 21, "thumbnail": 21, "xpath": 21, "archivebox": [1, 13], "httrack": [2, 15], "templat": 24, "up": 1, "advanc": [], "mirror": 2, "interact": [10, 22], "highlight": 22, "session": 22, "ui": 22, "overview": [], "kei": [], "compon": [], "interfac": [], "system": [], "manag": [], "user": [], "get": [], "start": [], "exampl": [], "basic": [], "scenario": [], "mode": 10, "refer": 9, "screencap": 10}, "envversion": {"sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx.ext.intersphinx": 1, "sphinx": 60}, "alltitles": {"mcp_server_webcrawl": [[26, "mcp-server-webcrawl"]], "Prompt Routines": [[27, "prompt-routines"]], "Usage": [[28, "usage"], [10, "usage"]], "Available Tools": [[28, "available-tools"]], "webcrawl_sites": [[28, "webcrawl-sites"]], "webcrawl_search": [[28, "webcrawl-search"]], "Crawler Features Support": [[28, "crawler-features-support"]], "Crawler Field Support": [[28, "crawler-field-support"]], "Boolean Search Syntax": [[28, "boolean-search-syntax"]], "Field Search Definitions": [[28, "field-search-definitions"]], "Field Content": [[28, "field-content"]], "Content Types": [[28, "content-types"]], "Extras": [[28, "extras"]], "Module contents": [[24, "module-mcp_server_webcrawl.templates"], [12, "module-mcp_server_webcrawl.crawlers"], [21, "module-mcp_server_webcrawl.extras"], [23, "module-mcp_server_webcrawl.models"], [25, "module-mcp_server_webcrawl.utils"], [11, "module-mcp_server_webcrawl"], [13, "module-mcp_server_webcrawl.crawlers.archivebox"], [14, "module-mcp_server_webcrawl.crawlers.base"], [15, "module-mcp_server_webcrawl.crawlers.httrack"], [16, "module-mcp_server_webcrawl.crawlers.interrobot"], [17, "module-mcp_server_webcrawl.crawlers.katana"], [18, "module-mcp_server_webcrawl.crawlers.siteone"], [19, "module-mcp_server_webcrawl.crawlers.warc"], [20, "module-mcp_server_webcrawl.crawlers.wget"], [22, "module-mcp_server_webcrawl.interactive"]], "Submodules": [[24, "submodules"], [21, "submodules"], [23, "submodules"], [25, "submodules"], [11, "submodules"], [13, "submodules"], [14, "submodules"], [15, "submodules"], [16, "submodules"], [17, "submodules"], [18, "submodules"], [19, "submodules"], [20, "submodules"], [22, "submodules"]], "mcp_server_webcrawl.templates package": [[24, "mcp-server-webcrawl-templates-package"]], "mcp_server_webcrawl.templates.tests module": [[24, "module-mcp_server_webcrawl.templates.tests"]], "Setup Guides": [[0, "setup-guides"]], "Available Guides:": [[0, null]], "Requirements": [[1, "requirements"], [2, "requirements"], [3, "requirements"], [4, "requirements"], [5, "requirements"], [6, "requirements"], [7, "requirements"], [9, "requirements"]], "ArchiveBox MCP Setup Guide": [[1, "archivebox-mcp-setup-guide"]], "What is ArchiveBox?": [[1, "what-is-archivebox"]], "Installation Steps": [[1, "installation-steps"], [2, "installation-steps"], [3, "installation-steps"], [4, "installation-steps"], [5, "installation-steps"], [6, "installation-steps"], [7, "installation-steps"]], "1. Install mcp-server-webcrawl": [[1, "install-mcp-server-webcrawl"], [2, "install-mcp-server-webcrawl"], [3, "install-mcp-server-webcrawl"], [4, "install-mcp-server-webcrawl"], [5, "install-mcp-server-webcrawl"], [6, "install-mcp-server-webcrawl"], [7, "install-mcp-server-webcrawl"]], "2. Install and Set Up ArchiveBox": [[1, "install-and-set-up-archivebox"]], "3. Configure Claude Desktop": [[1, "configure-claude-desktop"], [2, "configure-claude-desktop"], [3, "configure-claude-desktop"], [4, "configure-claude-desktop"], [5, "configure-claude-desktop"]], "4. Verify and Use": [[1, "verify-and-use"], [2, "verify-and-use"], [3, "verify-and-use"], [4, "verify-and-use"], [5, "verify-and-use"], [6, "verify-and-use"], [7, "verify-and-use"]], "Troubleshooting": [[1, "troubleshooting"], [2, "troubleshooting"], [3, "troubleshooting"], [4, "troubleshooting"], [5, "troubleshooting"], [6, "troubleshooting"], [7, "troubleshooting"]], "HTTrack MCP Setup Guide": [[2, "httrack-mcp-setup-guide"]], "What is HTTrack?": [[2, "what-is-httrack"]], "2. Create Website Mirrors with HTTrack": [[2, "create-website-mirrors-with-httrack"]], "InterroBot MCP Setup Guide": [[3, "interrobot-mcp-setup-guide"]], "What is InterroBot?": [[3, "what-is-interrobot"]], "2. Create Crawls with InterroBot": [[3, "create-crawls-with-interrobot"]], "Katana MCP Setup Guide": [[4, "katana-mcp-setup-guide"]], "What is Katana?": [[4, "what-is-katana"]], "2. Install and Run Katana": [[4, "install-and-run-katana"]], "SiteOne MCP Setup Guide": [[5, "siteone-mcp-setup-guide"]], "What is SiteOne?": [[5, "what-is-siteone"]], "2. Create Crawls with SiteOne": [[5, "create-crawls-with-siteone"]], "WARC MCP Setup Guide": [[6, "warc-mcp-setup-guide"]], "What are WARC Files?": [[6, "what-are-warc-files"]], "2. Configure Claude Desktop": [[6, "configure-claude-desktop"], [7, "configure-claude-desktop"]], "3. Create WARC Files with Wget": [[6, "create-warc-files-with-wget"]], "wget MCP Setup Guide": [[7, "wget-mcp-setup-guide"]], "3. Crawl Websites with wget": [[7, "crawl-websites-with-wget"]], "mcp_server_webcrawl.crawlers package": [[12, "mcp-server-webcrawl-crawlers-package"]], "Subpackages": [[12, "subpackages"], [11, "subpackages"]], "mcp_server_webcrawl.extras package": [[21, "mcp-server-webcrawl-extras-package"]], "mcp_server_webcrawl.extras.markdown module": [[21, "module-mcp_server_webcrawl.extras.markdown"]], "mcp_server_webcrawl.extras.regex module": [[21, "module-mcp_server_webcrawl.extras.regex"]], "mcp_server_webcrawl.extras.snippets module": [[21, "module-mcp_server_webcrawl.extras.snippets"]], "mcp_server_webcrawl.extras.thumbnails module": [[21, "module-mcp_server_webcrawl.extras.thumbnails"]], "mcp_server_webcrawl.extras.xpath module": [[21, "module-mcp_server_webcrawl.extras.xpath"]], "mcp_server_webcrawl.models package": [[23, "mcp-server-webcrawl-models-package"]], "mcp_server_webcrawl.models.resources module": [[23, "module-mcp_server_webcrawl.models.resources"]], "mcp_server_webcrawl.models.sites module": [[23, "module-mcp_server_webcrawl.models.sites"]], "mcp_server_webcrawl.utils package": [[25, "mcp-server-webcrawl-utils-package"]], "mcp_server_webcrawl.utils.cli module": [[25, "module-mcp_server_webcrawl.utils.cli"]], "mcp_server_webcrawl.utils.logger module": [[25, "module-mcp_server_webcrawl.utils.logger"]], "mcp_server_webcrawl.utils.server module": [[25, "module-mcp_server_webcrawl.utils.server"]], "mcp_server_webcrawl.utils.tools module": [[25, "module-mcp_server_webcrawl.utils.tools"]], "mcp-server-webcrawl": [[8, "mcp-server-webcrawl"]], "Supported Crawlers": [[8, "id7"]], "Contents:": [[8, null]], "Indices and tables": [[8, "indices-and-tables"]], "Installation": [[9, "installation"]], "MCP Configuration": [[9, "mcp-configuration"]], "Multiple Configurations": [[9, "multiple-configurations"]], "References": [[9, "references"]], "mcp_server_webcrawl package": [[11, "mcp-server-webcrawl-package"]], "mcp_server_webcrawl.main module": [[11, "mcp-server-webcrawl-main-module"]], "mcp_server_webcrawl.settings module": [[11, "module-mcp_server_webcrawl.settings"]], "mcp_server_webcrawl.settings_local module": [[11, "module-mcp_server_webcrawl.settings_local"]], "mcp_server_webcrawl.crawlers.archivebox package": [[13, "mcp-server-webcrawl-crawlers-archivebox-package"]], "mcp_server_webcrawl.crawlers.archivebox.adapter module": [[13, "module-mcp_server_webcrawl.crawlers.archivebox.adapter"]], "mcp_server_webcrawl.crawlers.archivebox.crawler module": [[13, "module-mcp_server_webcrawl.crawlers.archivebox.crawler"]], "mcp_server_webcrawl.crawlers.archivebox.tests module": [[13, "module-mcp_server_webcrawl.crawlers.archivebox.tests"]], "mcp_server_webcrawl.crawlers.base package": [[14, "mcp-server-webcrawl-crawlers-base-package"]], "mcp_server_webcrawl.crawlers.base.adapter module": [[14, "module-mcp_server_webcrawl.crawlers.base.adapter"]], "mcp_server_webcrawl.crawlers.base.api module": [[14, "module-mcp_server_webcrawl.crawlers.base.api"]], "mcp_server_webcrawl.crawlers.base.crawler module": [[14, "module-mcp_server_webcrawl.crawlers.base.crawler"]], "mcp_server_webcrawl.crawlers.base.indexed module": [[14, "module-mcp_server_webcrawl.crawlers.base.indexed"]], "mcp_server_webcrawl.crawlers.base.tests module": [[14, "module-mcp_server_webcrawl.crawlers.base.tests"]], "mcp_server_webcrawl.crawlers.httrack package": [[15, "mcp-server-webcrawl-crawlers-httrack-package"]], "mcp_server_webcrawl.crawlers.httrack.adapter module": [[15, "module-mcp_server_webcrawl.crawlers.httrack.adapter"]], "mcp_server_webcrawl.crawlers.httrack.crawler module": [[15, "module-mcp_server_webcrawl.crawlers.httrack.crawler"]], "mcp_server_webcrawl.crawlers.httrack.tests module": [[15, "module-mcp_server_webcrawl.crawlers.httrack.tests"]], "mcp_server_webcrawl.crawlers.interrobot package": [[16, "mcp-server-webcrawl-crawlers-interrobot-package"]], "mcp_server_webcrawl.crawlers.interrobot.adapter module": [[16, "module-mcp_server_webcrawl.crawlers.interrobot.adapter"]], "mcp_server_webcrawl.crawlers.interrobot.crawler module": [[16, "module-mcp_server_webcrawl.crawlers.interrobot.crawler"]], "mcp_server_webcrawl.crawlers.interrobot.tests module": [[16, "module-mcp_server_webcrawl.crawlers.interrobot.tests"]], "mcp_server_webcrawl.crawlers.katana package": [[17, "mcp-server-webcrawl-crawlers-katana-package"]], "mcp_server_webcrawl.crawlers.katana.adapter module": [[17, "module-mcp_server_webcrawl.crawlers.katana.adapter"]], "mcp_server_webcrawl.crawlers.katana.crawler module": [[17, "module-mcp_server_webcrawl.crawlers.katana.crawler"]], "mcp_server_webcrawl.crawlers.katana.tests module": [[17, "module-mcp_server_webcrawl.crawlers.katana.tests"]], "mcp_server_webcrawl.crawlers.siteone package": [[18, "mcp-server-webcrawl-crawlers-siteone-package"]], "mcp_server_webcrawl.crawlers.siteone.adapter module": [[18, "module-mcp_server_webcrawl.crawlers.siteone.adapter"]], "mcp_server_webcrawl.crawlers.siteone.crawler module": [[18, "module-mcp_server_webcrawl.crawlers.siteone.crawler"]], "mcp_server_webcrawl.crawlers.siteone.tests module": [[18, "module-mcp_server_webcrawl.crawlers.siteone.tests"]], "mcp_server_webcrawl.crawlers.warc package": [[19, "mcp-server-webcrawl-crawlers-warc-package"]], "mcp_server_webcrawl.crawlers.warc.adapter module": [[19, "module-mcp_server_webcrawl.crawlers.warc.adapter"]], "mcp_server_webcrawl.crawlers.warc.crawler module": [[19, "module-mcp_server_webcrawl.crawlers.warc.crawler"]], "mcp_server_webcrawl.crawlers.warc.tests module": [[19, "module-mcp_server_webcrawl.crawlers.warc.tests"]], "mcp_server_webcrawl.crawlers.wget package": [[20, "mcp-server-webcrawl-crawlers-wget-package"]], "mcp_server_webcrawl.crawlers.wget.adapter module": [[20, "module-mcp_server_webcrawl.crawlers.wget.adapter"]], "mcp_server_webcrawl.crawlers.wget.crawler module": [[20, "module-mcp_server_webcrawl.crawlers.wget.crawler"]], "mcp_server_webcrawl.crawlers.wget.tests module": [[20, "module-mcp_server_webcrawl.crawlers.wget.tests"]], "mcp_server_webcrawl.interactive package": [[22, "mcp-server-webcrawl-interactive-package"]], "mcp_server_webcrawl.interactive.highlights module": [[22, "module-mcp_server_webcrawl.interactive.highlights"]], "mcp_server_webcrawl.interactive.search module": [[22, "module-mcp_server_webcrawl.interactive.search"]], "mcp_server_webcrawl.interactive.session module": [[22, "module-mcp_server_webcrawl.interactive.session"]], "mcp_server_webcrawl.interactive.ui module": [[22, "module-mcp_server_webcrawl.interactive.ui"]], "Interactive Mode": [[10, "interactive-mode"]], "Screencaps": [[10, "screencaps"]]}, "indexentries": {}})