website_scraper_mcp/
├── app.py                   ← Entry point (stdio / SSE transport)
├── server.py                ← MCP server + tool dispatcher
├── config.py                ← Pydantic Settings (env vars)
├── models.py                ← Input/Output Pydantic models
└── tools/
    ├── scrape.py            ← Tool 1 – static/dynamic detection + scraping
    ├── crawl.py             ← Tool 2 – BFS crawler, robots.txt aware
    ├── clean.py             ← Tool 3 – Trafilatura + BS4 content cleaning
    ├── chunk.py             ← Tool 4 – sliding window chunking
    └── azure_ai_search.py   ← Tools 5 & 7 – index + search

# 1. Clone the repo
git clone https://github.com/your-org/website-scraper-mcp.git
cd website-scraper-mcp

# 2. Create and activate a virtual environment
python -m venv .venv
# Windows
.venv\Scripts\activate
# Linux/Mac
source .venv/bin/activate

# 3. Install dependencies
pip install -r requirements.txt

# 4. Install Playwright browsers (Chromium)
playwright install chromium

# 5. Copy and fill environment variables
cp .env.example .env
# Edit .env with your Azure credentials

python -m website_scraper_mcp.app
# or
python -m website_scraper_mcp.app --transport stdio

python -m website_scraper_mcp.app --transport sse --port 8000
# Server available at http://localhost:8000/sse

# Build and start in SSE mode
docker compose up --build

# Stop
docker compose down

python examples/mcp_client_example.py

{
  "mcpServers": {
    "website-scraper": {
      "command": "python",
      "args": ["-m", "website_scraper_mcp.app", "--transport", "stdio"],
      "cwd": "/path/to/website-scraper-mcp",
      "env": {
        "AZURE_SEARCH_ENDPOINT": "https://your-service.search.windows.net",
        "AZURE_SEARCH_KEY": "your-key",
        "AZURE_SEARCH_INDEX_NAME": "website-content"
      }
    }
  }
}

import asyncio
from mcp import ClientSession
from mcp.client.sse import sse_client

async def demo():
    async with sse_client("http://localhost:8000/sse") as (read, write):
        async with ClientSession(read, write) as session:
            await session.initialize()

            # Scrape a single page
            result = await session.call_tool("scrape_website", {"url": "https://example.com"})
            print(result)

            # Full pipeline
            result = await session.call_tool("index_website", {
                "url": "https://example.com",
                "max_depth": 2
            })
            print(result)

            # Search
            result = await session.call_tool("search_index", {
                "query": "What services does the company provide?",
                "top": 5
            })
            print(result)

asyncio.run(demo())

// Input
{"url": "https://example.com"}

// Output
{
  "title": "Example Domain",
  "url": "https://example.com",
  "content": "This domain is for use in illustrative examples...",
  "links": ["https://www.iana.org/domains/example"],
  "is_dynamic": false,
  "metadata": {"description": "..."}
}

// Input
{"url": "https://example.com", "max_depth": 2}

// Output
{
  "url": "https://example.com",
  "pages_crawled": 4,
  "total_chunks": 38,
  "indexed_documents": 38,
  "failed_documents": 0,
  "status": "success",
  "errors": []
}

// Input
{"query": "What services does the company provide?", "top": 5}

// Output
{
  "query": "What services does the company provide?",
  "total_results": 3,
  "hits": [
    {
      "id": "abc123",
      "url": "https://example.com/services",
      "title": "Our Services",
      "content": "We provide cloud, AI, and data services...",
      "chunk_number": 0,
      "score": 9.8
    }
  ]
}

{
  "error": "HTTP 404 when fetching https://example.com/missing",
  "tool": "scrape_website"
}