Scrapy MCP — Universal Web Crawling & Scraping

User (Claude) → MCP Tools → Engine → Scheduler → Downloader → Spider → Pipeline → Result
                                ↓            ↓           ↓
                             Settings    Middleware    Extractors

pip install -r requirements.txt

{
  "mcpServers": {
    "scrapy-mcp": {
      "command": "python",
      "args": ["d:/project/scrapy-mcp/server.py"]
    }
  }
}

crawl({
  "urls": ["https://books.toscrape.com"],
  "item_selector": "article.product_pod",
  "fields": {
    "title": {"selector": "h3 a", "attr": "title"},
    "price": {"selector": ".price_color", "regex": "([\\d.]+)"}
  },
  "pagination": {"type": "next_link", "selector": ".next a", "max_pages": 3}
})

name: my-spider
description: "What this spider does"
start_urls:
  - "https://example.com/items"
item_selector: ".item"
fields:
  title:
    selector: ".title"
    type: text
  link:
    selector: "a"
    type: attr
    attr: href
    transform: absolute_url
pagination:
  type: next_link
  selector: ".next a"
  max_pages: 5

cd d:/project/scrapy-mcp

# Test extraction
python -c "
from mcp_scrapy.extractors import FieldExtractor
import httpx, asyncio

async def test():
    async with httpx.AsyncClient() as c:
        r = await c.get('https://httpbin.org/html')
    ext = FieldExtractor(r.text)
    print('Title:', ext.extract_field({'selector': 'h1'}))
    print('Links:', len(ext.extract_all_links()))

asyncio.run(test())
"

# Test spider loading
python -c "
from mcp_scrapy.spider import SpiderLoader
loader = SpiderLoader(['spiders'])
for s in loader.list_all():
    print(f'{s[\"name\"]}: {s[\"description\"]}')
"

scrapy-mcp/
├── server.py                  # MCP entry point
├── requirements.txt
├── mcp_scrapy/
│   ├── __init__.py
│   ├── engine.py              # CrawlEngine coordinator
│   ├── scheduler.py           # Request queue + dedup
│   ├── downloader.py          # Async HTTP + middleware
│   ├── spider.py              # Spider + YAML loader
│   ├── request.py             # Request/Response/Item types
│   ├── pipelines.py           # Item post-processing
│   ├── middleware.py          # Downloader middleware
│   ├── extractors.py          # CSS/XPath/regex field extraction
│   ├── exporters.py           # JSON/Markdown/CSV output
│   ├── schema_extractor.py    # Schema.org / Open Graph
│   ├── settings.py            # Configuration
│   └── tools/
│       ├── __init__.py
│       ├── crawl.py           # crawl tool
│       ├── scrape_page.py     # scrape_page tool
│       ├── search.py          # search_and_crawl tool
│       ├── extract_schema.py  # extract_schema tool
│       └── list_spiders.py    # list_spiders tool
├── spiders/                   # Spider YAML configs
│   ├── example.yml
│   ├── hackernews.yml
│   └── books_toscrape.yml
└── tests/
    ├── test_extractors.py
    ├── test_pipelines.py
    └── test_scheduler.py