PyScrappy

pip install pyscrappy

# Browser support (for JS-rendered pages)
pip install 'pyscrappy[browser]'
playwright install chromium

# DataFrame support
pip install 'pyscrappy[dataframe]'

# MCP server (use PyScrappy's scrapers as AI-agent tools)
pip install 'pyscrappy[mcp]'

# Everything
pip install 'pyscrappy[all]'

AI agent  ──MCP tool call──▶  PyScrappy  ──fetch + extract──▶  Any website
   ▲                                                                │
   └──────────────  clean Markdown / JSON  ◀───────────────────────┘

pip install 'pyscrappy[mcp]'
claude mcp add pyscrappy pyscrappy-mcp

pip install 'pyscrappy[mcp]'                 # needs Python 3.10+
pyscrappy chat --model qwen2.5 "what's the current AAPL quote?"

pip install 'pyscrappy[mcp]'

claude mcp add pyscrappy pyscrappy-mcp

{
  "mcpServers": {
    "pyscrappy": {
      "command": "pyscrappy-mcp"
    }
  }
}

{
  "mcpServers": {
    "pyscrappy": {
      "command": "pyscrappy-mcp",
      "env": { "OMDB_API_KEY": "your-key" }
    }
  }
}

from pyscrappy import BaseScraper, register_scraper, get_scraper
from pyscrappy.core.models import ScrapeResult, ScrapeMetadata

@register_scraper("reddit")
class RedditScraper(BaseScraper):
    def scrape(self, subreddit: str, **kwargs) -> ScrapeResult:
        data = self.fetch_and_parse(f"https://old.reddit.com/r/{subreddit}/.json")
        # ... build a list of dicts ...
        return ScrapeResult(data=[...], metadata=ScrapeMetadata(scraper="reddit"))

get_scraper("reddit")().scrape(subreddit="python")

[project.entry-points."pyscrappy.scrapers"]
reddit = "pyscrappy_reddit:RedditScraper"

from pyscrappy import scrape

result = scrape("https://en.wikipedia.org/wiki/Web_scraping")

print(result.to_markdown())   # feed straight to an LLM
# ...or result.to_json() / result.to_dataframe()

print(result.data[0]["metadata"]["title"])
print(result.data[0]["text"]["word_count"])

from pyscrappy import GenericScraper

with GenericScraper() as gs:
    result = gs.scrape(
        url="https://news.ycombinator.com",
        selectors={"title": ".titleline a", "score": ".score"},
    )
    for item in result.data:
        print(item["title"], item.get("score", ""))

from pyscrappy import WikipediaScraper

with WikipediaScraper() as ws:
    result = ws.scrape(query="Python (programming language)", mode="summary")
    print(result.data[0]["text"])

from pyscrappy import StockScraper

with StockScraper() as ss:
    result = ss.scrape(symbol="AAPL", mode="history", period="1mo")
    df = result.to_dataframe()
    print(df.head())

from pyscrappy import IMDBScraper

with IMDBScraper() as scraper:      # reads OMDB_API_KEY from the environment
    # Search by title
    result = scraper.scrape(query="inception")
    # ...or look up a specific IMDB id
    result = scraper.scrape(query="tt1375666")
    df = result.to_dataframe()
    print(df[["title", "year", "rating", "genre"]])

from pyscrappy import NewsScraper

with NewsScraper() as ns:
    result = ns.scrape(feed_url="https://rss.nytimes.com/services/xml/rss/nyt/World.xml")
    for article in result.data[:5]:
        print(article["title"])

from pyscrappy import ImageSearchScraper

with ImageSearchScraper() as iss:
    result = iss.scrape(query="golden retriever", max_images=10, download_to="./dogs")

from pyscrappy import YouTubeScraper

with YouTubeScraper() as scraper:
    result = scraper.scrape(query="python tutorial", max_results=10)
    for video in result.data:
        print(video["title"], video.get("views", ""))

from pyscrappy import SoundCloudScraper

with SoundCloudScraper() as scraper:
    result = scraper.scrape(query="lo-fi beats", max_results=10)

from pyscrappy import AmazonScraper, NeweggScraper, IKEAScraper

# Amazon — general marketplace
with AmazonScraper() as scraper:
    result = scraper.scrape(query="laptop", max_pages=2)

# Newegg — electronics / computer hardware
with NeweggScraper() as scraper:
    result = scraper.scrape(query="graphics card", max_pages=2)

# IKEA — furniture / home (uses IKEA's JSON search API).
# Prices and products are per-country: pass the store's country/lang.
with IKEAScraper(country="gb", lang="en") as scraper:   # or "us"/"en", "de"/"de", …
    result = scraper.scrape(query="desk", max_results=24)
    df = result.to_dataframe()

from pyscrappy import ZomatoScraper, UberEatsScraper

with ZomatoScraper() as scraper:
    result = scraper.scrape(city="bangalore", max_results=20)

# Uber Eats: restaurants by city (any country Uber Eats operates in),
# then the full menu of any restaurant. Pass the country's locale.
with UberEatsScraper(locale="gb") as scraper:
    result = scraper.scrape(city="London", max_results=30)
    store = result.data[0]
    menu = scraper.get_menu(store["url"])   # items with prices

from pyscrappy import ScraperConfig, GenericScraper

config = ScraperConfig(
    timeout=20.0,            # request timeout in seconds
    max_retries=3,           # retry failed requests
    rate_limit=2.0,          # seconds between requests per domain
    proxy="http://...",      # proxy URL, or a list to rotate through
    scraper_api=None,        # route via a scraping-API service (see below)
    headless=True,           # browser runs headless
    render_js="auto",        # auto-detect if JS rendering is needed
    cache_ttl=0,             # response cache TTL in seconds (0 = disabled)
)

with GenericScraper(config) as gs:
    result = gs.scrape(url="https://example.com")

from pyscrappy import ScraperConfig, AmazonScraper

# Single proxy
config = ScraperConfig(proxy="http://user:pass@host:port")

# Rotating list (one picked per request)
config = ScraperConfig(proxy=["http://p1:8080", "http://p2:8080"])

config = ScraperConfig(scraper_api={
    "provider": "scraperapi",   # or "scrapeops", "scrapingbee"
    "api_key": "YOUR_KEY",
    "render_js": True,           # optional
})

# Now any scraper works through the service, unchanged:
with AmazonScraper(config) as scraper:
    result = scraper.scrape(query="laptop")

from pyscrappy import scrape_many, scrape_all, AmazonScraper, WikipediaScraper, NewsScraper

# One scraper, many queries, concurrently:
results = scrape_many(AmazonScraper, [{"query": "laptop"}, {"query": "phone"}])

# Different scrapers at once:
results = scrape_all([
    lambda: WikipediaScraper().scrape(query="Python"),
    lambda: NewsScraper().scrape(feed_url="https://rss.nytimes.com/services/xml/rss/nyt/World.xml"),
])

from pyscrappy import WikipediaScraper
from pyscrappy import ScraperConfig

config = ScraperConfig(cache_ttl=300)   # cache for 5 minutes

with WikipediaScraper(config) as ws:
    ws.scrape(query="Python")   # fetched over the network
    ws.scrape(query="Python")   # served from cache