mcp-web-scraper/
├── server.py              # Main server entry point
├── tools/
│   ├── __init__.py       # Tools package initialization
│   ├── search.py         # Search tools (web_search, news_search, etc.)
│   └── scraping.py       # Scraping tools (scrape_html, extract_article, etc.)
├── utils/
│   ├── __init__.py       # Utils package initialization
│   └── helpers.py        # Helper functions (clean_text, validate_url)
├── requirements.txt       # Python dependencies
├── render.yaml           # Render deployment configuration
├── .gitignore            # Git ignore rules
├── README.md             # This file
└── config.example.json   # Claude Desktop config example

mkdir mcp-web-scraper
cd mcp-web-scraper

# Create directory structure
mkdir -p tools utils

# Create all files (copy from artifacts above):
# - server.py
# - tools/__init__.py
# - tools/search.py
# - tools/scraping.py
# - utils/__init__.py
# - utils/helpers.py
# - requirements.txt
# - render.yaml
# - .gitignore
# - README.md

git init
git add .
git commit -m "Initial commit: Modular MCP Web Scraper"
git remote add origin https://github.com/YOUR_USERNAME/mcp-web-scraper.git
git push -u origin main

{
  "mcpServers": {
    "web-scraper": {
      "type": "streamable-http",
      "url": "https://your-app.onrender.com/mcp"
    }
  }
}

# Clone and setup
git clone https://github.com/YOUR_USERNAME/mcp-web-scraper.git
cd mcp-web-scraper

# Create virtual environment
python3 -m venv venv
source venv/bin/activate  # Windows: venv\Scripts\activate

# Install dependencies
pip install -r requirements.txt

# Run server
python server.py

# List tools
curl -X POST http://localhost:8000/mcp \
  -H "Content-Type: application/json" \
  -d '{"jsonrpc":"2.0","id":1,"method":"tools/list"}'

# Test web search
curl -X POST http://localhost:8000/mcp \
  -H "Content-Type: application/json" \
  -d '{
    "jsonrpc":"2.0",
    "id":2,
    "method":"tools/call",
    "params":{
      "name":"web_search",
      "arguments":{"query":"AI news","max_results":3}
    }
  }'

@mcp.tool()
def my_custom_search(query: str) -> dict:
    """Your custom search tool"""
    # Implementation here
    return {"success": True, "data": []}

@mcp.tool()
def my_custom_scraper(url: str) -> dict:
    """Your custom scraper"""
    # Implementation here
    return {"success": True, "content": ""}

git add .
git commit -m "Add new tools"
git push origin main
# Render auto-deploys!

curl https://your-app.onrender.com/health

# Ensure you're in project root
cd mcp-web-scraper

# Check Python path
export PYTHONPATH="${PYTHONPATH}:$(pwd)"

# Run server
python server.py