cn-scraper-mcp

git clone https://github.com/goesByhc/cn-scraper-mcp.git
cd cn-scraper-mcp
pip install -e ".[dev]"

mkdir -p ~/.cn-scraper-cookies

cn-scraper-mcp
# 或: python -m cn_scraper_mcp.server

docker build -t cn-scraper-mcp .
docker run -i --rm \
  -v ~/.cn-scraper-cookies:/root/.cn-scraper-cookies \
  -v ~/.jd_login_profile:/root/.jd_login_profile \
  cn-scraper-mcp

# Codex ~/.codex/config.toml
[mcp_servers.cn-scraper]
type = "stdio"
command = "docker"
args = ["run", "-i", "--rm",
  "-v", "~/.cn-scraper-cookies:/root/.cn-scraper-cookies",
  "-v", "~/.jd_login_profile:/root/.jd_login_profile",
  "cn-scraper-mcp"]

from cn_scraper_mcp.engines import (
    TaobaoEngine, JDEngine, PDDEngine,
    XiaohongshuEngine, ZhihuEngine, ZsxqEngine, WeiboEngine, DouyinEngine,
)

# 淘宝 —— 纯脚本，免浏览器
tb = TaobaoEngine()
r = tb.search("华为mate70", limit=5)
print(r["items"][0]["price"])  # "3099.00"

# 京东 —— 需要 Chrome headful
jd = JDEngine()
r = jd.search("京东京造沐光")

# 拼多多 —— ⚠️ 仅一次搜索
pdd = PDDEngine()
r = pdd.search("儿童学习桌", limit=5)
detail = pdd.product_detail("123456789")  # 不限次数

# 小红书 —— Obscura 优先，Chrome 兜底
xhs = XiaohongshuEngine()
notes = xhs.search("测评")
detail = xhs.get_note(notes["items"][0]["noteId"])

# 知乎 —— 需要登录 Cookie
zh = ZhihuEngine()
r = zh.search("半导体")
hot = zh.hot_list()

# 知识星球 —— REST API
zs = ZsxqEngine()
topics = zs.get_topics("28888555451", count=5)

# 微博 —— 热搜免登，搜索需 Cookie
wb = WeiboEngine()
hot = wb.hot_list()
r = wb.search("热搜话题")
timeline = wb.user_timeline("2803301701")  # 人民日报 UID

# 抖音 —— CDP 浏览器，需登录
dy = DouyinEngine()
dy.ensure_chrome()
r = dy.search("华为")
hot = dy.hot_list()

# 跨平台比价
from cn_scraper_mcp.compare import compare_prices
result = compare_prices("华为mate70", platforms=["taobao", "jd"])
print(result["best_deal"])  # 最低价商品

[mcp_servers.cn-scraper]
type = "stdio"
command = "cn-scraper-mcp"
args = []
autoApprove = ["*"]

{
  "mcp": {
    "servers": {
      "cn-scraper": {
        "command": "cn-scraper-mcp",
        "args": []
      }
    }
  }
}

[[plugins]]
name = "cn-scraper"
command = "cn-scraper-mcp"
args = []

AI Agent (Codex / Claude / Cursor / Trae / Reasonix)
    │ MCP stdio
    ▼
┌─────────────────────────────────────────────────────┐
│                    server.py (FastMCP)               │
│  19 个工具 · 参数校验 · 统一错误模型 · stderr 日志   │
├─────────────────────────────────────────────────────┤
│  引擎层                                              │
│  taobao.py ──→ curl_cffi + MTOP ──→ h5api.m.taobao   │
│  jd.py     ──→ Chrome CDP        ──→ search.jd.com   │
│  pdd.py    ──→ Chrome CDP + iUA  ──→ mobile.yangkeduo │
│  xiaohongshu.py ─→ Chrome CDP    ──→ xiaohongshu.com  │
│  zhihu.py  ──→ REST API v4       ──→ zhihu.com       │
│  weibo.py  ──→ REST API          ──→ weibo.com       │
│  zsxq.py   ──→ REST API v2       ──→ api.zsxq.com    │
│  douyin.py ──→ Chrome CDP + 验证码轮询 ─→ douyin.com │
├─────────────────────────────────────────────────────┤
│  基础设施                                            │
│  auth.py     — Cookie 管理与字段校验                 │
│  http.py     — 超时/重试/退避/限速                   │
│  cdp.py      — Chrome CDP 驱动 + BrowserLock + 进程管理 │
│  logging.py  — stderr 脱敏日志 + 错误记录            │
│  errors.py   — 异常类型 + 统一 error_response()      │
│  compare.py  — 跨平台比价聚合层                      │
│  cookie_harvest.py — CDP Cookie 收割 + 引导登录      │
└─────────────────────────────────────────────────────┘

pytest tests/ -v         # 全量单元测试
ruff check src/ tests/   # 零警告
pytest tests/ --cov      # 覆盖率报告
python scripts/mcp_smoke_test.py  # 本地 MCP 协议冒烟测试