webcli

# Clone and install
git clone https://github.com/ashark-ai-05/webcli.git
cd webcli
npm install

# List available adapters
npx tsx src/main.ts list

# Search Hacker News
npx tsx src/main.ts run hackernews search -a query=AI -a limit=5

# Get live crypto prices
npx tsx src/main.ts run coingecko prices -a coins=bitcoin,ethereum,solana

# Search Bluesky users
npx tsx src/main.ts run bluesky search -a query=typescript -a limit=5

# Search stories
webcli run hackernews search -a query=rust -a limit=10

# Output as JSON (for piping to other tools)
webcli run hackernews search -a query=rust -f json

# Output as CSV (for spreadsheets)
webcli run hackernews search -a query=rust -f csv

# Live prices
webcli run coingecko prices -a coins=bitcoin,ethereum,solana

# Trending coins
webcli run coingecko trending -a limit=10

# Search users
webcli run bluesky search -a query=developer -a limit=5

# Get a profile
webcli run bluesky profile -a handle=bsky.app

# Start MCP server
npx tsx src/main.ts --mcp

{
  "mcpServers": {
    "webcli": {
      "command": "npx",
      "args": ["tsx", "/path/to/webcli/src/main.ts", "--mcp"]
    }
  }
}

site: coingecko
name: prices
description: Live cryptocurrency prices
version: 1

source:
  type: api
  base_url: https://api.coingecko.com/api/v3
  auth: none
  rate_limit: { requests: 30, per: 60 }

args:
  coins: { type: string, default: "bitcoin,ethereum" }
  currency: { type: string, default: usd }

schema:
  type: price_feed
  entity: cryptocurrency
  fields:
    id: { role: id }
    price: { role: value, unit: currency }
    change_24h: { role: delta }
    market_cap: { role: metric }
  update_frequency: "~60s"

pipeline:
  - fetch:
      url: /simple/price
      params:
        ids: "${{ args.coins }}"
        vs_currencies: "${{ args.currency }}"
        include_market_cap: true
        include_24hr_change: true
  - transform: |
      Object.entries(data).map(([id, v]) => ({
        id, price: v[args.currency],
        change_24h: v[args.currency + '_24h_change'].toFixed(2) + '%',
        market_cap: v[args.currency + '_market_cap']
      }))

columns: [id, price, change_24h, market_cap]

                    ┌─────────┐   ┌────────────┐
                    │   CLI   │   │ MCP Server │
                    └────┬────┘   └─────┬──────┘
                         └───────┬──────┘
                                 v
                    ┌────────────────────────┐
                    │      Core Engine       │
                    │  Pipeline Executor     │
                    │  Adapter Registry      │
                    │  Rate Limiter          │
                    └───────────┬────────────┘
                                v
              ┌─────────────────────────────────┐
              │        Extractor Cascade        │
              │  HTTP/API > RSS > HTML > Browser│
              └─────────────────────────────────┘
                                v
              ┌─────────────────────────────────┐
              │       Discovery Engine          │
              │  Schema Inference               │
              │  Known Site Registry (8 sites)  │
              │  Field Role Detection           │
              └─────────────────────────────────┘
                                v
              ┌─────────────────────────────────┐
              │          Data Layer             │
              │  Snapshot Store + Diff Engine   │
              │  Subscription Manager           │
              │  Adaptive Interval Scheduler    │
              └─────────────────────────────────┘

Available:  args, item, index, data, Math, Date, JSON, String, Array, Object
BLOCKED:    fetch, require, import, process, globalThis, eval, Function,
            setTimeout, Buffer, fs, child_process

rate_limits:
  global: { max_requests_per_minute: 120 }
  per_site:
    default: { max_requests_per_minute: 30 }
    overrides:
      api.coingecko.com: 50

User-Agent: webcli/0.1.0 (+https://github.com/webcli)

# Pretty table (default)
webcli run hackernews search -a query=AI

# JSON (for piping to jq, agents, scripts)
webcli run hackernews search -a query=AI -f json

# CSV (for spreadsheets, pandas)
webcli run hackernews search -a query=AI -f csv

// Snapshot store saves results to ~/.webcli/data/{site}/{name}/
// Diff engine compares snapshots to detect added/removed/changed items
{
  "added": [{ "id": "solana", "price": 142.50 }],
  "removed": [],
  "changed": [{
    "id": "bitcoin",
    "fields": { "price": { "old": 71000, "new": 71250 } }
  }],
  "unchanged_count": 1
}

# Run tests
npm test

# Watch mode
npm run test:watch

# Type check
npm run lint

# Run the CLI in dev mode
npm run dev -- list

webcli/
  src/
    core/           # Pipeline engine, types, adapter system
    extractors/     # HTTP, RSS, HTML, cascade
    discovery/      # Schema inference, known sites
    data/           # Snapshots, diffs, subscriptions
    scheduler/      # Adaptive intervals
    interfaces/     # CLI (commander) + MCP server
    utils/          # Config, logging
  adapters/         # Built-in YAML adapters
  tests/e2e/        # End-to-end tests (hit live APIs)

site: mysite
name: feed
description: My custom feed
version: 1
created_by: manual

source:
  type: api
  base_url: https://api.mysite.com
  auth: none

args:
  limit: { type: number, default: 10 }

schema:
  type: feed
  entity: article
  fields:
    title: { role: title }
    url: { role: url }

pipeline:
  - fetch: { url: /posts, params: { limit: "${{ args.limit }}" } }
  - select: data
  - map:
      title: "${{ item.title }}"
      url: "${{ item.url }}"
      date: "${{ item.published_at }}"
  - limit: "${{ args.limit }}"

columns: [title, url, date]

webcli validate ~/.webcli/adapters/mysite/feed.yaml

webcli run mysite feed -a limit=5