bun install
bun run dev          # HTTP API + web UI on port 3000

curl "https://mdingest.knightker.workers.dev/v1/medium?url=https://medium.com/@user/article-id"
curl "https://mdingest.knightker.workers.dev/v1/devto?url=https://dev.to/user/article-slug"
curl "https://mdingest.knightker.workers.dev/v1/substack?url=https://pub.substack.com/p/article-slug"

curl "https://mdingest.knightker.workers.dev/v1/substack?url=https://substack.com/home/post/p-212696442"

---
title: "Article Title"
author: "Author Name"
date: "2026-01-15"
reading_time: "5 min read"
free: true
source_url: "https://medium.com/@user/article-id"
provider: "medium"
tags:
  - "Distributed Systems"
---

# Article content in clean Markdown...

curl "https://mdingest.knightker.workers.dev/v1/medium?url=https://medium.com/@user/article-id&format=json"

{
  "metadata": {
    "title": "Article Title",
    "author": "Author Name",
    "date": "2026-01-15",
    "reading_time": "5 min read",
    "free": true,
    "source_url": "https://medium.com/@user/article-id",
    "provider": "medium",
    "tags": ["Distributed Systems"]
  },
  "markdown": "---\ntitle: \"Article Title\"...\n"
}

bun run dev
curl "http://localhost:3000/v1/medium?url=https://medium.com/@user/article-id"

# Ingest any article URL → markdown to stdout (auto-detects provider)
bun run src/cli.ts https://dev.to/user/post > article.md

# JSON output (metadata + markdown)
bun run src/cli.ts https://dev.to/user/post --json

# Override provider auto-detection
bun run src/cli.ts https://example.com/post --provider medium

# List supported providers
bun run src/cli.ts providers

{
  "mcpServers": {
    "mdingest": {
      "url": "https://mdingest.knightker.workers.dev/v1/mcp"
    }
  }
}

{
  "mcpServers": {
    "mdingest": {
      "command": "bun",
      "args": ["run", "src/cli.ts", "mcp"]
    }
  }
}

src/
  core/
    config/         Zod-validated config (boot-time, frozen object)
    cache/          In-memory LRU cache
  integrations/
    freedium/       HTTP client for Freedium mirror (download + data endpoints)
  common/
    types/          Shared types: ArticleMetadata schema, Provider interface
    pipes/          ZodValidationPipe (validates controller input)
    filters/        AllExceptionsFilter (shapes errors to { code, message, details?, traceId })
    guards/         RateLimitGuard (30 req/min per IP, global via APP_GUARD)
    llm-visibility.ts  Fastify preHandler: Accept: text/markdown negotiation, Link headers, Vary, 406
    errors/         shapeError() — shared error shaping (filter, CLI, MCP)
  modules/
    medium/         Medium feature: controller, service, DTOs, errors
    devto/          Dev.to feature: controller, service, DTOs, errors
    substack/       Substack feature: controller, service, DTOs, errors
  mcp/              MCP server: server.ts (createMcpServer/startMcpServer/handleHttpRequest), tools.ts (ingest_article, list_providers)
  app.module.ts     Root module
  main.ts           Bootstrap (NestJS + Fastify + Bun, URI versioning, global filter)
  ingest.ts         Shared router: service registry + ingest(url) — used by CLI + MCP
  cli.ts            CLI entry: citty binary, `mdingest <url>` + `mdingest mcp` subcommand
  mcp.controller.ts MCP HTTP endpoint at /v1/mcp (Streamable HTTP transport)
  worker.ts         Cloudflare Worker — routes requests to the Docker container

shared/
  providers.ts      Single source of truth: MEDIUM_DOMAINS, PROVIDERS, detectProvider(), error codes

web/                 Astro static site (React islands)
  src/
    islands/        Ingestor.tsx — URL input, provider selector, output display
    pages/          index.astro (landing), ingest.astro (ingest UI), docs.astro (API docs)
    layouts/        Base.astro — shared header (sticky, backdrop-blur), footer, meta tags
    styles/         global.css — design tokens, base styles, shared components
  public/
    icons/          Provider SVG logos (medium, devto, substack)
    robots.txt      Allow all crawlers + Content-Signal directive + sitemap reference
    llms.txt        Curated markdown index for AI-mediated conversations
    llms-full.txt   All 3 pages concatenated for single-fetch LLM ingestion
    *.md             Markdown twins of each HTML page (index.md, ingest.md, docs.md)
    .well-known/
      ai-catalog.json          AI Catalog — domain-level agent discovery
      api-catalog              RFC 9727 API Catalog (linkset+json)
      mcp/server-card.json     MCP Server Card — pre-connection MCP client metadata

bun run verify    # typecheck + lint + impeccable (UI anti-pattern scan)
bun run dev       # start backend dev server with hot reload (port 3000)
bun run dev:web   # start Astro dev server (frontend only, port 4321)
bun run build:web # build Astro frontend to web/dist/
bun run test      # run unit tests (vitest)
bun run cli       # run CLI (bun run src/cli.ts <url>)
bun run mcp       # start MCP server (bun run src/cli.ts mcp)