query → planner (LLM) → search+scrape×N → quality filter → dedup → rerank → summarize×M → report (LLM)

topic
  │
  ▼
Planner (LLM) ──── generates N sub-questions
  │
  ▼
Crawler (parallel per query)
  ├─ SearXNG search (→ DuckDuckGo fallback)
  └─ scrape URLs concurrently (reqwest + scraper crate)
  │
  ▼
Quality filter ──── min word count, text density
  │
  ▼
Dedup (TEI embed → cosine sim) ──── optional, requires EMBED_BASE_URL
  │
  ▼
Cross-encoder rerank (TEI) ──── optional, requires RERANK_BASE_URL
  │
  ▼
Summarizer (LLM, join_all — all calls concurrent)
  │
  ▼
Publisher (LLM) ──── final markdown report / streaming tokens

# 1. Clone and configure
git clone https://github.com/your-org/researcher.git
cd researcher
cp .env.example .env
cp infra/.env.example infra/.env
# Edit infra/.env: set LLAMA_MODELS_PATH to where your GGUF files live

# 2. Download a model (Qwen3.5-4B — ~3GB VRAM, works great)
huggingface-cli download unsloth/Qwen3.5-4B-GGUF \
  --include "Qwen3.5-4B-Q4_K_M.gguf" \
  --local-dir /path/to/your/models/unsloth/Qwen3.5-4B-GGUF/

# 3. Start infrastructure (llama-cpp + SearXNG + TEI embed + TEI rerank)
make infra-up

# 4. Start the researcher service
make up

# 5. Research something
curl -X POST http://localhost:33100/research \
  -H 'Content-Type: application/json' \
  -d '{"query": "What are the latest advances in fusion energy?"}'

cp .env.example .env
# Edit .env:
#   LLM_BASE_URL=https://api.openai.com/v1
#   LLM_MODEL=gpt-4.1-mini
#   LLM_API_KEY=sk-...
#   LLM_FAST_BASE_URL=   (leave empty — use same backend for all stages)

# Start infra (SearXNG only; llama-cpp is not required)
make infra-up
make up

curl -X POST http://localhost:33100/research \
  -H 'Content-Type: application/json' \
  -d '{"query": "Impact of quantum computing on cryptography"}'

cargo build --release

# Run against a local llama.cpp + SearXNG
LLM_BASE_URL=http://localhost:8080/v1 \
SEARXNG_URL=http://localhost:4000 \
RUST_LOG=info \
./target/release/researcher --query "Rust async runtime internals"

# Save report to file
./target/release/researcher --query "..." --output report.md

# In .env:
LLM_BASE_URL=http://host.docker.internal:11434/v1
LLM_MODEL=qwen2.5:7b
LLM_API_KEY=ollama
LLM_FAST_BASE_URL=   # empty = same model for all stages

cp .env.example .env

LLM_BASE_URL=https://generativelanguage.googleapis.com/v1beta/openai/
LLM_MODEL=gemini-2.5-flash
LLM_API_KEY=AIzaSy...          # your Google AI Studio key
LLM_MAX_TOKENS=8192
STRIP_THINKING_TOKENS=false    # not needed for Gemini

# Fast model (same endpoint — leave blank to reuse LLM_MODEL for all stages)
LLM_FAST_BASE_URL=
LLM_FAST_MODEL=

# SearXNG (bundled in infra stack)
SEARXNG_URL=http://localhost:4000

# Start infra (SearXNG only — no llama.cpp needed)
make infra-up
make up

curl -X POST http://localhost:33100/research \
  -H 'Content-Type: application/json' \
  -d '{"query": "What are the latest advances in fusion energy?"}'

cargo build --release --bin researcher-mcp
# → target/release/researcher-mcp  (~6MB)

{
  "mcpServers": {
    "researcher": {
      "command": "/path/to/researcher-mcp",
      "env": {
        "LLM_BASE_URL": "http://localhost:8080/v1",
        "LLM_MODEL": "Qwen3.5-4B-Q4_K_M",
        "SEARXNG_URL": "http://localhost:4000",
        "STRIP_THINKING_TOKENS": "true",
        "EMBED_BASE_URL": "http://localhost:30082",
        "RERANK_BASE_URL": "http://localhost:30083"
      }
    }
  }
}

{
  "mcpServers": {
    "researcher": {
      "command": "/path/to/researcher-mcp",
      "env": {
        "LLM_BASE_URL": "https://generativelanguage.googleapis.com/v1beta/openai/",
        "LLM_MODEL": "gemini-2.5-flash",
        "LLM_API_KEY": "AIzaSy...",
        "LLM_MAX_TOKENS": "8192",
        "STRIP_THINKING_TOKENS": "false",
        "SEARXNG_URL": "http://localhost:4000"
      }
    }
  }
}

curl -X POST http://localhost:33100/research \
  -H 'Content-Type: application/json' \
  -d '{
    "query": "Rust async runtimes compared",
    "mode": "report",
    "max_queries": 4,
    "max_sources": 4
  }'

{
  "topic": "Rust async runtimes compared",
  "queries": ["What is Tokio?", "..."],
  "source_count": 14,
  "report": "# Research Report\n\n..."
}

curl -X POST http://localhost:33100/research/stream \
  -H 'Content-Type: application/json' \
  -d '{"query": "history of the internet"}' \
  --no-buffer

data: {"type":"progress","message":"🔍 Planning research queries..."}
data: {"type":"progress","message":"📋 Generated 4 search queries","data":{"queries":[...]}}
data: {"type":"progress","message":"🌐 Crawling 4 queries in parallel..."}
data: {"type":"token","token":"# Research Report\n\n"}
...
event: complete
data: {"type":"complete","topic":"...","report":"# Research Report\n\n..."}

[my-profile]
domains = ["example.com", "docs.example.com"]

[job-profile]
title = "Senior AI Engineer"
seniority = "senior"
salary_floor = "150000 USD"
remote_only = true
skills = ["Rust", "Python", "LLMs", "MLOps", "RAG"]
preferred_company_size = "startup to mid-size"
avoid_industries = ["gambling", "crypto"]
about_me = """
Brief summary of your background and what you're looking for.
"""

infra/docker-compose.yml   ← always-on: SearXNG, llama-cpp, TEI embed, TEI rerank
docker-compose.yml         ← researcher app only (joins ai-infra-net)

# Start infrastructure first
make infra-up

# Then start the researcher app
make up

# Logs
make infra-logs   # infrastructure services
make logs         # researcher app

# Stop everything
make stop-all

# Prerequisites (Debian/Ubuntu)
sudo apt-get install pkg-config libssl-dev

# Type-check only (fast)
cargo check

# Build both binaries (optimized — ~30-60s with LTO)
cargo build --release

# Lint
cargo clippy -- -D warnings

docker build -t researcher .
# Multi-stage build: rust:slim builder → distroless runtime (~8MB total)