graph TD
    subgraph Clients
        A[MCP Host\nClaude Desktop / CLI]
        B[Browser UI\nlocalhost:4242]
    end

    subgraph vkb process
        MCP[MCP Server\nstdio or HTTP Streamable :3333]
        HTTP[HTTP Server\n:4242]
        COORD[Coordinator\nWorker Pool]

        subgraph Workers
            IW[ingest-worker × N]
            FW[finetune-worker]
            RW[retune-worker]
        end

        subgraph Pipelines
            IPIPE[Ingest Pipeline]
            FPIPE[Finetune Pipeline]
            RPIPE[Retune Pipeline]
        end

        subgraph Adapters
            FETCH[fetch\nreadability / pdf / epub]
            CHUNK[chunk\nsliding-window]
            EMBED[embed\nollama]
            SECT[section\nsimilarity-valley]
            REL[relation\nheuristic + LLM]
            LLM[llm\nollama]
            RAW[rawstore\nfilesystem or s3]
        end
    end

    subgraph External
        PG[(Postgres\npgvector)]
        OLL[Ollama]
        FS[Filesystem / S3\nrawstore/]
    end

    A -->|MCP tools| MCP
    B -->|REST / WebSocket| HTTP
    MCP --> COORD
    HTTP --> COORD
    COORD --> IW & FW & RW
    IW --> IPIPE
    FW --> FPIPE
    RW --> RPIPE
    IPIPE & FPIPE & RPIPE --> FETCH & CHUNK & EMBED & SECT & REL & LLM & RAW
    EMBED -->|vectors| OLL
    LLM --> OLL
    RAW --> FS
    IPIPE & FPIPE & RPIPE -->|upsert| PG

flowchart TD
    START([Job queued]) --> DEDUP

    DEDUP{Content hash\nalready known?}
    DEDUP -- yes, unchanged --> SKIP([Skip — existing entity returned])
    DEDUP -- no / changed --> FETCH

    subgraph FETCH["fetching — Persist"]
        FETCH2[Fetch raw text\nURL → readability / PDF / EPUB\nor staging file]
        FETCH2 --> HASH[Compute SHA-256\ncontent hash]
        HASH --> WRITE[Write entity.md + chunks.ndjson\nto RawStore]
        WRITE --> F_COMMIT[Commit raw_store_key\nto DB ✓]
    end

    F_COMMIT --> CHUNK_S

    subgraph CHUNK_S["chunking"]
        CHUNK2[Sliding-window chunk\nwrite chunk rows to DB]
    end

    CHUNK_S --> EMBED_S

    subgraph EMBED_S["embedding"]
        EMBED2[Embed each chunk\nvia Ollama]
    end

    EMBED_S --> SECT_S

    subgraph SECT_S["sectioning"]
        SECT2[Group chunks into sections\nsimilarity-valley or positional]
    end

    SECT_S --> SUM_S

    subgraph SUM_S["summarising"]
        SUM_C[Summarise each chunk — LLM]
        SUM_SEC[Summarise each section — LLM]
        SUM_E[Summarise entity — LLM]
    end

    SUM_S --> LINK_S

    subgraph LINK_S["linking"]
        REL_H[Heuristic relation extraction]
        REL_L[LLM relation extraction\noptional]
    end

    LINK_S --> TAG_S

    subgraph TAG_S["tagging"]
        TAG[Extract tags from meta\nAssert tag:* relations\nbetween co-tagged entities]
    end

    TAG_S --> DONE([Entity ready — status = ready])

flowchart TD
    START([Finetune job queued]) --> EX

    subgraph EX["extracting"]
        TOP[Find top-N nearest neighbours\nvia embedding similarity]
        TOP --> LLM_REL[LLM relation extractor\nupsert content_llm relations]
    end

    EX --> TAG

    subgraph TAG["tagging"]
        LLM_TAG[LLM keyword tagger\nmerge tags into meta.tags]
        LLM_TAG --> TAG_REL[Assert tag:* relations\nbetween co-tagged entities]
    end

    TAG --> DONE([Finetune complete])

ollama pull nomic-embed-text
ollama pull gemma4:e4b

# 1. Clone and install
git clone <repo-url> galactic-vkb
cd galactic-vkb
npm install

# 2. Configure
cp .env.example .env
# Edit .env if needed — defaults work out of the box with Docker Compose

# 3. Start Postgres (pgvector-enabled)
npm run db:up

# 4. Run migrations
npm run migrate

# 5. Start vkb
npm run dev          # development (tsx watch)
# or
npm run build && npm start   # production

{
  "mcpServers": {
    "vkb": {
      "command": "node",
      "args": ["/absolute/path/to/galactic-vkb/dist/index.js"],
      "env": {
        "DATABASE_URL": "postgres://vkb:vkb@localhost:5433/vkb",
        "OLLAMA_BASE_URL": "http://localhost:11434",
        "EMBED_MODEL": "nomic-embed-text",
        "LLM_MODEL": "gemma4:e4b",
        "MCP_PORT": "0"
      }
    }
  }
}

{
  "mcpServers": {
    "vkb": {
      "command": "npx",
      "args": ["tsx", "/absolute/path/to/galactic-vkb/src/index.ts"],
      "env": {
        "DATABASE_URL": "postgres://vkb:vkb@localhost:5433/vkb",
        "OLLAMA_BASE_URL": "http://localhost:11434",
        "MCP_PORT": "0"
      }
    }
  }
}

MCP_PORT=0 DATABASE_URL=postgres://vkb:vkb@localhost:5433/vkb \
  claude --mcp-server "vkb:node /absolute/path/to/galactic-vkb/dist/index.js"

# 1. Query for relevant chunks
vkb_query { text: "transformer attention mechanism" }

# 2. Identify a cross-document insight, then assert it
vkb_relate { source_id: "<chunk-A>", target_id: "<chunk-B>",
             rel_type: "shares_mechanism_with" }

# 3. Traverse what's grown
vkb_neighbors { id: "<chunk-A>", hops: 2 }

Authorization: Bearer your-secret

GET /health

{ "ok": true, "uptime": 42.3, "postgres": true, "ollama": true }

# URL
curl -X POST http://localhost:4242/ingest \
  -H "Content-Type: application/json" \
  -d '{ "type": "url", "ref": "https://example.com/article" }'

# Local file (supports .md, .txt, .pdf, .epub, .yaml, .json)
curl -X POST http://localhost:4242/ingest \
  -H "Content-Type: application/json" \
  -d '{ "type": "doc", "ref": "/absolute/path/to/file.md" }'

# Inline text
curl -X POST http://localhost:4242/ingest \
  -H "Content-Type: application/json" \
  -d '{ "type": "note", "text": "Some text…", "source_context": "conversation", "meta": { "tags": ["example"] } }'

{ "ok": true, "data": { "job_id": "3f2a1b4c-…", "entity_id": "9d8e7f6a-…" } }

# Re-run pipeline for a single entity (must have stored raw content)
curl -X POST http://localhost:4242/reingest \
  -H "Content-Type: application/json" \
  -d '{ "entity_id": "9d8e7f6a-…" }'

# Re-queue all entities that have stored raw content
curl -X POST http://localhost:4242/reingest \
  -H "Content-Type: application/json" \
  -d '{}'

# Finetune specific entities
curl -X POST http://localhost:4242/finetune \
  -H "Content-Type: application/json" \
  -d '{ "entity_ids": ["9d8e7f6a-…"] }'

# Finetune all entities of a given type
curl -X POST http://localhost:4242/finetune \
  -H "Content-Type: application/json" \
  -d '{ "scope": "url" }'

curl -X POST http://localhost:4242/query \
  -H "Content-Type: application/json" \
  -d '{ "text": "How does quantum entanglement work?", "k": 5, "threshold": 0.7 }'

curl -X POST http://localhost:4242/retune \
  -H "Content-Type: application/json" \
  -d '{ "scope": "all", "force": false }'

GET /jobs?kind=ingest&stage=queued&limit=50

GET  /entities?type=url&status=ready&source_context=external&q=quantum&id=<uuid>&from=2025-01-01&limit=50&offset=0
GET  /entities/broken
GET  /entities/projection?offset=0&limit=500
GET  /entities/:id
GET  /entities/:id/raw
DELETE /entities/:id
POST /entities/bulk-action

{ "ids": ["<uuid>", "…"], "action": "delete" | "reingest" | "reingest_force" | "finetune" }

GET /chunks?entity_id=<uuid>&limit=500&offset=0
GET /chunks/projection?offset=0&limit=500
GET /chunks/:id
GET /chunks/:id/raw

GET /relations?origin=heuristic&rel_type=related_to&min_confidence=0.7&limit=50

GET /status

scripts/gen-cert.ps1