chmod +x snap-mcp

mkdir -p data/logs data/staging data/repos data/projects

# SQLite is the default — no additional setup required
SNAP_DB_MODE=sqlite
SNAP_SQLITE_PATH=data/snap.db

# Optional: PostgreSQL
# SNAP_DB_MODE=postgres
# SNAP_POSTGRES_DSN=postgresql://user:pass@localhost:5432/snap

# Logging
SNAP_LOG_LEVEL=INFO
SNAP_LOG_JSON=true

# Windows
snap-mcp.exe --help

# Linux / macOS
./snap-mcp --help

LLM: clone_to_repos(repo_url, vendor_id)
    └─► Clones into repos/{project_id}/. project_id = repo name, derived by SNAP.
SNAP: auto-ingests in background thread → stores in DB → clears repos/
LLM (on request): query_snapshots / get_project_notebook

LLM: copy_to_staging(project_id, source_path)
    └─► Copies files to staging/{project_id}/. Returns immediately.
SNAP: auto-ingests in background thread → stores in DB → clears staging/
LLM (on request): query_snapshots / get_project_notebook

FileHeader (variable size):
  magic:            "SNAPFILE" (8 bytes)
  version:          uint16 (2 bytes)
  project_id_len:   uint16 (2 bytes)
  project_id:       utf-8 string (variable)
  snapshot_count:   uint32 (4 bytes)
  [file content follows]

from app.extraction.binary_packer import write_file_header, read_project_id_from_file

# Write file with project association
content = b"# Project Notes\n\nImplementation details..."
write_file_header("notes.md", "SNAP", content)

# Read project_id from file
project_id = read_project_id_from_file("notes.md")  # Returns "SNAP"

1. File with binary header uploaded via upload_to_staging
2. SNAP reads header → extracts project_id
3. File placed in staging/{project_id}/
4. Auto-ingested into {project_id} project in background
5. Staging cleared

# List all ingested projects with snapshot and run counts
snap-admin list-projects

# Show all runs for a project (active / superseded / failed)
snap-admin runs <project_id>

# Health check and active-run summary for a project
snap-admin manifest <project_id>

# Browse snapshots — summary by type, or drill in by type or file
snap-admin snapshots <project_id>
snap-admin snapshots <project_id> --type <snapshot_type>
snap-admin snapshots <project_id> --file <source_file_path>

# Delete a project and all its data (DB, repos, staging)
snap-admin delete-project <project_id>

# Copy a local directory into staging for a project
snap-admin upload-to-staging <project_id> <source_path>

# Clone a GitHub repo directly (no LLM involved) — repos_watcher ingests
snap-admin clone-github <repo_url>

SNAP_DB_MODE=sqlite        # Default
SNAP_DB_MODE=postgres      # Requires SNAP_POSTGRES_DSN
SNAP_DB_MODE=dual          # Both databases

SnapshotHeader (561 bytes):
  magic:         "SNAP" (4 bytes)
  version:       uint16 (2 bytes)
  snapshot_type: uint8 (1 byte)
  field_count:   uint16 (2 bytes)
  content_hash:  SHA-256 (32 bytes)
  simhash:       uint64 (8 bytes)
  minhash:       128 × uint32 (512 bytes)

FieldDescriptor (11 bytes each):
  field_id:      uint16 (2 bytes)
  data_type:     uint8 (1 byte)  # 0=string, 1=int, 2=binary, 3=array
  offset:        uint32 (4 bytes)
  length:        uint32 (4 bytes)

Data Block (variable):
  Packed field data referenced by descriptors

CREATE TABLE snapshot_notebooks (
    snapshot_id   TEXT PRIMARY KEY,
    run_id        TEXT NOT NULL,
    project_id    TEXT NOT NULL,
    snapshot_type TEXT NOT NULL,
    source_file   TEXT NOT NULL,
    binary_data   BYTEA NOT NULL,           -- Binary-packed snapshot
    source_hash   TEXT,
    content_hash  TEXT,                      -- SHA-256 hex
    simhash       BIGINT,                    -- 64-bit similarity hash
    minhash       TEXT,                      -- 128 × 32-bit MinHash (CSV)
    created_at    TIMESTAMPTZ NOT NULL DEFAULT NOW()
);

New file ingested:
  1. Calculate source_hash
  2. Query DB for existing snapshot with same source_file + source_hash
  3. If exists → skip (deduplication)
  4. If not exists → create new snapshot (versioning)
  5. Multiple versions coexist in DB (query by run_id or latest)

{"ts": "2026-02-15 12:00:00,000", "level": "INFO", "name": "snap", "msg": "Snapshot created", "snapshot_id": "...", "project_id": "...", "snapshot_type": "functions", "parser": "tree_sitter", "fields_count": 12}

2026-02-15 12:00:00,000 INFO snap Snapshot created

GitHub:
clone_to_repos(repo_url)
    ↓ clone completes synchronously
repos/{project_id}/
    ↓ auto-ingest background thread
ingest_cloned_repo()    ← security filtering, file enumeration
    ↓
file_router → parsers → field_mapper → snapshot_builder → DB
    ↓
repos/ cleared

Local:
copy_to_staging(source_path)
    ↓ stage_directory() filters and copies
staging/{project_id}/
    ↓ auto-ingest background thread
process_project()       ← security filtering, file enumeration
    ↓
file_router → parsers → field_mapper → snapshot_builder → DB
    ↓
staging/ cleared

SNAP/
├── app/
│   ├── admin.py                         # Admin CLI (human-only: delete, upload, clone, list)
│   ├── main.py                          # Orchestration pipeline
│   ├── config/
│   │   └── settings.py
│   ├── extraction/
│   │   ├── binary_packer.py             # Binary snapshot packer/unpacker (Nim-compatible)
│   │   ├── field_mapper.py              # Maps parser output to snapshot types
│   │   └── snapshot_builder.py
│   ├── ingest/
│   │   ├── file_router.py               # Routes files to parsers by extension
│   │   ├── github_cloner.py             # Shallow clone → repos/
│   │   └── local_loader.py              # stage_directory() + staging helpers
│   │
│   ├── logging/
│   │   └── logger.py
│   ├── mcp/
│   │   ├── auth.py                      # JWT + GitHub OAuth
│   │   ├── run.py                       # Entry point: stdio or HTTP+SSE
│   │   ├── security.py                  # Input validation, path traversal prevention
│   │   ├── server.py                    # MCP server, tool registry, Starlette app
│   │   └── tools.py                     # Tool handlers + permission enforcement
│   ├── parsers/
│   │   ├── nim_parser.nim               # Native parser: doc, csv, config formats (compile to binary)
│   │   ├── nim_parser.py                # Python wrapper for Nim parser
│   │   ├── pre_converter.nim            # Pre-processing helper for Nim parser
│   │   ├── semgrep_parser.py
│   │   └── tree_sitter_parser.py
│   ├── schemas/
│   │   ├── master_notebook.yaml
│   │   └── snapshot_templates/          # JSON templates (defined and gated by master_notebook.yaml)
│   ├── security/
│   │   └── network_policy.py
│   └── storage/
│       ├── db.py
│       └── snapshot_repo.py             # CRUD, upsert, run versioning
├── data/
│   ├── logs/
│   ├── projects/                        # Project manifests
│   ├── repos/                           # GitHub clones (cleared after ingest)
│   └── staging/                         # Local file staging (cleared after ingest)
├── docker/
│   └── Dockerfile
├── docker-compose.yml
├── pyproject.toml
├── run_mcp.bat
└── run_mcp.sh

SNAP_POSTGRES_DSN=postgresql://user:pass@localhost:5432/snap

.venv\Scripts\python.exe -m pip install --upgrade semgrep