┌─────────────────────────────────────────────────────────────────────────┐
│                            ARBuilder                                    │
├─────────────────────────────────────────────────────────────────────────┤
│                                                                         │
│  DATA PIPELINE                                                          │
│  ┌──────────┐    ┌──────────┐    ┌───────────┐    ┌──────────────────┐ │
│  │ Scraper  │───▶│Processor │───▶│ Embedder  │───▶│    ChromaDB      │ │
│  │ crawl4ai │    │ 3-layer  │    │ BGE-M3    │    │ (local vectors)  │ │
│  │ + GitHub │    │ filters  │    │ 1024-dim  │    │                  │ │
│  └──────────┘    └──────────┘    └───────────┘    └────────┬─────────┘ │
│                                                             │           │
│  RETRIEVAL                                                  │           │
│  ┌──────────────────────────────────────────────────────────▼─────────┐ │
│  │                    Hybrid Search Engine                             │ │
│  │  ┌──────────┐    ┌──────────┐    ┌───────────┐                    │ │
│  │  │  Vector  │    │   BM25   │    │CrossEncoder│   RRF Fusion      │ │
│  │  │  Search  │───▶│ Keywords │───▶│ Reranker  │──▶ + MMR           │ │
│  │  └──────────┘    └──────────┘    └───────────┘                    │ │
│  └───────────────────────────────────────────────────────────────────┘ │
│                                         │                               │
│  GENERATION                             ▼                               │
│  ┌───────────────────────────────────────────────────────────────────┐  │
│  │                      MCP Server (19 tools)                        │  │
│  │                                                                   │  │
│  │  M1: Stylus        M2: SDK           M3: dApp Builder            │  │
│  │  ┌─────────────┐   ┌─────────────┐   ┌──────────────────────┐   │  │
│  │  │ generate_    │   │ generate_   │   │ generate_backend     │   │  │
│  │  │ stylus_code  │   │ bridge_code │   │ generate_frontend    │   │  │
│  │  │ ask_stylus   │   │ generate_   │   │ generate_indexer     │   │  │
│  │  │ get_context  │   │ messaging   │   │ generate_oracle      │   │  │
│  │  │ gen_tests    │   │ ask_bridging│   │ orchestrate_dapp     │   │  │
│  │  │ get_workflow │   │             │   │                      │   │  │
│  │  │ validate_code│   │             │   │                      │   │  │
│  │  └─────────────┘   └─────────────┘   └──────────────────────┘   │  │
│  │                                                                   │  │
│  │  M4: Orbit Chain                                                  │  │
│  │  ┌──────────────────────┐                                        │  │
│  │  │ generate_orbit_config│                                        │  │
│  │  │ generate_orbit_deploy│                                        │  │
│  │  │ gen_validator_setup  │                                        │  │
│  │  │ ask_orbit            │                                        │  │
│  │  │ orchestrate_orbit    │                                        │  │
│  │  └──────────────────────┘                                        │  │
│  └───────────────────────────────────────────────────────────────────┘  │
│                           │                                             │
│  IDE INTEGRATION          ▼                                             │
│  ┌───────────────────────────────────────────────────────────────────┐  │
│  │  Cursor / VS Code / Claude Desktop / Any MCP Client              │  │
│  │  <- via local stdio or remote mcp-remote proxy ->                │  │
│  └───────────────────────────────────────────────────────────────────┘  │
│                                                                         │
│  HOSTED SERVICE (Cloudflare Workers)                                    │
│  ┌──────────────┐  ┌──────────┐  ┌──────────┐  ┌──────────────────┐  │
│  │  Workers AI  │  │ Vectorize│  │    D1    │  │      KV         │  │
│  │  BGE-M3 +    │  │ 1024-dim │  │  Users   │  │   Source registry│  │
│  │  Reranker    │  │  index   │  │  API keys│  │   + Ingest state │  │
│  └──────────────┘  └──────────┘  └──────────┘  └──────────────────┘  │
│                                                                       │
│  INGESTION PIPELINE (Worker-native, cron every 6h)                    │
│  ┌──────────┐    ┌──────────┐    ┌───────────┐    ┌──────────────┐  │
│  │ scraper  │───▶│ chunker  │───▶│ Workers AI│───▶│  Vectorize   │  │
│  │ HTML/    │    │ doc+code │    │  BGE-M3   │    │   upsert     │  │
│  │ GitHub   │    │ splitter │    │ embedding │    │              │  │
│  └──────────┘    └──────────┘    └───────────┘    └──────────────┘  │
│                       │                ▲                             │
│                       │ >30 files      │ embed messages              │
│                       ▼                │                             │
│              ┌─────────────────────────┴──┐                         │
│              │    CF Queue (async path)    │                         │
│              │  embed │ continue │finalize │                         │
│              └────────────────────────────┘                         │
└─────────────────────────────────────────────────────────────────────────┘

# No local setup needed - just configure your IDE
# Add to ~/.cursor/mcp.json:
{
  "mcpServers": {
    "arbbuilder": {
      "command": "npx",
      "args": ["-y", "mcp-remote", "https://arbuilder.app/mcp",
               "--header", "Authorization: Bearer YOUR_API_KEY"]
    }
  }
}

# 1. Clone and setup
git clone https://github.com/Quantum3-Labs/ARBuilder.git
cd ARBuilder
conda env create -f environment.yml
conda activate arbbuilder

# 2. Configure API key
cp .env.example .env
# Edit .env and add your OPENROUTER_API_KEY and NVIDIA_API_KEY

# 3. Generate vector database (required)
python -m src.embeddings.vectordb

# 4. Test MCP server
python -m src.mcp.server
# Should show: "Capabilities: 19 tools, 11 resources, 5 prompts"

# 5. Configure Cursor IDE (~/.cursor/mcp.json) - see Setup section below

ArbBuilder/
├── sources.json          # Single source of truth for all data sources
├── scraper/              # Data collection module
│   ├── config.py         # Thin wrapper around sources.json (backward-compat helpers)
│   ├── scraper.py        # Web scraping with crawl4ai
│   ├── github_scraper.py # GitHub repository cloning
│   └── run.py            # Pipeline entry point
├── src/
│   ├── preprocessing/    # Text cleaning and chunking
│   │   ├── cleaner.py    # Text normalization
│   │   ├── chunker.py    # Document chunking with token limits
│   │   └── processor.py  # Main preprocessing pipeline
│   ├── embeddings/       # Embedding and vector storage
│   │   ├── embedder.py   # OpenRouter embedding client
│   │   ├── vectordb.py   # ChromaDB wrapper with hybrid search (BM25 + vector)
│   │   └── reranker.py   # CrossEncoder, MMR, LLM reranking
│   ├── templates/        # Code generation templates
│   │   ├── stylus_templates.py   # M1: Stylus contract templates
│   │   ├── backend_templates.py  # M3: NestJS/Express templates
│   │   ├── frontend_templates.py # M3: Next.js + wagmi templates
│   │   ├── indexer_templates.py  # M3: Subgraph templates
│   │   ├── oracle_templates.py   # M3: Chainlink templates
│   │   └── orbit_templates.py    # M4: Orbit chain deployment templates
│   ├── utils/            # Shared utilities
│   │   ├── version_manager.py   # SDK version management
│   │   ├── env_config.py        # Centralized env var configuration
│   │   ├── abi_extractor.py     # Stylus ABI extraction from Rust code
│   │   └── compiler_verifier.py # Docker-based cargo check verification
│   ├── mcp/              # MCP server for IDE integration
│   │   ├── server.py     # MCP server (tools, resources, prompts)
│   │   ├── tools/        # MCP tool implementations (19 tools)
│   │   │   ├── get_stylus_context.py       # M1
│   │   │   ├── generate_stylus_code.py     # M1
│   │   │   ├── ask_stylus.py               # M1
│   │   │   ├── generate_tests.py           # M1
│   │   │   ├── get_workflow.py             # M1
│   │   │   ├── validate_stylus_code.py     # M1
│   │   │   ├── generate_bridge_code.py     # M2
│   │   │   ├── generate_messaging_code.py  # M2
│   │   │   ├── ask_bridging.py             # M2
│   │   │   ├── generate_backend.py         # M3
│   │   │   ├── generate_frontend.py        # M3
│   │   │   ├── generate_indexer.py         # M3
│   │   │   ├── generate_oracle.py          # M3
│   │   │   ├── orchestrate_dapp.py         # M3
│   │   │   ├── generate_orbit_config.py    # M4
│   │   │   ├── generate_orbit_deployment.py # M4
│   │   │   ├── generate_validator_setup.py # M4
│   │   │   ├── ask_orbit.py                # M4
│   │   │   └── orchestrate_orbit.py        # M4
│   │   ├── resources/    # Static knowledge (11 resources)
│   │   │   ├── stylus_cli.py      # M1
│   │   │   ├── workflows.py       # M1
│   │   │   ├── networks.py        # M1
│   │   │   ├── coding_rules.py    # M1
│   │   │   ├── sdk_rules.py       # M2
│   │   │   ├── backend_rules.py   # M3
│   │   │   ├── frontend_rules.py  # M3
│   │   │   ├── indexer_rules.py   # M3
│   │   │   └── oracle_rules.py    # M3
│   │   └── prompts/      # Workflow templates
│   └── rag/              # RAG pipeline (TBD)
├── tests/
│   ├── mcp_tools/        # MCP tool test cases and benchmarks
│   │   ├── test_get_stylus_context.py
│   │   ├── test_generate_stylus_code.py
│   │   ├── test_ask_stylus.py
│   │   ├── test_generate_tests.py
│   │   ├── test_m2_e2e.py    # M2 end-to-end tests
│   │   ├── test_m3_tools.py  # M3 full dApp tests
│   │   └── benchmark.py      # Evaluation framework
│   └── test_retrieval.py # Retrieval quality tests
├── docs/
│   └── mcp_tools_spec.md # MCP tools specification
├── apps/web/               # Hosted service (Cloudflare Workers + Next.js)
│   ├── src/lib/
│   │   ├── scraper.ts      # Web doc scraping (HTMLRewriter)
│   │   ├── github.ts       # GitHub repo scraping (Trees/Contents API)
│   │   ├── chunker.ts      # Document + code chunking
│   │   ├── ingestPipeline.ts # Ingestion orchestrator (sync + async queue paths)
│   │   └── vectorize.ts    # Search + embedding utilities
│   ├── src/app/api/admin/  # Admin APIs (sources, ingest, migrate)
│   ├── worker.ts           # Worker entry + cron + queue consumer handler
│   └── wrangler.prod.jsonc # Production config (D1, KV, Vectorize, Queue)
├── scripts/
│   ├── run_benchmarks.py     # Benchmark runner
│   ├── diff-migrate.ts       # Push chunks to CF Vectorize
│   ├── sync_sources.ts       # Sync sources.json to CF KV registry
│   └── ingest_m3_sources.py  # M3 source ingestion
├── data/
│   ├── raw/              # Raw scraped data (docs + curated repos)
│   ├── processed/        # Pre-processed chunks
│   └── chroma_db/        # ChromaDB vector store (generated locally, not in repo)
├── environment.yml       # Conda environment specification
├── pyproject.toml        # Project metadata and dependencies
└── .env                  # Environment variables (not committed)

# Create and activate the environment
conda env create -f environment.yml
conda activate arbbuilder

cp .env.example .env

OPENROUTER_API_KEY=your-api-key
NVIDIA_API_KEY=your-nvidia-api-key
DEFAULT_MODEL=deepseek/deepseek-v3.2
DEFAULT_EMBEDDING=baai/bge-m3
DEFAULT_CROSS_ENCODER=nvidia/llama-3.2-nv-rerankqa-1b-v2

# Generate the vector database (required before using MCP tools)
python -m src.embeddings.vectordb

# Run the MCP server directly (press Ctrl+C to exit)
python -m src.mcp.server

ARBuilder MCP Server started
Capabilities: 19 tools, 11 resources, 5 prompts

# Run full pipeline (web scraping + GitHub cloning)
python -m scraper.run

# Then preprocess the raw data
python -m src.preprocessing.processor

# And re-ingest into ChromaDB
python -m src.embeddings.vectordb --reset

# Audit: compare repos on disk vs config
python scripts/audit_data.py

# Show what orphan repos would be deleted
python scripts/audit_data.py --prune

# Actually delete orphan repos
python scripts/audit_data.py --prune --confirm

# Include ChromaDB stats in audit
python scripts/audit_data.py --chromadb

# GitHub scraper also supports audit/prune
python -m scraper.github_scraper --audit
python -m scraper.github_scraper --prune --dry-run

# Dry run: show what would change without modifying anything
python scripts/fork_and_migrate.py --all --dry-run

# Migrate all 13 Stylus repos
python scripts/fork_and_migrate.py --all

# Migrate a specific repo
python scripts/fork_and_migrate.py --repo OffchainLabs/stylus-hello-world

# Re-verify already-forked repos after manual fixes
python scripts/fork_and_migrate.py --all --verify-only

{
  "mcpServers": {
    "arbbuilder": {
      "command": "/path/to/miniconda3/envs/arbbuilder/bin/python3",
      "args": ["-m", "src.mcp.server"],
      "env": {
        "OPENROUTER_API_KEY": "your-api-key",
        "PYTHONPATH":"/path/to/ArbBuilder"
      }
    }
  }
}

{
  "mcpServers": {
    "arbbuilder": {
      "command": "python",
      "args": ["-m", "src.mcp.server"],
      "cwd": "/path/to/ArbBuilder",
      "env": {
        "OPENROUTER_API_KEY": "your-api-key"
      }
    }
  }
}

{
  "mcpServers": {
    "arbbuilder": {
      "command": "npx",
      "args": ["-y", "mcp-remote", "https://arbuilder.app/mcp",
               "--header", "Authorization: Bearer YOUR_API_KEY"]
    }
  }
}

conda activate arbbuilder

# Run full pipeline (web scraping + GitHub cloning)
python -m scraper.run

# Preprocess and push to CF Vectorize
python -m src.preprocessing.processor
AUTH_SECRET=xxx npx tsx scripts/diff-migrate.ts --full

{
  "url": "https://github.com/ARBuilder-Forks/stylus-hello-world",
  "versions": [
    { "sdkVersion": "0.10.0", "branch": "main" },
    { "sdkVersion": "0.9.0", "branch": "v0.9.0" }
  ]
}

ARBBUILDER_ADMIN_SECRET=xxx npx tsx scripts/sync_sources.ts
ARBBUILDER_ADMIN_SECRET=xxx npx tsx scripts/sync_sources.ts --dry-run
ARBBUILDER_ADMIN_SECRET=xxx npx tsx scripts/sync_sources.ts --remove-stale

https://arbuilder.app/mcp

{
  "workflow_type": "deploy",
  "network": "arbitrum_sepolia",
  "include_troubleshooting": true
}

# Check balance
cast balance YOUR_ADDRESS --rpc-url https://sepolia-rollup.arbitrum.io/rpc

# Deploy contract
cargo stylus deploy --private-key-path=./key.txt --endpoint=https://sepolia-rollup.arbitrum.io/rpc

User: "Deploy my contract to Arbitrum Sepolia"
    ↓
AI IDE calls get_workflow(workflow_type="deploy", network="arbitrum_sepolia")
    ↓
Returns structured commands + troubleshooting
    ↓
AI IDE presents commands to user (user executes locally)

User: "Generate a counter contract for SDK 0.9.0"
AI uses: generate_stylus_code(prompt="...", target_version="0.9.0")
Returns: Code using msg::sender(), .getter(), print_abi() patterns

User: "Create an ERC20 token called MyToken with 1 million supply"

AI uses: generate_stylus_code tool
Returns: Complete Rust contract based on SimpleERC20 template with proper imports, storage, and methods

User: "Show me how to implement a mapping in Stylus"

AI uses: get_stylus_context tool
Returns: Relevant documentation and code snippets from official examples

User: "Why am I getting 'storage not initialized' error?"

AI uses: ask_stylus tool
Returns: Explanation with solution based on documentation context

User: "Write unit tests for this counter contract: [paste code]"

AI uses: generate_tests tool
Returns: Comprehensive test module with edge cases

User: "How do I deploy to Arbitrum Sepolia?"

AI uses: get_workflow tool
Returns: Commands for checking balance, deploying, and verifying

# Example: Generate ETH deposit code
echo '{"method": "tools/call", "id": 1, "params": {"name": "generate_bridge_code", "arguments": {"bridge_type": "eth_deposit", "amount": "0.5"}}}' | python -m src.mcp.server

# Example: Generate full dApp scaffold
echo '{"method": "tools/call", "params": {"name": "orchestrate_dapp", "arguments": {"prompt": "Create a token staking dApp", "components": ["contract", "backend", "frontend", "indexer"]}}}' | python -m src.mcp.server

# Example: Generate backend only
echo '{"method": "tools/call", "params": {"name": "generate_backend", "arguments": {"prompt": "Create a staking API", "framework": "nestjs"}}}' | python -m src.mcp.server

# Example: Generate frontend with contract ABI
echo '{"method": "tools/call", "params": {"name": "generate_frontend", "arguments": {"prompt": "Create token dashboard", "contract_abi": "[...]"}}}' | python -m src.mcp.server

# Example: Scaffold a complete Orbit chain deployment project
echo '{"method": "tools/call", "params": {"name": "orchestrate_orbit", "arguments": {"prompt": "Deploy an AnyTrust chain on Arbitrum Sepolia", "chain_name": "my-orbit-chain", "chain_id": 412346, "is_anytrust": true, "parent_chain": "arbitrum-sepolia"}}}' | python -m src.mcp.server

# Example: Generate chain configuration
echo '{"method": "tools/call", "params": {"name": "generate_orbit_config", "arguments": {"prompt": "Configure a custom gas token chain", "native_token": "0x...", "parent_chain": "arbitrum-sepolia"}}}' | python -m src.mcp.server

# Example: Ask about Orbit deployment
echo '{"method": "tools/call", "params": {"name": "ask_orbit", "arguments": {"question": "How do I deploy an Orbit chain with a custom gas token?"}}}' | python -m src.mcp.server

# Run all unit tests
pytest tests/ -m "not integration"

# Run retrieval quality tests
pytest tests/test_retrieval.py -v

# Run MCP tool tests (requires tool implementations)
pytest tests/mcp_tools/ -v

# Run template selection and validation tests
pytest tests/test_templates.py -v -m "not integration"

# Run template compilation tests (requires Rust toolchain + cargo-stylus)
pytest tests/test_templates.py -v -m integration

# Run all benchmarks
python scripts/run_benchmarks.py

# Run only P0 (critical) tests
python scripts/run_benchmarks.py --priority P0

# Run benchmarks for a specific tool
python scripts/run_benchmarks.py --tool get_stylus_context

black .
ruff check .

# Verify your .env file has a valid API key
cat .env | grep OPENROUTER_API_KEY

# Run with smaller batch size
python -m src.embeddings.vectordb --batch-size 25

# Or modify max_workers in vectordb.py to 1 for sequential processing

import logging
logging.basicConfig(level=logging.INFO)
# For more verbose output:
# logging.basicConfig(level=logging.DEBUG)

# Check your network connection
ping github.com

# Try cloning manually to diagnose
git clone --depth 1 https://github.com/OffchainLabs/stylus-hello-world

# If behind a proxy, configure git
git config --global http.proxy http://proxy:port

python -m scraper.run --max-concurrent 1

# The vector database must be generated locally (it's not included in the repo)
# Run this command to populate the database:
python -m src.embeddings.vectordb

# If that doesn't work, try resetting first:
python -m src.embeddings.vectordb --reset

# This is usually a conda environment issue
# Make sure you're in the correct environment
conda activate arbbuilder

# Or reinstall chromadb
pip uninstall chromadb
pip install chromadb

# Reset and re-ingest
python -m src.embeddings.vectordb --reset