git clone git@github.com:mcity/mcity_MSight_agent.git
cd mcity_MSight_agent

# Use python3.12 explicitly -- plain `python3` may resolve to a newer default
# (3.13/3.14) on your system, which several pinned packages (e.g. numba==0.61.0)
# do not yet support.
python3.12 -m venv .venv
source .venv/bin/activate
pip install --upgrade pip
pip install -r requirements.txt
pip install -r agent_requirements.txt

# fiftyone requires sse-starlette<1, but mcp (via fastmcp) requires >=1.6.1 --
# no version satisfies both, so fiftyone is installed separately with --no-deps
# rather than forcing the wrong one onto the rest of the stack.
pip install "fiftyone==1.4.1" --no-deps

wandb login
hf auth login

# Choose your LLM provider (openai, claude, gemini, or groq)
LLM_PROVIDER=openai

# Add your API key
OPENAI_API_KEY=sk-...
# OR
ANTHROPIC_API_KEY=sk-ant-...
# OR
GEMINI_API_KEY=...
# OR
GROQ_API_KEY=...

# Required only for the MSight Pipeline workflow — path to an independent
# MSight_Vision checkout (see "MSight Pipeline Workflow" above).
MSIGHT_VISION_PATH=/path/to/your/MSight_Vision

# Optional — enables a one-click "Demo" source in the MSight Pipeline workflow
# without asking the user for a video path.
MSIGHT_DEMO_VIDEO_PATH=/path/to/a/demo/video/or/folder

# Terminal 1: MCP Tool Server (port 8000)
python mcp_layer/mcp_server.py

# Terminal 2: Chat Server (port 8001)
uvicorn mcp_layer.chat_server:app --port 8001 --reload

# Terminal 3: Data Ingestion Server (port 8002)
uvicorn mcp_layer.ingest_server:app --host 0.0.0.0 --port 8002 --reload

# Terminal 4: Launch Web UI (port 5225)
python mcp_layer/client_chat.py --ui

User: Hi, I would like to explore the data engine for processing my data.

Agent: Great! I can help you with Auto Labeling — training or running an object
       detection model on your dataset. Which dataset would you like to use?

User: fisheye8k

Agent: Dataset 'fisheye8k' confirmed. Do you have credentials configured for
       CVAT or Label Studio? I detected Label Studio credentials — using
       Label Studio for annotation. How would you like to annotate your
       dataset?
          - Manual Labeling: export your dataset for manual annotation.
          - Auto Generated Labeling: run a detection model to generate
            predictions automatically, then review/correct them.

User: Auto Generated Labeling.

Agent: Here are the available model sources and models: [...]
       Which model would you like to use?

User: rfdetr_2xlarge

Agent: rfdetr_2xlarge (roboflow) has been configured.
       Here are the current hyperparameters: [...]
       Would you like to modify any of these hyperparameters, or are you
       ready to start?

User: Let's go with the defaults.

Agent: Here's a summary of what will be run: [...] Shall I proceed with
       auto-labeling?

User: Yes, proceed.

[Data Engine Server]: Running auto-labeling — live logs are streamed here and to Weights and Biases.

Agent: Auto-labeling completed. Predictions have been exported to Label Studio
       for review. Let me know when you're done and I'll import your labels
       back, or I can launch Voxel51 so you can explore the results now.

User: I want to run the MSight camera detection pipeline.

Agent: Welcome to MSight Agent! Would you like to:
          - Demo: see it running immediately with a test video.
          - Run your own pipeline: set up your own camera calibration,
            video/RTSP source, recording & archiving, and optionally
            auto-labeling — in any order.

User: I want to run my own pipeline.

Agent: Here's your checklist — do these in any order, come back anytime:
          1. Camera calibration (currently: default demo calibration)
          2. Video/RTSP source (currently: not set)
          3. Record & Archive (available once a real source is running)
          4. Auto Labeling (optional, if you don't have a labeled dataset yet)
       Which would you like to start with?

User: Use this video file: /data/gs_mcity_1/2026-07-01.mp4

Agent: Here's what will run:
          - Source: /data/gs_mcity_1/2026-07-01.mp4
          - Calibration: default demo calibration (no user upload yet)
          - Recording: off · Archiving: off
       Shall I start the pipeline with these settings?

User: Yes, go ahead.

Agent: MSight_Vision pipeline started. Watch live detections at
       http://localhost:9010. You can also check status, view logs,
       stop it, or restart with a different input — just ask.

User: How's it doing?

Agent: All 4 containers are running (redis, video_source, rfdetr_detector,
       detection_viewer). No errors in the last 200 log lines.

[filter "strip-notebook-output-engine"]
    clean = <your_path>/mcity_MSight_agent/.venv/bin/jupyter nbconvert --ClearOutputPreprocessor.enabled=True --ClearMetadataPreprocessor.enabled=True --to=notebook --stdin --stdout
    smudge = cat
    required = true

[filter "strip-notebook-output-scripts"]
    clean = <your_path>/mcity_MSight_agent/.venv/bin/jupyter nbconvert --ClearOutputPreprocessor.enabled=True --ClearMetadataPreprocessor.enabled=True --to=notebook --stdin --stdout
    smudge = cat
    required = true

git submodule update --recursive --remote
git add .gitmodules $(git submodule foreach --quiet 'echo $name')

.
├── main.py                     # Entry point of the framework → Terminal 1
├── session_v51.py              # Script to launch Voxel51 session → Terminal 2
├── workflows/                  # Workflows for the Mcity Data Engine
├── config/                     # Local configuration files
├── utils/                      # General-purpose utility functions
├── cloud/                      # Scripts run in the cloud to pre-process data
├── docs/                       # Documentation generated with `pdoc`
├── tests/                      # Tests using Pytest
├── custom_models/              # External models with containerized environments
├── MSight/                      # Offline dataset geolocation (not the live pipeline — see MSight/README.md)
│  ├── localize_dataset.py      # Fisheye ground-contact + lat/lon localization
│  ├── utils/
│  │   ├── fiftyone_to_msight_det.py  # FiftyOne detections -> MSight detection objects
│  │   └── load_locamaps.py     # Calibration NPZ/intrinsics loading + pixel localizer
│  ├── data/                    # Hardcoded single-camera (Ashley/Huron) calibration files
│  └── install.sh               # Installs msight_base/msight_core into the active venv
├── mcp_layer/                   # Agentic (MCP) layer
│  ├── mcp_server.py            # MCP tool registry (port 8000)
│  ├── chat_server.py           # FastAPI chat endpoint (port 8001) + /msight/upload_calibration
│  ├── ingest_server.py         # File upload & processing (port 8002)
│  ├── client_chat.py           # Web/terminal client (port 5225)
│  ├── chat_pipeline.py         # Tool dispatch, state routing, reply formatting
│  ├── pipeline_handlers/       # Per-workflow handler mixins for ChatPipeline
│  │   ├── auto_labeling.py
│  │   └── msight_pipeline.py
│  ├── pipeline_common.py       # Shared routing/sentinel primitives for the handlers above
│  ├── progress_relay.py        # Relays live docker-compose build output to the active /chat/stream
│  ├── validate_workflow_state.py # Pydantic workflow state machine (both workflows)
│  ├── host_utils.py            # AWS IMDSv2 host/IP resolution for the viewer URL
│  ├── mcptools/                # Tool implementations
│  │   ├── __init__.py
│  │   ├── workflow_selector.py
│  │   ├── auto_labeling.py
│  │   ├── data_ingest.py
│  │   ├── cvat_export.py
│  │   ├── label_studio_export.py
│  │   ├── msight_docker.py     # start/stop/status/logs — docker compose control for MSight_Vision
│  │   ├── msight_record_archive.py # Local recording + S3 archiving as tracked host subprocesses
│  │   ├── msight_calibration_helper.py # Fisheye-intrinsics auto-detect (not yet registered)
│  │   └── v51.py               # Voxel51 integration
│  ├── msight_nodes/            # Standalone MSight nodes (not MCP tools) launched as
│  │   │                        # subprocesses via MSight_Vision's own venv interpreter
│  │   └── annotated_frame_publisher.py # Draws detection boxes, republishes for Record & Archive
│  ├── llm_clients.py           # Multi-LLM support (OpenAI, Claude, Gemini, Groq)
│  ├── tool_schema.py           # Tool definitions exposed to the LLM
│  ├── prompts/                 # System + per-workflow prompt text
│  │   ├── state_hints.txt      # Per-turn SESSION_STATE hint fragments
│  │   └── workflows/
│  │       ├── auto_labeling.txt
│  │       └── msight_pipeline.txt
│  ├── tests/                   # Pytest coverage for state routing/tool filtering
│  └── ui/                      # Web interface assets
│      └── index.html
├── agent_deployment/            # AWS CloudFormation / Docker launcher for the agent
├── mcity_data_engine_scripts/   # Experiment scripts and one-time operations (Mcity internal)
├── .vscode                     # Settings for VS Code IDE
├── .github/workflows/          # GitHub Action workflows
├── .gitignore                  # Files and directories to be ignored by Git
├── .gitattributes              # Rules for handling files like Notebooks during commits
├── .gitmodules                 # Configuration for managing Git submodules
├── .secret                     # Secret tokens (not tracked by Git)
└── requirements.txt            # Python dependencies (pip install -r requirements.txt)

echo 'export WANDB_DIR="<your_path>/mcity_MSight_agent/logs"' >> ~/.profile
source ~/.profile

@article{bogdoll2025mcitydataengine,
  title={Mcity Data Engine},
  author={Bogdoll, Daniel and Anata, Rajanikant Patnaik and Stevens, Gregory},
  journal={GitHub. Note: https://github.com/mcity/mcity_data_engine},
  year={2025}
}