dbt MCP Server

#!/bin/bash # Modern Data Stack Pipeline Orchestration Script # Meltano (ELT) → DuckDB → dbt Core (Transform) set -e # Exit on any error echo "🚀 Starting Data Stack Pipeline..." echo "==================================" # Check if virtual environment exists if [ ! -d "venv" ]; then echo "❌ Virtual environment not found. Please run setup first." exit 1 fi # Activate virtual environment source venv/bin/activate # Step 1: Extract and Load with Meltano echo "" echo "📥 Step 1: Extracting and Loading Data with Meltano..." echo "--------------------------------------------------------" meltano run tap-csv target-duckdb if [ $? -eq 0 ]; then echo "✅ Meltano ELT completed successfully" else echo "❌ Meltano ELT failed" exit 1 fi # Step 2: Transform with dbt echo "" echo "🔄 Step 2: Transforming Data with dbt Core..." echo "----------------------------------------------" cd transform DBT_PROFILES_DIR=./profiles/duckdb dbt run --profile data_stack --project-dir . if [ $? -eq 0 ]; then echo "✅ dbt transformations completed successfully" else echo "❌ dbt transformations failed" exit 1 fi # Step 3: Run dbt tests echo "" echo "🧪 Step 3: Running Data Quality Tests..." echo "----------------------------------------" DBT_PROFILES_DIR=./profiles/duckdb dbt test --profile data_stack --project-dir . if [ $? -eq 0 ]; then echo "✅ Data quality tests passed" else echo "⚠️ Some data quality tests failed (check logs)" fi cd .. # Step 4: Generate Pipeline Report echo "" echo "📊 Step 4: Generating Pipeline Report..." echo "----------------------------------------" python scripts/generate_report.py echo "" echo "✅ Data Stack Pipeline Completed Successfully!" echo "==============================================" echo "📊 Check data/warehouse/data_stack.duckdb for results" echo "📄 Check output/ directory for reports"