# Performance Optimizations Applied:
# 1. Enhanced caching strategy with cache reuse between jobs
# 2. Comprehensive nox session usage for all tools (lint, test, build, security, cli_test)
# 3. Cross-platform tests only on Mac/Windows (Linux already covered)
# 4. Conditional coverage reporting (only on main branch pushes)
# 5. Reduced timeouts and sleep times for faster feedback
# 6. Bytecode caching for faster Python imports
# 7. Platform-specific cache keys for better hit rates
# 8. Parallel execution for analysis tasks with robust error handling
# 9. Use UV's dev dependencies instead of separate nox installation
# 10. Standardized tool execution with noxfile.py for reproducibility and isolation
name: CI/CD Pipeline
on:
push:
branches: [ "main", "master" ]
paths-ignore:
- 'docs/**'
- '*.md'
- 'README.md'
- 'CONTRIBUTING.md'
- 'LICENSE'
- '.gitignore'
pull_request:
branches: [ "main", "master" ]
paths-ignore:
- 'docs/**'
- '*.md'
- 'README.md'
- 'CONTRIBUTING.md'
- 'LICENSE'
- '.gitignore'
workflow_dispatch:
permissions:
contents: read
env:
PYTHON_VERSION: "3.13"
jobs:
# Fast feedback loop using UV (matches development workflow)
fast-checks:
name: Fast Checks (UV)
runs-on: ubuntu-latest
outputs:
cache-key: ${{ steps.cache-info.outputs.cache-key }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Generate cache info
id: cache-info
run: |
echo "cache-key=uv-${{ runner.os }}-${{ hashFiles('pyproject.toml', 'uv.lock') }}" >> $GITHUB_OUTPUT
- name: Install uv
uses: astral-sh/setup-uv@v6
with:
version: "latest"
enable-cache: true
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
# Enhanced UV cache with better key strategy
- name: Cache UV dependencies
uses: actions/cache@v4
with:
path: |
~/.cache/uv
.venv
key: ${{ steps.cache-info.outputs.cache-key }}
restore-keys: |
uv-${{ runner.os }}-
- name: Install dependencies with cache
run: |
uv sync --extra dev
# More specific nox cache with session-level caching
- name: Cache nox environments
uses: actions/cache@v4
with:
path: .nox
key: nox-${{ runner.os }}-fast-${{ hashFiles('pyproject.toml', 'uv.lock', 'noxfile.py') }}
restore-keys: |
nox-${{ runner.os }}-fast-
nox-${{ runner.os }}-
# Cache Python bytecode to speed up subsequent runs
- name: Cache Python bytecode
uses: actions/cache@v4
with:
path: |
src/**/__pycache__
tests/**/__pycache__
key: pycache-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ hashFiles('src/**/*.py', 'tests/**/*.py') }}
- name: Lint with Ruff (using nox)
run: |
# Use the standardized nox lint session
uv run nox -s lint
- name: Fast tests with optimizations
run: |
# Nox is already installed via dev dependencies
# Use pytest's parallel execution and fail-fast for speed
uv run nox -s test_fast -- --numprocesses=auto --maxfail=3
- name: System verification (cached)
run: |
uv run nox -s verify_system
# Comprehensive testing across Python versions using nox (optimized)
comprehensive-tests:
name: Python ${{ matrix.python-version }}
runs-on: ubuntu-latest
needs: [fast-checks]
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13"] # Reduced from 4 to 3 versions for performance
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
# Reuse cache from fast-checks job
- name: Cache UV dependencies (reuse)
uses: actions/cache@v4
with:
path: |
~/.cache/uv
.venv
key: ${{ needs.fast-checks.outputs.cache-key }}
restore-keys: |
uv-${{ runner.os }}-
# Version-specific nox cache
- name: Cache nox environments
uses: actions/cache@v4
with:
path: .nox
key: nox-${{ runner.os }}-${{ matrix.python-version }}-comprehensive-${{ hashFiles('pyproject.toml', 'uv.lock', 'noxfile.py') }}
restore-keys: |
nox-${{ runner.os }}-${{ matrix.python-version }}-
nox-${{ runner.os }}-
# Cache pip packages to avoid repeated downloads
- name: Cache pip packages
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: pip-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
- name: Install nox (optimized)
run: |
python -m pip install --upgrade pip
pip install nox
- name: Run comprehensive tests (optimized)
run: |
# Use parallel execution where possible
nox --session "test-${{ matrix.python-version }}" -- --numprocesses=auto
# Integration testing for CLI and real workflows
integration-tests:
name: Integration Tests
runs-on: ubuntu-latest
needs: [fast-checks]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v6
with:
version: "latest"
enable-cache: true
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
# Reuse dependencies cache
- name: Cache UV dependencies (reuse)
uses: actions/cache@v4
with:
path: |
~/.cache/uv
.venv
key: ${{ needs.fast-checks.outputs.cache-key }}
restore-keys: |
uv-${{ runner.os }}-
- name: Install dependencies (cached)
run: |
uv sync --extra dev
- name: Cache nox environments
uses: actions/cache@v4
with:
path: .nox
key: nox-${{ runner.os }}-integration-${{ hashFiles('pyproject.toml', 'uv.lock', 'noxfile.py') }}
restore-keys: |
nox-${{ runner.os }}-integration-
nox-${{ runner.os }}-
- name: Test CLI commands (using nox)
run: |
# Use the standardized nox cli_test session
uv run nox -s cli_test
# Test orchestrator workflow (reduced timeout for faster feedback)
timeout 15s uv run lightfast-mcp-orchestrator start mock-server --verbose --hide-logs &
ORCHESTRATOR_PID=$!
sleep 3 # Reduced from 5 seconds
kill $ORCHESTRATOR_PID || true
- name: Test development workflows (using nox)
run: |
# Test taskipy integration (still use tasks for this)
uv run task lint
uv run task check_format
# Test nox demo session
uv run nox -s demo
- name: Integration test suite (optimized)
run: |
# Nox is already installed via dev dependencies
uv run nox -s test_integration -- --numprocesses=auto
- name: Coverage report (conditional)
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
run: |
uv run nox -s test_coverage
uv run coverage xml
- name: Upload coverage reports
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
uses: codecov/codecov-action@v5
with:
files: ./coverage.xml
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false
# Combined static analysis and security checks
static-analysis:
name: Static Analysis & Security
runs-on: ubuntu-latest
needs: [fast-checks]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
# Reuse dependencies cache
- name: Cache UV dependencies (reuse)
uses: actions/cache@v4
with:
path: |
~/.cache/uv
.venv
key: ${{ needs.fast-checks.outputs.cache-key }}
restore-keys: |
uv-${{ runner.os }}-
- name: Cache nox environments
uses: actions/cache@v4
with:
path: .nox
key: nox-${{ runner.os }}-static-${{ hashFiles('pyproject.toml', 'uv.lock', 'noxfile.py') }}
restore-keys: |
nox-${{ runner.os }}-static-
nox-${{ runner.os }}-
# Cache nox installation
- name: Cache nox installation
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: nox-install-${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
- name: Install nox
run: |
python -m pip install --upgrade pip
pip install nox
- name: Run analysis in parallel (using nox)
run: |
# Run type checking and security scanning using nox sessions
nox -s typecheck &
TYPECHECK_PID=$!
nox -s security &
SECURITY_PID=$!
# Wait for both processes and capture their exit codes
wait $TYPECHECK_PID
TYPECHECK_EXIT=$?
wait $SECURITY_PID
SECURITY_EXIT=$?
# Report results
echo "Type checking exit code: $TYPECHECK_EXIT"
echo "Security scan exit code: $SECURITY_EXIT"
# Only fail if typecheck fails (security warnings are acceptable)
if [ $TYPECHECK_EXIT -ne 0 ]; then
echo "Type checking failed"
exit $TYPECHECK_EXIT
fi
echo "Analysis completed successfully"
- name: Upload security report
uses: actions/upload-artifact@v4
if: always()
with:
name: security-report
path: bandit-report.json
# End-to-end testing for full system workflows (optimized)
e2e-tests:
name: End-to-End Tests
runs-on: ubuntu-latest
needs: [fast-checks, integration-tests]
timeout-minutes: 8 # Reduced from 10 minutes
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v6
with:
version: "latest"
enable-cache: true
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
# Reuse dependencies cache
- name: Cache UV dependencies (reuse)
uses: actions/cache@v4
with:
path: |
~/.cache/uv
.venv
key: ${{ needs.fast-checks.outputs.cache-key }}
restore-keys: |
uv-${{ runner.os }}-
- name: Install dependencies (cached)
run: |
uv sync --extra dev
- name: Cache nox environments
uses: actions/cache@v4
with:
path: .nox
key: nox-${{ runner.os }}-e2e-${{ hashFiles('pyproject.toml', 'uv.lock', 'noxfile.py') }}
restore-keys: |
nox-${{ runner.os }}-e2e-
nox-${{ runner.os }}-
- name: Run E2E tests (optimized)
run: |
# Nox is already installed via dev dependencies
uv run nox -s test_e2e -- --numprocesses=auto --maxfail=1
- name: Upload E2E test results
uses: actions/upload-artifact@v4
if: always()
with:
name: e2e-test-results
path: |
pytest-results.xml
e2e-logs/
# Package building and verification (optimized)
build-and-verify:
name: Build & Verify Package
runs-on: ubuntu-latest
needs: [fast-checks] # Removed comprehensive-tests dependency for performance
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v6
with:
version: "latest"
enable-cache: true
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
# Cache build tools
- name: Cache build tools
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: build-tools-${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
- name: Install nox for build
run: |
python -m pip install --upgrade pip
pip install nox
- name: Build and verify package (using nox)
run: |
# Use the standardized nox build session
nox -s build
- name: Test package installation
run: |
# Test package installation
uv pip install dist/*.whl --system
# Test CLI script works
lightfast-mcp-orchestrator --help
python -c "import lightfast_mcp; print('Package imported successfully')"
- name: Upload build artifacts
uses: actions/upload-artifact@v4
with:
name: python-package-distributions
path: dist/
# Cross-platform tests (Mac and Windows only - Linux already tested)
cross-platform-tests:
name: Cross Platform Tests (${{ matrix.os }})
runs-on: ${{ matrix.os }}
needs: [fast-checks]
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
strategy:
fail-fast: false
matrix:
os: [macos-latest, windows-latest] # Only Mac and Windows - Linux already covered
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v6
with:
version: "latest"
enable-cache: true
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
# Platform-specific UV cache
- name: Cache UV dependencies
uses: actions/cache@v4
with:
path: |
~/.cache/uv
.venv
key: uv-${{ runner.os }}-cross-platform-${{ hashFiles('pyproject.toml', 'uv.lock') }}
restore-keys: |
uv-${{ runner.os }}-
- name: Install dependencies with cache
run: |
uv sync --extra dev
# Platform-specific nox cache
- name: Cache nox environments
uses: actions/cache@v4
with:
path: .nox
key: nox-${{ runner.os }}-cross-platform-${{ hashFiles('pyproject.toml', 'uv.lock', 'noxfile.py') }}
restore-keys: |
nox-${{ runner.os }}-
- name: Run core tests (optimized for platform)
run: |
# Nox is already installed via dev dependencies
# Run system verification with platform-specific optimizations
uv run nox -s verify_system
- name: Test CLI on ${{ matrix.os }} (using nox)
shell: bash
run: |
# Use the standardized nox cli_test session for cross-platform verification
uv run nox -s cli_test
# Quick smoke test (reduced from full workflow test)
echo "✅ CLI commands work on ${{ matrix.os }}"
