"""
Docker deployment and scaling utilities for MCP system
Provides containerization, orchestration, and scaling capabilities
"""
import os
import json
import logging
import subprocess
from typing import Dict, List, Any, Optional
from pathlib import Path
from dataclasses import dataclass
import yaml
logger = logging.getLogger(__name__)
@dataclass
class ContainerConfig:
"""Container configuration"""
name: str
image: str
ports: Dict[str, str]
environment: Dict[str, str]
volumes: Dict[str, str]
depends_on: List[str] = None
healthcheck: Dict[str, Any] = None
class DockerManager:
"""Docker deployment and management utilities"""
def __init__(self, project_name: str = "mcp-system"):
self.project_name = project_name
self.base_path = Path(__file__).parent
def generate_dockerfile(self, deployment_type: str = "production") -> str:
"""Generate Dockerfile for MCP system"""
if deployment_type == "development":
dockerfile_content = """
# Development Dockerfile for MCP System
FROM python:3.11-slim
WORKDIR /app
# Install system dependencies
RUN apt-get update && apt-get install -y \\
postgresql-client \\
redis-tools \\
curl \\
&& rm -rf /var/lib/apt/lists/*
# Copy requirements and install Python dependencies
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# Install development dependencies
RUN pip install pytest pytest-asyncio pytest-mock
# Copy application code
COPY . .
# Create logs directory
RUN mkdir -p logs
# Expose port
EXPOSE 8000
# Set environment variables
ENV PYTHONPATH=/app
ENV MCP_ENVIRONMENT=development
# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \\
CMD curl -f http://localhost:8000/health || exit 1
# Run application
CMD ["python", "http_bridge.py"]
"""
elif deployment_type == "production":
dockerfile_content = """
# Production Dockerfile for MCP System
FROM python:3.11-slim
WORKDIR /app
# Create non-root user
RUN groupadd -r mcp && useradd --no-log-init -r -g mcp mcp
# Install system dependencies
RUN apt-get update && apt-get install -y \\
postgresql-client \\
redis-tools \\
curl \\
&& rm -rf /var/lib/apt/lists/*
# Copy requirements and install Python dependencies
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# Copy application code
COPY . .
# Create logs directory and set permissions
RUN mkdir -p logs && chown -R mcp:mcp /app
# Switch to non-root user
USER mcp
# Expose port
EXPOSE 8000
# Set environment variables
ENV PYTHONPATH=/app
ENV MCP_ENVIRONMENT=production
# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \\
CMD curl -f http://localhost:8000/health || exit 1
# Run application with gunicorn
CMD ["gunicorn", "http_bridge:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "4"]
"""
else:
raise ValueError(f"Unknown deployment type: {deployment_type}")
return dockerfile_content.strip()
def generate_docker_compose(self, environment: str = "development") -> str:
"""Generate docker-compose.yml for MCP system"""
if environment == "development":
compose_content = {
'version': '3.8',
'services': {
'mcp-app': {
'build': {
'context': '.',
'args': {
'DEPLOYMENT_TYPE': 'development'
}
},
'ports': ['8000:8000'],
'environment': {
'MCP_ENVIRONMENT': 'development',
'DATABASE_URL': 'postgresql://postgres:password@postgres:5432/mcp_dev',
'REDIS_URL': 'redis://redis:6379/0',
'OPENAI_API_KEY': '${OPENAI_API_KEY}',
'SECRET_KEY': '${SECRET_KEY:-dev_secret_key}'
},
'volumes': [
'.:/app',
'mcp_logs:/app/logs'
],
'depends_on': ['postgres', 'redis'],
'restart': 'unless-stopped'
},
'postgres': {
'image': 'pgvector/pgvector:pg15',
'environment': {
'POSTGRES_DB': 'mcp_dev',
'POSTGRES_USER': 'postgres',
'POSTGRES_PASSWORD': 'password'
},
'volumes': [
'postgres_data:/var/lib/postgresql/data'
],
'ports': ['5432:5432'],
'restart': 'unless-stopped'
},
'redis': {
'image': 'redis:7-alpine',
'volumes': [
'redis_data:/data'
],
'ports': ['6379:6379'],
'restart': 'unless-stopped'
},
'nginx': {
'image': 'nginx:alpine',
'ports': ['80:80'],
'volumes': [
'./deployment/nginx.conf:/etc/nginx/nginx.conf'
],
'depends_on': ['mcp-app'],
'restart': 'unless-stopped'
}
},
'volumes': {
'postgres_data': {},
'redis_data': {},
'mcp_logs': {}
}
}
elif environment == "production":
compose_content = {
'version': '3.8',
'services': {
'mcp-app': {
'image': 'mcp-system:latest',
'environment': {
'MCP_ENVIRONMENT': 'production',
'DATABASE_URL': '${DATABASE_URL}',
'REDIS_URL': '${REDIS_URL}',
'OPENAI_API_KEY': '${OPENAI_API_KEY}',
'SECRET_KEY': '${SECRET_KEY}'
},
'volumes': [
'mcp_logs:/app/logs'
],
'depends_on': ['postgres', 'redis'],
'restart': 'unless-stopped',
'deploy': {
'replicas': 3,
'update_config': {
'parallelism': 1,
'delay': '10s'
},
'restart_policy': {
'condition': 'on-failure'
}
}
},
'postgres': {
'image': 'pgvector/pgvector:pg15',
'environment': {
'POSTGRES_DB': '${POSTGRES_DB}',
'POSTGRES_USER': '${POSTGRES_USER}',
'POSTGRES_PASSWORD': '${POSTGRES_PASSWORD}'
},
'volumes': [
'postgres_data:/var/lib/postgresql/data'
],
'restart': 'unless-stopped',
'deploy': {
'replicas': 1,
'placement': {
'constraints': ['node.role == manager']
}
}
},
'redis': {
'image': 'redis:7-alpine',
'volumes': [
'redis_data:/data'
],
'restart': 'unless-stopped',
'command': 'redis-server --appendonly yes'
},
'nginx': {
'image': 'nginx:alpine',
'ports': ['80:80', '443:443'],
'volumes': [
'./deployment/nginx.prod.conf:/etc/nginx/nginx.conf',
'./deployment/ssl:/etc/nginx/ssl'
],
'depends_on': ['mcp-app'],
'restart': 'unless-stopped'
}
},
'volumes': {
'postgres_data': {},
'redis_data': {},
'mcp_logs': {}
},
'networks': {
'mcp_network': {
'driver': 'overlay'
}
}
}
else:
raise ValueError(f"Unknown environment: {environment}")
return yaml.dump(compose_content, default_flow_style=False)
def generate_nginx_config(self, environment: str = "development") -> str:
"""Generate nginx configuration"""
if environment == "development":
nginx_config = """
events {
worker_connections 1024;
}
http {
upstream mcp_backend {
server mcp-app:8000;
}
server {
listen 80;
location / {
proxy_pass http://mcp_backend;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
location /health {
proxy_pass http://mcp_backend/health;
access_log off;
}
}
}
"""
elif environment == "production":
nginx_config = """
events {
worker_connections 1024;
}
http {
# Rate limiting
limit_req_zone $binary_remote_addr zone=api:10m rate=10r/s;
limit_req_zone $binary_remote_addr zone=admin:10m rate=1r/s;
upstream mcp_backend {
server mcp-app:8000 max_fails=3 fail_timeout=30s;
keepalive 32;
}
# SSL settings
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
ssl_session_cache shared:SSL:10m;
# Gzip compression
gzip on;
gzip_types text/plain application/json application/javascript text/css;
server {
listen 80;
return 301 https://$server_name$request_uri;
}
server {
listen 443 ssl http2;
ssl_certificate /etc/nginx/ssl/cert.pem;
ssl_certificate_key /etc/nginx/ssl/key.pem;
# Security headers
add_header X-Frame-Options DENY;
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
location / {
limit_req zone=api burst=20 nodelay;
proxy_pass http://mcp_backend;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# Timeouts
proxy_connect_timeout 60s;
proxy_send_timeout 60s;
proxy_read_timeout 60s;
}
location /api/admin/ {
limit_req zone=admin burst=5 nodelay;
proxy_pass http://mcp_backend;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
location /health {
proxy_pass http://mcp_backend/health;
access_log off;
}
}
}
"""
else:
raise ValueError(f"Unknown environment: {environment}")
return nginx_config.strip()
def generate_kubernetes_manifests(self) -> Dict[str, str]:
"""Generate Kubernetes deployment manifests"""
manifests = {}
# Namespace
manifests['namespace.yaml'] = """
apiVersion: v1
kind: Namespace
metadata:
name: mcp-system
"""
# ConfigMap
manifests['configmap.yaml'] = """
apiVersion: v1
kind: ConfigMap
metadata:
name: mcp-config
namespace: mcp-system
data:
MCP_ENVIRONMENT: "production"
DATABASE_HOST: "postgres-service"
REDIS_URL: "redis://redis-service:6379/0"
"""
# Secret
manifests['secret.yaml'] = """
apiVersion: v1
kind: Secret
metadata:
name: mcp-secrets
namespace: mcp-system
type: Opaque
data:
DATABASE_PASSWORD: <base64-encoded-password>
OPENAI_API_KEY: <base64-encoded-api-key>
SECRET_KEY: <base64-encoded-secret-key>
"""
# PostgreSQL Deployment
manifests['postgres.yaml'] = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: postgres
namespace: mcp-system
spec:
replicas: 1
selector:
matchLabels:
app: postgres
template:
metadata:
labels:
app: postgres
spec:
containers:
- name: postgres
image: pgvector/pgvector:pg15
env:
- name: POSTGRES_DB
value: "mcp_prod"
- name: POSTGRES_USER
value: "postgres"
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: mcp-secrets
key: DATABASE_PASSWORD
ports:
- containerPort: 5432
volumeMounts:
- name: postgres-storage
mountPath: /var/lib/postgresql/data
volumes:
- name: postgres-storage
persistentVolumeClaim:
claimName: postgres-pvc
---
apiVersion: v1
kind: Service
metadata:
name: postgres-service
namespace: mcp-system
spec:
selector:
app: postgres
ports:
- port: 5432
targetPort: 5432
"""
# Redis Deployment
manifests['redis.yaml'] = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: redis
namespace: mcp-system
spec:
replicas: 1
selector:
matchLabels:
app: redis
template:
metadata:
labels:
app: redis
spec:
containers:
- name: redis
image: redis:7-alpine
ports:
- containerPort: 6379
volumeMounts:
- name: redis-storage
mountPath: /data
volumes:
- name: redis-storage
persistentVolumeClaim:
claimName: redis-pvc
---
apiVersion: v1
kind: Service
metadata:
name: redis-service
namespace: mcp-system
spec:
selector:
app: redis
ports:
- port: 6379
targetPort: 6379
"""
# MCP App Deployment
manifests['mcp-app.yaml'] = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: mcp-app
namespace: mcp-system
spec:
replicas: 3
selector:
matchLabels:
app: mcp-app
template:
metadata:
labels:
app: mcp-app
spec:
containers:
- name: mcp-app
image: mcp-system:latest
env:
- name: MCP_ENVIRONMENT
valueFrom:
configMapKeyRef:
name: mcp-config
key: MCP_ENVIRONMENT
- name: DATABASE_URL
value: "postgresql://postgres:$(DATABASE_PASSWORD)@postgres-service:5432/mcp_prod"
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: mcp-secrets
key: DATABASE_PASSWORD
- name: OPENAI_API_KEY
valueFrom:
secretKeyRef:
name: mcp-secrets
key: OPENAI_API_KEY
- name: SECRET_KEY
valueFrom:
secretKeyRef:
name: mcp-secrets
key: SECRET_KEY
ports:
- containerPort: 8000
livenessProbe:
httpGet:
path: /health
port: 8000
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
httpGet:
path: /health
port: 8000
initialDelaySeconds: 5
periodSeconds: 5
resources:
requests:
memory: "256Mi"
cpu: "250m"
limits:
memory: "512Mi"
cpu: "500m"
---
apiVersion: v1
kind: Service
metadata:
name: mcp-app-service
namespace: mcp-system
spec:
selector:
app: mcp-app
ports:
- port: 8000
targetPort: 8000
type: ClusterIP
"""
# Ingress
manifests['ingress.yaml'] = """
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: mcp-ingress
namespace: mcp-system
annotations:
nginx.ingress.kubernetes.io/rate-limit: "100"
nginx.ingress.kubernetes.io/ssl-redirect: "true"
spec:
tls:
- hosts:
- mcp.example.com
secretName: mcp-tls
rules:
- host: mcp.example.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: mcp-app-service
port:
number: 8000
"""
return manifests
def build_image(self, tag: str = "latest", deployment_type: str = "production") -> bool:
"""Build Docker image"""
try:
# Generate Dockerfile
dockerfile_content = self.generate_dockerfile(deployment_type)
# Write Dockerfile
dockerfile_path = self.base_path / "Dockerfile"
with open(dockerfile_path, 'w') as f:
f.write(dockerfile_content)
# Build image
cmd = [
"docker", "build",
"-t", f"mcp-system:{tag}",
"--build-arg", f"DEPLOYMENT_TYPE={deployment_type}",
"."
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode == 0:
logger.info(f"Successfully built image mcp-system:{tag}")
return True
else:
logger.error(f"Failed to build image: {result.stderr}")
return False
except Exception as e:
logger.error(f"Failed to build Docker image: {e}")
return False
def deploy_with_docker_compose(self, environment: str = "development") -> bool:
"""Deploy using docker-compose"""
try:
# Generate docker-compose.yml
compose_content = self.generate_docker_compose(environment)
compose_path = self.base_path / f"docker-compose.{environment}.yml"
with open(compose_path, 'w') as f:
f.write(compose_content)
# Generate nginx config
nginx_content = self.generate_nginx_config(environment)
nginx_dir = self.base_path / "deployment"
nginx_dir.mkdir(exist_ok=True)
nginx_file = "nginx.conf" if environment == "development" else "nginx.prod.conf"
with open(nginx_dir / nginx_file, 'w') as f:
f.write(nginx_content)
# Deploy with docker-compose
cmd = [
"docker-compose",
"-f", str(compose_path),
"up", "-d"
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode == 0:
logger.info(f"Successfully deployed with docker-compose ({environment})")
return True
else:
logger.error(f"Failed to deploy: {result.stderr}")
return False
except Exception as e:
logger.error(f"Failed to deploy with docker-compose: {e}")
return False
def scale_service(self, service_name: str, replicas: int) -> bool:
"""Scale a service"""
try:
cmd = [
"docker-compose",
"scale",
f"{service_name}={replicas}"
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode == 0:
logger.info(f"Successfully scaled {service_name} to {replicas} replicas")
return True
else:
logger.error(f"Failed to scale service: {result.stderr}")
return False
except Exception as e:
logger.error(f"Failed to scale service: {e}")
return False
def get_deployment_status(self) -> Dict[str, Any]:
"""Get deployment status"""
try:
# Get container status
cmd = ["docker-compose", "ps", "--format", "json"]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode == 0:
containers = []
for line in result.stdout.strip().split('\n'):
if line:
containers.append(json.loads(line))
return {
'success': True,
'containers': containers,
'total_containers': len(containers),
'running_containers': len([c for c in containers if c.get('State') == 'running'])
}
else:
return {
'success': False,
'error': result.stderr
}
except Exception as e:
logger.error(f"Failed to get deployment status: {e}")
return {
'success': False,
'error': str(e)
}
# CLI interface
def main():
"""Command-line interface for deployment manager"""
import argparse
parser = argparse.ArgumentParser(description="MCP System Deployment Manager")
parser.add_argument("action", choices=[
"build", "deploy", "scale", "status", "generate-k8s"
])
parser.add_argument("--environment", default="development",
choices=["development", "production"])
parser.add_argument("--tag", default="latest")
parser.add_argument("--service")
parser.add_argument("--replicas", type=int, default=1)
args = parser.parse_args()
manager = DockerManager()
if args.action == "build":
success = manager.build_image(args.tag, args.environment)
exit(0 if success else 1)
elif args.action == "deploy":
success = manager.deploy_with_docker_compose(args.environment)
exit(0 if success else 1)
elif args.action == "scale":
if not args.service:
print("--service required for scale action")
exit(1)
success = manager.scale_service(args.service, args.replicas)
exit(0 if success else 1)
elif args.action == "status":
status = manager.get_deployment_status()
print(json.dumps(status, indent=2))
elif args.action == "generate-k8s":
manifests = manager.generate_kubernetes_manifests()
k8s_dir = Path("k8s")
k8s_dir.mkdir(exist_ok=True)
for filename, content in manifests.items():
with open(k8s_dir / filename, 'w') as f:
f.write(content.strip())
print(f"Generated Kubernetes manifests in {k8s_dir}/")
if __name__ == "__main__":
main()
