#!/bin/bash
# Script to create a Lambda layer with all Python dependencies
# Usage: ./create_dependencies_layer.sh [python-version] [layer-name] [region] [requirements-file]
# Example: ./create_dependencies_layer.sh 3.11 mcp-dependencies-python311 us-east-1 ../requirements.txt
set -e
PYTHON_VERSION=${1:-3.11}
LAYER_NAME=${2:-mcp-dependencies-python${PYTHON_VERSION//./}}
REGION=${3:-us-east-1}
REQUIREMENTS_FILE=${4:-../requirements.txt}
TEMP_DIR=$(mktemp -d)
echo "Creating Python dependencies Lambda layer..."
echo "Python version: $PYTHON_VERSION"
echo "Layer name: $LAYER_NAME"
echo "Region: $REGION"
echo "Requirements file: $REQUIREMENTS_FILE"
echo "Temp directory: $TEMP_DIR"
# Check if requirements file exists
if [ ! -f "$REQUIREMENTS_FILE" ]; then
echo "Error: Requirements file not found: $REQUIREMENTS_FILE"
exit 1
fi
# Create directory structure for Lambda layer
mkdir -p "$TEMP_DIR/python/lib/python${PYTHON_VERSION}/site-packages"
# Filter out dependencies that are already in Lambda runtime or not needed
# boto3 and botocore are already in Lambda runtime
# Testing dependencies (pytest*) are not needed in Lambda
echo "Installing dependencies (excluding boto3, botocore, and test dependencies)..."
# Create a filtered requirements file
FILTERED_REQUIREMENTS="$TEMP_DIR/filtered_requirements.txt"
grep -v "^#" "$REQUIREMENTS_FILE" | \
grep -v "^$" | \
grep -v "boto3" | \
grep -v "botocore" | \
grep -v "pytest" > "$FILTERED_REQUIREMENTS" || true
# Install dependencies using Docker to ensure correct platform (Linux x86_64 for Lambda)
if [ -s "$FILTERED_REQUIREMENTS" ]; then
echo "Installing dependencies in Docker container (Lambda-compatible Linux x86_64)..."
# Check if Docker is available
if command -v docker &> /dev/null; then
echo "Using Docker to build Lambda-compatible layer..."
# Create Dockerfile for building the layer
# FILTERED_REQUIREMENTS is already in TEMP_DIR, so we can reference it directly
# IMPORTANT: Use --platform linux/amd64 to build for x86_64 (Lambda's architecture)
cat > "$TEMP_DIR/Dockerfile" <<EOF
FROM --platform=linux/amd64 python:${PYTHON_VERSION}-slim
WORKDIR /build
# Copy requirements (file is already in temp dir)
COPY filtered_requirements.txt .
# Install dependencies to the Lambda Python path structure
# Build for x86_64 architecture (Lambda uses amd64)
RUN mkdir -p python/lib/python${PYTHON_VERSION}/site-packages && \
pip install --no-cache-dir -r filtered_requirements.txt -t python/lib/python${PYTHON_VERSION}/site-packages/ && \
find python -type d -exec chmod 755 {} \; && \
find python -type f -exec chmod 644 {} \;
EOF
# Build in Docker (FILTERED_REQUIREMENTS is already in TEMP_DIR, no need to copy)
cd "$TEMP_DIR"
echo "Building Docker image for linux/amd64 (x86_64) architecture (this may take a few minutes)..."
if docker build --platform linux/amd64 -t lambda-layer-builder:latest . > /tmp/docker-build.log 2>&1; then
# Extract the python directory from Docker container
echo "Extracting layer contents..."
CONTAINER_ID=$(docker create lambda-layer-builder:latest 2>/dev/null)
if [ -n "$CONTAINER_ID" ]; then
# Remove existing python dir if it exists
rm -rf "$TEMP_DIR/python"
docker cp "$CONTAINER_ID:/build/python" "$TEMP_DIR/" 2>&1
docker rm "$CONTAINER_ID" > /dev/null 2>&1
# Verify pydantic_core was installed with correct architecture
if [ -d "$TEMP_DIR/python/lib/python${PYTHON_VERSION}/site-packages/pydantic_core" ]; then
echo "✓ pydantic_core found in layer"
# Check for compiled extension - must be x86_64 (amd64), not aarch64
X86_64_SO=$(find "$TEMP_DIR/python/lib/python${PYTHON_VERSION}/site-packages/pydantic_core" -name "*x86_64*.so" -o -name "*amd64*.so" | head -1)
if [ -n "$X86_64_SO" ]; then
echo "✓ pydantic_core x86_64 compiled extension found: $(basename "$X86_64_SO")"
else
# Check if it's the wrong architecture
WRONG_ARCH=$(find "$TEMP_DIR/python/lib/python${PYTHON_VERSION}/site-packages/pydantic_core" -name "*aarch64*.so" | head -1)
if [ -n "$WRONG_ARCH" ]; then
echo "✗ ERROR: Found ARM64 (aarch64) extension instead of x86_64!"
echo " Found: $(basename "$WRONG_ARCH")"
echo " Lambda requires x86_64 (amd64) architecture"
exit 1
else
echo "⚠ WARNING: pydantic_core compiled extension (.so file) not found!"
fi
fi
else
echo "⚠ WARNING: pydantic_core directory not found in layer!"
fi
echo "Docker build completed successfully"
else
echo "Error: Failed to create Docker container"
exit 1
fi
else
echo "Error: Docker build failed. Check /tmp/docker-build.log for details"
tail -20 /tmp/docker-build.log
exit 1
fi
else
echo "Docker not available, installing directly (may have platform compatibility issues)..."
echo "WARNING: For Lambda, dependencies should be built in Docker for correct platform"
# Install directly (may not work for compiled extensions)
pip install -r "$FILTERED_REQUIREMENTS" \
-t "$TEMP_DIR/python/lib/python${PYTHON_VERSION}/site-packages/" \
--platform manylinux2014_x86_64 \
--only-binary=:all: \
--implementation cp \
--python-version ${PYTHON_VERSION} \
--no-deps || pip3 install -r "$FILTERED_REQUIREMENTS" \
-t "$TEMP_DIR/python/lib/python${PYTHON_VERSION}/site-packages/" \
--platform manylinux2014_x86_64 \
--only-binary=:all: \
--implementation cp \
--python-version ${PYTHON_VERSION} \
--no-deps
# Install with dependencies
pip install -r "$FILTERED_REQUIREMENTS" \
-t "$TEMP_DIR/python/lib/python${PYTHON_VERSION}/site-packages/" \
--platform manylinux2014_x86_64 \
--only-binary=:all: || pip3 install -r "$FILTERED_REQUIREMENTS" \
-t "$TEMP_DIR/python/lib/python${PYTHON_VERSION}/site-packages/" \
--platform manylinux2014_x86_64 \
--only-binary=:all:
fi
else
echo "Warning: No dependencies to install after filtering"
fi
# Remove unnecessary files to reduce layer size
# NOTE: Keep .dist-info directories - they contain package metadata required by Python
echo "Cleaning up unnecessary files..."
find "$TEMP_DIR/python" -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
find "$TEMP_DIR/python" -type f -name "*.pyc" -delete 2>/dev/null || true
find "$TEMP_DIR/python" -type f -name "*.pyo" -delete 2>/dev/null || true
# DO NOT remove .dist-info - it's required for package metadata
# find "$TEMP_DIR/python" -type d -name "*.dist-info" -exec rm -rf {} + 2>/dev/null || true
find "$TEMP_DIR/python" -type d -name "tests" -exec rm -rf {} + 2>/dev/null || true
find "$TEMP_DIR/python" -type d -name "test" -exec rm -rf {} + 2>/dev/null || true
find "$TEMP_DIR/python" -type d -name "__tests__" -exec rm -rf {} + 2>/dev/null || true
# Create zip file
cd "$TEMP_DIR"
ZIP_FILE="dependencies-layer.zip"
echo "Creating zip file..."
zip -r "$ZIP_FILE" python/ > /dev/null
# Get absolute path to zip file
ZIP_PATH=$(pwd)/$ZIP_FILE
ZIP_SIZE=$(du -h "$ZIP_PATH" | cut -f1)
echo "Layer package created: $ZIP_PATH (Size: $ZIP_SIZE)"
echo "Publishing layer to AWS..."
# Publish layer
LAYER_OUTPUT=$(aws lambda publish-layer-version \
--layer-name "$LAYER_NAME" \
--zip-file "fileb://$ZIP_PATH" \
--compatible-runtimes "python${PYTHON_VERSION}" \
--region "$REGION" \
--description "MCP Gateway Python dependencies layer" \
--output json)
# Extract LayerVersionArn
LAYER_ARN=$(echo "$LAYER_OUTPUT" | grep -o '"LayerVersionArn": "[^"]*' | cut -d'"' -f4)
echo ""
echo "✅ Layer created successfully!"
echo "Layer ARN: $LAYER_ARN"
echo "Layer size: $ZIP_SIZE"
echo ""
echo "Add this to your terraform.tfvars:"
echo "python_dependencies_layer_arn = \"$LAYER_ARN\""
# Cleanup
rm -rf "$TEMP_DIR"
