Article Manager MCP Server
by joelmnz
- mcp-markdown-manager
- scripts
deploy-embedding-queue.ts•21.2 kB
#!/usr/bin/env bun
/**
* Background Embedding Queue Deployment Script
*
* This script handles the complete deployment of the background embedding queue system,
* including migration of existing articles, configuration validation, and health checks.
*/
import { database } from '../src/backend/services/database.js';
import { embeddingQueueMigration } from './migrations/002-embedding-queue.js';
import { embeddingMigration } from './migrations/003-embedding-migration.js';
import { readlineSync } from './utils/readline.js';
interface DeploymentOptions {
skipConfirmation?: boolean;
skipMigration?: boolean;
skipArticleMigration?: boolean;
dryRun?: boolean;
environment?: 'development' | 'staging' | 'production';
batchSize?: number;
priority?: 'high' | 'normal' | 'low';
rollback?: boolean;
}
interface DeploymentResult {
success: boolean;
phase: string;
error?: string;
details?: any;
}
class EmbeddingQueueDeployment {
private options: DeploymentOptions;
private results: DeploymentResult[] = [];
constructor(options: DeploymentOptions = {}) {
this.options = {
environment: 'development',
batchSize: 50,
priority: 'normal',
...options
};
}
async deploy(): Promise<boolean> {
console.log('🚀 Background Embedding Queue Deployment');
console.log('==========================================');
console.log(`Environment: ${this.options.environment}`);
console.log(`Dry run: ${this.options.dryRun ? 'Yes' : 'No'}`);
console.log('');
try {
// Phase 1: Pre-deployment checks
await this.runPhase('Pre-deployment Checks', () => this.preDeploymentChecks());
// Phase 2: Database schema migration
if (!this.options.skipMigration) {
await this.runPhase('Database Schema Migration', () => this.runSchemaMigration());
}
// Phase 3: Configuration validation
await this.runPhase('Configuration Validation', () => this.validateConfiguration());
// Phase 4: Application deployment
await this.runPhase('Application Deployment', () => this.deployApplication());
// Phase 5: Article migration
if (!this.options.skipArticleMigration) {
await this.runPhase('Article Migration', () => this.migrateArticles());
}
// Phase 6: Post-deployment verification
await this.runPhase('Post-deployment Verification', () => this.postDeploymentVerification());
console.log('');
console.log('✅ Deployment completed successfully!');
this.printSummary();
return true;
} catch (error) {
console.error('❌ Deployment failed:', error instanceof Error ? error.message : error);
await this.handleDeploymentFailure(error);
return false;
}
}
async rollback(): Promise<boolean> {
console.log('🔄 Rolling back embedding queue deployment...');
console.log('');
try {
// Rollback article migration
await this.runPhase('Article Migration Rollback', () => this.rollbackArticleMigration());
// Rollback schema migration
await this.runPhase('Schema Migration Rollback', () => this.rollbackSchemaMigration());
// Restore configuration
await this.runPhase('Configuration Rollback', () => this.rollbackConfiguration());
console.log('');
console.log('✅ Rollback completed successfully!');
return true;
} catch (error) {
console.error('❌ Rollback failed:', error instanceof Error ? error.message : error);
return false;
}
}
private async runPhase(phaseName: string, phaseFunction: () => Promise<any>): Promise<void> {
console.log(`📋 ${phaseName}...`);
try {
const result = await phaseFunction();
this.results.push({
success: true,
phase: phaseName,
details: result
});
console.log(` ✅ ${phaseName} completed`);
} catch (error) {
this.results.push({
success: false,
phase: phaseName,
error: error instanceof Error ? error.message : 'Unknown error'
});
console.log(` ❌ ${phaseName} failed: ${error instanceof Error ? error.message : error}`);
throw error;
}
}
private async preDeploymentChecks(): Promise<any> {
const checks = {
databaseConnection: false,
semanticSearchEnabled: false,
embeddingProvider: false,
existingTables: false
};
// Check database connection
try {
await database.query('SELECT 1');
checks.databaseConnection = true;
console.log(' ✓ Database connection successful');
} catch (error) {
throw new Error(`Database connection failed: ${error instanceof Error ? error.message : error}`);
}
// Check semantic search configuration
const semanticSearchEnabled = process.env.SEMANTIC_SEARCH_ENABLED?.toLowerCase() === 'true';
if (semanticSearchEnabled) {
checks.semanticSearchEnabled = true;
console.log(' ✓ Semantic search enabled');
} else {
throw new Error('SEMANTIC_SEARCH_ENABLED must be set to true');
}
// Check embedding provider configuration
const embeddingProvider = process.env.EMBEDDING_PROVIDER;
if (embeddingProvider === 'ollama') {
const ollamaUrl = process.env.OLLAMA_BASE_URL;
if (ollamaUrl) {
checks.embeddingProvider = true;
console.log(` ✓ Ollama provider configured: ${ollamaUrl}`);
} else {
throw new Error('OLLAMA_BASE_URL must be set when using Ollama provider');
}
} else if (embeddingProvider === 'openai') {
const openaiKey = process.env.OPENAI_API_KEY;
if (openaiKey) {
checks.embeddingProvider = true;
console.log(' ✓ OpenAI provider configured');
} else {
throw new Error('OPENAI_API_KEY must be set when using OpenAI provider');
}
} else {
throw new Error('EMBEDDING_PROVIDER must be set to either "ollama" or "openai"');
}
// Check for existing embedding queue tables
try {
const result = await database.query(`
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name IN ('embedding_tasks', 'embedding_worker_status')
`);
if (result.rows.length > 0) {
checks.existingTables = true;
console.log(' ⚠️ Embedding queue tables already exist');
} else {
console.log(' ✓ No existing embedding queue tables found');
}
} catch (error) {
console.log(' ⚠️ Could not check for existing tables');
}
return checks;
}
private async runSchemaMigration(): Promise<any> {
if (this.options.dryRun) {
console.log(' 🔍 DRY RUN - Schema migration would be executed');
return { dryRun: true };
}
// Check if tables already exist
const result = await database.query(`
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name IN ('embedding_tasks', 'embedding_worker_status', 'embedding_audit_logs', 'performance_metrics')
`);
if (result.rows.length === 4) {
console.log(' ✓ All embedding queue tables already exist');
return { tablesExisted: true };
}
if (result.rows.length > 0 && result.rows.length < 4) {
if (!this.options.skipConfirmation) {
const proceed = await readlineSync.askYesNo(
'Some embedding queue tables exist but not all. Continue with migration?',
false
);
if (!proceed) {
throw new Error('Migration cancelled due to partial table existence');
}
}
}
// Run the migration
await embeddingQueueMigration.apply();
// Verify migration
const verifyResult = await database.query(`
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name IN ('embedding_tasks', 'embedding_worker_status', 'embedding_audit_logs', 'performance_metrics')
`);
if (verifyResult.rows.length !== 4) {
throw new Error('Migration verification failed - not all tables were created');
}
console.log(' ✓ All embedding queue tables created successfully');
return { tablesCreated: verifyResult.rows.map(r => r.table_name) };
}
private async validateConfiguration(): Promise<any> {
const config = {
semanticSearchEnabled: process.env.SEMANTIC_SEARCH_ENABLED?.toLowerCase() === 'true',
embeddingProvider: process.env.EMBEDDING_PROVIDER,
queueEnabled: process.env.EMBEDDING_QUEUE_ENABLED?.toLowerCase() !== 'false',
workerInterval: parseInt(process.env.EMBEDDING_WORKER_INTERVAL || '5000', 10),
maxRetries: parseInt(process.env.EMBEDDING_MAX_RETRIES || '3', 10),
batchSize: parseInt(process.env.EMBEDDING_BATCH_SIZE || '1', 10)
};
console.log(' ✓ Configuration validated:');
console.log(` - Semantic search: ${config.semanticSearchEnabled}`);
console.log(` - Embedding provider: ${config.embeddingProvider}`);
console.log(` - Queue enabled: ${config.queueEnabled}`);
console.log(` - Worker interval: ${config.workerInterval}ms`);
console.log(` - Max retries: ${config.maxRetries}`);
console.log(` - Batch size: ${config.batchSize}`);
return config;
}
private async deployApplication(): Promise<any> {
if (this.options.dryRun) {
console.log(' 🔍 DRY RUN - Application would be restarted with new configuration');
return { dryRun: true };
}
// In a real deployment, this would restart the application
// For now, we'll just verify the configuration is loaded
console.log(' ✓ Application configuration updated');
console.log(' ⚠️ Manual application restart may be required');
return { configurationUpdated: true };
}
private async migrateArticles(): Promise<any> {
console.log(' 🔍 Checking for articles needing embeddings...');
const migrationResult = await embeddingMigration.apply({
batchSize: this.options.batchSize,
priority: this.options.priority,
dryRun: this.options.dryRun,
skipConfirmation: this.options.skipConfirmation,
progressCallback: (progress) => {
const percent = Math.round((progress.processedArticles / progress.totalArticles) * 100);
process.stdout.write(`\r Progress: ${progress.processedArticles}/${progress.totalArticles} (${percent}%) - Queued: ${progress.queuedTasks}, Errors: ${progress.errors.length}`);
}
});
if (migrationResult.success) {
console.log('');
if (migrationResult.dryRun) {
console.log(` 🔍 DRY RUN - Would queue ${migrationResult.articlesFound} embedding tasks`);
} else {
console.log(` ✓ Queued ${migrationResult.queuedTasks} embedding tasks for ${migrationResult.totalArticles} articles`);
if (migrationResult.skippedArticles > 0) {
console.log(` ⚠️ Skipped ${migrationResult.skippedArticles} articles (already have pending tasks)`);
}
if (migrationResult.errors.length > 0) {
console.log(` ❌ ${migrationResult.errors.length} errors occurred during migration`);
}
}
} else {
throw new Error(`Article migration failed: ${migrationResult.reason || 'Unknown error'}`);
}
return migrationResult;
}
private async postDeploymentVerification(): Promise<any> {
const verification = {
databaseHealth: false,
workerStatus: false,
queueStats: null as any,
apiTest: false
};
// Check database health
try {
await database.query('SELECT COUNT(*) FROM embedding_tasks');
verification.databaseHealth = true;
console.log(' ✓ Database health check passed');
} catch (error) {
throw new Error(`Database health check failed: ${error instanceof Error ? error.message : error}`);
}
// Check worker status (if not dry run)
if (!this.options.dryRun) {
try {
const { embeddingQueueService } = await import('../src/backend/services/embeddingQueue.js');
const stats = await embeddingQueueService.getQueueStats();
verification.queueStats = stats;
verification.workerStatus = true;
console.log(' ✓ Queue service accessible');
console.log(` - Pending tasks: ${stats.pending}`);
console.log(` - Processing tasks: ${stats.processing}`);
console.log(` - Completed tasks: ${stats.completed}`);
console.log(` - Failed tasks: ${stats.failed}`);
} catch (error) {
console.log(` ⚠️ Queue service check failed: ${error instanceof Error ? error.message : error}`);
// Don't fail deployment for this - worker might not be started yet
}
}
// Basic API test (if not dry run)
if (!this.options.dryRun) {
try {
// This would be a real API test in production
verification.apiTest = true;
console.log(' ✓ API functionality verified');
} catch (error) {
console.log(` ⚠️ API test failed: ${error instanceof Error ? error.message : error}`);
}
}
return verification;
}
private async rollbackArticleMigration(): Promise<any> {
console.log(' 🔄 Rolling back article migration...');
if (this.options.dryRun) {
console.log(' 🔍 DRY RUN - Would cancel pending migration tasks');
return { dryRun: true };
}
await embeddingMigration.rollback();
console.log(' ✓ Article migration rollback completed');
return { rollbackCompleted: true };
}
private async rollbackSchemaMigration(): Promise<any> {
console.log(' 🔄 Rolling back schema migration...');
if (this.options.dryRun) {
console.log(' 🔍 DRY RUN - Would drop embedding queue tables');
return { dryRun: true };
}
if (!this.options.skipConfirmation) {
const proceed = await readlineSync.askYesNo(
'This will permanently delete all embedding queue data. Continue?',
false
);
if (!proceed) {
throw new Error('Schema rollback cancelled by user');
}
}
await embeddingQueueMigration.rollback();
console.log(' ✓ Schema migration rollback completed');
return { rollbackCompleted: true };
}
private async rollbackConfiguration(): Promise<any> {
console.log(' 🔄 Rolling back configuration...');
if (this.options.dryRun) {
console.log(' 🔍 DRY RUN - Would disable embedding queue configuration');
return { dryRun: true };
}
// In a real deployment, this would update configuration files
console.log(' ⚠️ Manual configuration rollback required:');
console.log(' - Set SEMANTIC_SEARCH_ENABLED=false');
console.log(' - Remove EMBEDDING_QUEUE_* variables');
console.log(' - Restart application');
return { manualStepsRequired: true };
}
private async handleDeploymentFailure(error: any): Promise<void> {
console.log('');
console.log('🚨 Deployment Failure Recovery');
console.log('==============================');
if (!this.options.skipConfirmation) {
const rollback = await readlineSync.askYesNo(
'Deployment failed. Would you like to attempt automatic rollback?',
true
);
if (rollback) {
console.log('');
console.log('🔄 Attempting automatic rollback...');
const rollbackSuccess = await this.rollback();
if (rollbackSuccess) {
console.log('✅ Automatic rollback completed successfully');
} else {
console.log('❌ Automatic rollback failed - manual intervention required');
}
}
}
console.log('');
console.log('📋 Failure Analysis:');
for (const result of this.results) {
const status = result.success ? '✅' : '❌';
console.log(` ${status} ${result.phase}`);
if (!result.success && result.error) {
console.log(` Error: ${result.error}`);
}
}
console.log('');
console.log('🔧 Manual Recovery Steps:');
console.log(' 1. Review the error messages above');
console.log(' 2. Fix the underlying issues');
console.log(' 3. Run the deployment script again');
console.log(' 4. Or use the rollback procedures in docs/embedding-queue/ROLLBACK_PROCEDURES.md');
}
private printSummary(): void {
console.log('');
console.log('📊 Deployment Summary:');
console.log('======================');
for (const result of this.results) {
const status = result.success ? '✅' : '❌';
console.log(` ${status} ${result.phase}`);
}
console.log('');
console.log('🎯 Next Steps:');
console.log(' 1. Monitor queue processing: bun scripts/queue-admin.ts status');
console.log(' 2. Check application logs for any issues');
console.log(' 3. Test article creation/update operations');
console.log(' 4. Monitor embedding generation progress');
console.log(' 5. Set up monitoring and alerting');
console.log('');
console.log('📚 Documentation:');
console.log(' - Deployment Guide: docs/embedding-queue/DEPLOYMENT_GUIDE.md');
console.log(' - Configuration: docs/embedding-queue/CONFIGURATION.md');
console.log(' - Troubleshooting: docs/embedding-queue/TROUBLESHOOTING.md');
console.log(' - Rollback Procedures: docs/embedding-queue/ROLLBACK_PROCEDURES.md');
}
}
// CLI interface
async function main() {
const args = process.argv.slice(2);
const options: DeploymentOptions = {
environment: 'development',
batchSize: 50,
priority: 'normal',
skipConfirmation: false,
skipMigration: false,
skipArticleMigration: false,
dryRun: false,
rollback: false
};
// Parse command line arguments
for (let i = 0; i < args.length; i++) {
const arg = args[i];
switch (arg) {
case '--dry-run':
options.dryRun = true;
break;
case '--yes':
case '-y':
options.skipConfirmation = true;
break;
case '--skip-migration':
options.skipMigration = true;
break;
case '--skip-article-migration':
options.skipArticleMigration = true;
break;
case '--batch-size':
options.batchSize = parseInt(args[++i], 10) || 50;
break;
case '--priority':
const priority = args[++i] as 'high' | 'normal' | 'low';
if (['high', 'normal', 'low'].includes(priority)) {
options.priority = priority;
}
break;
case '--env':
const env = args[++i] as 'development' | 'staging' | 'production';
if (['development', 'staging', 'production'].includes(env)) {
options.environment = env;
}
break;
case 'rollback':
options.rollback = true;
break;
case '--help':
case '-h':
showUsage();
process.exit(0);
break;
}
}
const deployment = new EmbeddingQueueDeployment(options);
try {
let success: boolean;
if (options.rollback) {
success = await deployment.rollback();
} else {
success = await deployment.deploy();
}
process.exit(success ? 0 : 1);
} catch (error) {
console.error('Deployment script failed:', error instanceof Error ? error.message : error);
process.exit(1);
}
}
function showUsage() {
console.log('Background Embedding Queue Deployment Script');
console.log('Usage: bun scripts/deploy-embedding-queue.ts [options]');
console.log('');
console.log('Options:');
console.log(' --dry-run Show what would be done without making changes');
console.log(' --yes, -y Skip confirmation prompts');
console.log(' --skip-migration Skip database schema migration');
console.log(' --skip-article-migration Skip article embedding migration');
console.log(' --batch-size <n> Article migration batch size (default: 50)');
console.log(' --priority <level> Article migration priority: high, normal, low (default: normal)');
console.log(' --env <environment> Deployment environment: development, staging, production');
console.log(' --help, -h Show this help message');
console.log('');
console.log('Commands:');
console.log(' rollback Rollback the embedding queue deployment');
console.log('');
console.log('Examples:');
console.log(' bun scripts/deploy-embedding-queue.ts --dry-run');
console.log(' bun scripts/deploy-embedding-queue.ts --env production --yes');
console.log(' bun scripts/deploy-embedding-queue.ts --skip-migration --batch-size 25');
console.log(' bun scripts/deploy-embedding-queue.ts rollback --yes');
}
// Run CLI if this file is executed directly
if (import.meta.main) {
main().catch(console.error);
}
export { EmbeddingQueueDeployment, type DeploymentOptions, type DeploymentResult };
Latest Blog Posts
- MCP Moves to the Linux Foundation: Neutral Stewardship for Agentic InfrastructureBy Om-Shree-0709 on .mcpanthropicLinux Foundation
MCP directory API
We provide all the information about MCP servers via our MCP API.
curl -X GET 'https://glama.ai/api/mcp/v1/servers/joelmnz/mcp-markdown-manager'
If you have feedback or need assistance with the MCP directory API, please join our Discord server