download-database.jsā¢5.65 kB
#!/usr/bin/env node
/**
* Database Download Script
* Downloads the EGW database from cloud storage on startup
*/
import fs from 'fs';
import https from 'https';
import path from 'path';
import { fileURLToPath } from 'url';
import { dirname } from 'path';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const DATABASE_URL = process.env.EGW_DATABASE_URL || 'https://github.com/pythondev-pro/egw_writings_mcp_server/releases/download/v1.0.0/egw-writings.db';
const DATABASE_PATH = process.env.EGW_DATABASE_PATH || path.join(__dirname, '../data/egw-writings.db');
const MAX_RETRIES = 3;
const RETRY_DELAY = 5000; // 5 seconds
export async function downloadDatabase() {
console.log('š„ Starting database download...');
console.log(`š Source: ${DATABASE_URL}`);
console.log(`š¾ Destination: ${DATABASE_PATH}`);
// Create data directory if it doesn't exist
const dataDir = path.dirname(DATABASE_PATH);
if (!fs.existsSync(dataDir)) {
fs.mkdirSync(dataDir, { recursive: true });
console.log(`š Created directory: ${dataDir}`);
}
// Check if database already exists
if (fs.existsSync(DATABASE_PATH)) {
const stats = fs.statSync(DATABASE_PATH);
console.log(`ā
Database already exists: ${(stats.size / 1024 / 1024 / 1024).toFixed(2)} GB`);
// Verify it's not empty
if (stats.size > 1000000000) { // > 1GB
console.log('ā
Database appears valid, skipping download');
return true;
} else {
console.log('ā ļø Database file exists but is too small, re-downloading...');
fs.unlinkSync(DATABASE_PATH);
}
}
// Download with retry logic
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
try {
console.log(`š Download attempt ${attempt}/${MAX_RETRIES}...`);
const result = await downloadFile(DATABASE_URL, DATABASE_PATH);
if (result) {
const stats = fs.statSync(DATABASE_PATH);
console.log(`ā
Download successful: ${(stats.size / 1024 / 1024 / 1024).toFixed(2)} GB`);
return true;
}
} catch (error) {
console.error(`ā Download attempt ${attempt} failed:`, error.message);
if (attempt < MAX_RETRIES) {
console.log(`ā³ Retrying in ${RETRY_DELAY / 1000} seconds...`);
await new Promise(resolve => setTimeout(resolve, RETRY_DELAY));
} else {
console.error('ā All download attempts failed');
return false;
}
}
}
return false;
}
function downloadFile(url, filePath) {
return new Promise((resolve, reject) => {
const file = fs.createWriteStream(filePath);
let downloadedBytes = 0;
let totalBytes = 0;
https.get(url, (response) => {
// Handle redirects
if (response.statusCode === 302 || response.statusCode === 301) {
const redirectUrl = response.headers.location;
console.log(`š Following redirect to: ${redirectUrl}`);
https.get(redirectUrl, handleResponse).on('error', reject);
return;
}
handleResponse(response);
}).on('error', reject);
function handleResponse(response) {
if (response.statusCode !== 200) {
reject(new Error(`HTTP ${response.statusCode}: ${response.statusMessage}`));
return;
}
totalBytes = parseInt(response.headers['content-length'] || '0', 10);
console.log(`š Total size: ${(totalBytes / 1024 / 1024 / 1024).toFixed(2)} GB`);
response.on('data', (chunk) => {
downloadedBytes += chunk.length;
file.write(chunk);
// Show progress every 100MB
if (Math.floor(downloadedBytes / (100 * 1024 * 1024)) > Math.floor((downloadedBytes - chunk.length) / (100 * 1024 * 1024))) {
const progress = totalBytes > 0 ? (downloadedBytes / totalBytes * 100).toFixed(1) : '?';
console.log(`š Download progress: ${(downloadedBytes / 1024 / 1024 / 1024).toFixed(2)} GB (${progress}%)`);
}
});
response.on('end', () => {
file.end();
console.log(`ā
Download completed: ${(downloadedBytes / 1024 / 1024 / 1024).toFixed(2)} GB`);
resolve(true);
});
response.on('error', (error) => {
file.end();
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath); // Clean up partial file
}
reject(error);
});
}
// Set timeout for slow downloads (30 minutes)
const timeout = setTimeout(() => {
file.end();
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath);
}
reject(new Error('Download timeout after 30 minutes'));
}, 30 * 60 * 1000);
file.on('finish', () => {
clearTimeout(timeout);
});
});
}
// Health check function
export function checkDatabaseHealth() {
if (!fs.existsSync(DATABASE_PATH)) {
console.error('ā Database file not found');
return false;
}
const stats = fs.statSync(DATABASE_PATH);
console.log(`š Database size: ${(stats.size / 1024 / 1024 / 1024).toFixed(2)} GB`);
if (stats.size < 1000000000) { // Less than 1GB
console.error('ā Database file too small, likely corrupted');
return false;
}
console.log('ā
Database health check passed');
return true;
}
// Main execution
if (import.meta.url === `file://${process.argv[1]}`) {
downloadDatabase()
.then(success => {
if (success) {
const healthy = checkDatabaseHealth();
process.exit(healthy ? 0 : 1);
} else {
process.exit(1);
}
})
.catch(error => {
console.error('ā Fatal error:', error);
process.exit(1);
});
}