/**
* Monitored Folders Orchestrator
*
* Singleton service that orchestrates all FolderLifecycleManager instances,
* controlling their state transitions and updating FMDM.
*/
import { EventEmitter } from 'events';
import { FolderLifecycleService } from '../../application/indexing/folder-lifecycle-service.js';
import { IFolderLifecycleManager } from '../../domain/folders/folder-lifecycle-manager.js';
import { IIndexingOrchestrator, IFileSystemService, ILoggingService } from '../../di/interfaces.js';
import { FMDMService } from './fmdm-service.js';
import { SQLiteVecStorage } from '../../infrastructure/embeddings/sqlite-vec/sqlite-vec-storage.js';
import { FileStateService } from '../../infrastructure/files/file-state-service.js';
import { FolderConfig, FolderIndexingStatus } from '../models/fmdm.js';
import { getSupportedExtensions } from '../../domain/files/supported-extensions.js';
import { ResourceManager, ResourceLimits, ResourceStats } from '../../application/indexing/resource-manager.js';
import { WindowsPerformanceService, IWindowsPerformanceService } from './windows-performance-service.js';
import { ModelDownloadManager, IModelDownloadManager } from './model-download-manager.js';
import { FolderIndexingQueue } from './folder-indexing-queue.js';
import { UnifiedModelFactory } from '../factories/unified-model-factory.js';
import { PythonEmbeddingServiceRegistry } from '../factories/model-factories.js';
import { getDefaultModelId, getSentenceTransformerIdFromModelId, getModelMetadata } from '../../config/model-registry.js';
import { OnnxConfiguration } from '../../infrastructure/config/onnx-configuration.js';
import { PeriodicSyncService } from '../../application/monitoring/periodic-sync-service.js';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { fileURLToPath } from 'url';
import { dirname } from 'path';
import { ActivityService } from './activity-service.js';
/**
* Checks if an error is an environment/system error that should NOT trigger data cleanup.
* Environment errors are issues with the Node.js environment, native modules, or system
* configuration that are NOT the user's fault and should be retried after fixing the environment.
*
* CRITICAL: When these errors occur, we must:
* - NOT delete the .folder-mcp directory (preserve indexed data)
* - NOT remove from configuration (allow retry after fix)
* - Show clear error message about environment issue
*
* @param error The error to check
* @returns True if this is an environment error that should preserve data
*/
function isEnvironmentError(error: Error | string): boolean {
const errorMessage = error instanceof Error ? error.message : String(error);
const errorStack = error instanceof Error ? error.stack || '' : '';
const fullErrorText = `${errorMessage} ${errorStack}`.toLowerCase();
// Native module version mismatch (Node.js upgrade)
if (fullErrorText.includes('node_module_version') ||
fullErrorText.includes('compiled against a different node.js version') ||
fullErrorText.includes('please try re-compiling or re-installing')) {
return true;
}
// Native module loading failures
if (fullErrorText.includes('dlopen') ||
fullErrorText.includes('error loading native module') ||
fullErrorText.includes('cannot load such file') ||
fullErrorText.includes('cannot open shared object')) {
return true;
}
// Specific native module issues (better-sqlite3, sqlite-vec)
if ((fullErrorText.includes('better-sqlite3') || fullErrorText.includes('sqlite-vec')) &&
(fullErrorText.includes('.node') || fullErrorText.includes('native'))) {
return true;
}
// Missing native module binaries
if (fullErrorText.includes('was compiled against') ||
fullErrorText.includes('cannot find module') && fullErrorText.includes('.node')) {
return true;
}
// Python environment issues (should not delete data)
if (fullErrorText.includes('python') &&
(fullErrorText.includes('not found') ||
fullErrorText.includes('no module named') ||
fullErrorText.includes('modulenotfounderror'))) {
return true;
}
return false;
}
/**
* Get a user-friendly error message for environment errors
*/
function getEnvironmentErrorMessage(error: Error | string): string {
const errorMessage = error instanceof Error ? error.message : String(error);
const fullText = errorMessage.toLowerCase();
if (fullText.includes('node_module_version') ||
fullText.includes('compiled against a different node.js version')) {
return 'Native module version mismatch - please run: npm rebuild better-sqlite3';
}
if (fullText.includes('dlopen') || fullText.includes('error loading native module')) {
return 'Failed to load native module - please reinstall dependencies: npm install';
}
if (fullText.includes('python') && fullText.includes('not found')) {
return 'Python not found - GPU models require Python 3.8+';
}
// Return original message if no specific match
return errorMessage;
}
export interface IMonitoredFoldersOrchestrator {
/**
* Add a folder to be monitored
*/
addFolder(path: string, model: string): Promise<void>;
/**
* Add folder to FMDM immediately (synchronous, prevents race conditions)
* Called by WebSocket handler before async operations begin
*/
addFolderToFMDMImmediate(path: string, model: string): void;
/**
* Remove a folder from monitoring
*/
removeFolder(folderPath: string): Promise<void>;
/**
* Start managing all configured folders
*/
startAll(): Promise<void>;
/**
* Stop all folder managers
*/
stopAll(): Promise<void>;
/**
* Get manager for a specific folder
*/
getManager(folderPath: string): IFolderLifecycleManager | undefined;
/**
* Get the model download manager (for setting cache callbacks)
*/
getModelDownloadManager(): IModelDownloadManager;
}
// Helper function to get model dimensions from curated models
function getModelDimensions(modelId: string): number {
try {
// Load curated models JSON dynamically - ES module compatible
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const curatedModelsPath = path.join(__dirname, '../../config/curated-models.json');
const curatedModelsContent = fs.readFileSync(curatedModelsPath, 'utf-8');
const curatedModels = JSON.parse(curatedModelsContent);
// Check GPU models
const gpuModel = curatedModels.gpuModels.models.find((m: any) => m.id === modelId);
if (gpuModel) {
if (!gpuModel.dimensions) {
throw new Error(`Model ${modelId} found in curated models but missing dimensions property`);
}
return gpuModel.dimensions;
}
// Check CPU models
const cpuModel = curatedModels.cpuModels.models.find((m: any) => m.id === modelId);
if (cpuModel) {
if (!cpuModel.dimensions) {
throw new Error(`Model ${modelId} found in curated models but missing dimensions property`);
}
return cpuModel.dimensions;
}
// Model not found in curated list - this is an error
throw new Error(`Model ${modelId} not found in curated models. Cannot determine dimensions.`);
} catch (error) {
// Re-throw with context about model dimension lookup
throw new Error(`Failed to get dimensions for model ${modelId}: ${error instanceof Error ? error.message : String(error)}`);
}
}
// Factory function to create FolderLifecycleService instances
function createFolderLifecycleService(
id: string,
path: string,
indexingOrchestrator: IIndexingOrchestrator,
fileSystemService: IFileSystemService,
storage: any,
fileStateService: any,
logger: ILoggingService,
model?: string,
maxConcurrentFiles?: number
): IFolderLifecycleManager {
return new FolderLifecycleService(id, path, indexingOrchestrator, fileSystemService, storage, fileStateService, logger, model, maxConcurrentFiles);
}
export class MonitoredFoldersOrchestrator extends EventEmitter implements IMonitoredFoldersOrchestrator {
private folderManagers = new Map<string, IFolderLifecycleManager>();
// Removed errorFolders - redundant, error state is tracked in folderManagers
private monitoringOrchestrator: any; // Will be imported dynamically when needed
private folderValidationTimer?: NodeJS.Timeout;
private resourceManager: ResourceManager;
private onnxConfiguration: OnnxConfiguration;
private windowsPerformanceService: IWindowsPerformanceService;
private modelDownloadManager: IModelDownloadManager;
private folderIndexingQueue: FolderIndexingQueue;
private modelFactory: UnifiedModelFactory;
private pythonInitializationPromise?: Promise<void>;
private periodicSyncService: PeriodicSyncService;
private folderStorages = new Map<string, SQLiteVecStorage>(); // Track storage per folder
private activityService: ActivityService | undefined = undefined;
constructor(
private indexingOrchestrator: IIndexingOrchestrator,
private fmdmService: FMDMService,
private fileSystemService: IFileSystemService,
private logger: ILoggingService,
private configService: any, // TODO: Add proper type
windowsPerformanceService?: IWindowsPerformanceService,
modelDownloadManager?: IModelDownloadManager,
activityService?: ActivityService
) {
super();
// Store activity service for progress event emission
this.activityService = activityService;
// Initialize ONNX configuration service with config component
this.onnxConfiguration = new OnnxConfiguration(this.configService);
// Initialize Windows performance service (default if not provided)
this.windowsPerformanceService = windowsPerformanceService || new WindowsPerformanceService(this.logger);
// Initialize Model Download Manager (default if not provided)
this.modelDownloadManager = modelDownloadManager || new ModelDownloadManager(this.logger, this.fmdmService);
// Initialize the sequential folder indexing queue
this.folderIndexingQueue = new FolderIndexingQueue(this.logger);
// Wire ModelDownloadManager to queue for sequential processing
this.folderIndexingQueue.setModelDownloadManager(this.modelDownloadManager);
// Initialize the unified model factory
this.modelFactory = new UnifiedModelFactory(this.logger);
// Set the model factory in the queue
this.folderIndexingQueue.setModelFactory(this.modelFactory);
// Subscribe to queue events for FMDM updates
this.setupQueueEventHandlers();
// Wire Python embedding service to model download manager
// Store the promise so we can wait for it in startAll()
this.pythonInitializationPromise = this.initializePythonEmbeddingService();
// Wire ONNX downloader to model download manager
this.initializeONNXDownloader();
// Initialize resource manager with simple concurrency limits
const resourceLimits: Partial<ResourceLimits> = {
maxConcurrentOperations: 2, // Optimal limit from testing
maxQueueSize: 20 // Reasonable queue size for folders
};
this.resourceManager = new ResourceManager(this.logger, resourceLimits);
// Optional: Log resource stats for monitoring (informational only)
this.resourceManager.on('stats', (stats: ResourceStats) => {
if (stats.activeOperations > 0) {
this.logger.debug('[ORCHESTRATOR] Resource usage', {
memoryUsedMB: Math.round(stats.memoryUsedMB),
activeOperations: stats.activeOperations,
queuedOperations: stats.queuedOperations
});
}
});
// Initialize periodic sync service (60 second interval)
this.periodicSyncService = new PeriodicSyncService(this.logger, this.fileSystemService, {
intervalMs: 60000, // 60 seconds
vec0CleanupEnabled: true
});
// Start periodic folder validation (every 30 seconds)
this.startFolderValidation();
// Start periodic filesystem sync (every 60 seconds)
this.startPeriodicSync();
}
async addFolder(path: string, model: string): Promise<void> {
// Check if already managing this folder (use normalized path key for case-insensitive comparison on Windows)
const pathKey = this.normalizePathKey(path);
if (this.folderManagers.has(pathKey)) {
this.logger.warn(`Already managing folder: ${path}`);
return;
}
// Check if folder exists first
const fs = await import('fs');
if (!fs.existsSync(path)) {
this.logger.error(`[ORCHESTRATOR] Cannot add non-existent folder: ${path}`);
// Create a dummy error folder config for FMDM
const errorFolderConfig: FolderConfig = {
path: path,
model: model,
status: 'error',
progress: 0,
notification: this.createErrorNotification('Folder does not exist')
};
// Update FMDM directly with the error folder state
// Since the folder doesn't exist in FMDM yet, we need to add it directly
const currentFolders = this.getCurrentFolderConfigs();
currentFolders.push(errorFolderConfig);
this.fmdmService.updateFolders(currentFolders);
const error = new Error(`Folder does not exist: ${path}`);
this.logger.error(`Failed to add folder: ${path}`, error);
throw error;
}
// Submit folder addition operation through resource manager
const operationId = `add-folder-${path}`;
const estimatedMemoryMB = 100; // Estimated memory for folder addition
try {
await this.resourceManager.submitOperation(
operationId,
path,
() => this.executeAddFolder(path, model),
{
priority: 1, // High priority for folder additions
estimatedMemoryMB
}
);
} catch (error) {
this.logger.error(`[ORCHESTRATOR] Resource manager rejected add folder operation: ${path}`, error instanceof Error ? error : new Error(String(error)));
// Extract error message
let errorMessage = error instanceof Error ? error.message : String(error);
// Create error folder config for FMDM
const errorFolderConfig: FolderConfig = {
path: path,
model: model,
status: 'error',
progress: 0,
notification: this.createErrorNotification(errorMessage)
};
// Update FMDM directly with the error folder state
// Get current folders from FMDM (not just from managers)
const currentFMDM = this.fmdmService.getFMDM();
const currentFolders = [...currentFMDM.folders];
// Check if folder already exists in current folders
const existingIndex = currentFolders.findIndex(f => this.pathsEqual(f.path, path));
if (existingIndex >= 0) {
// Update existing folder
currentFolders[existingIndex] = errorFolderConfig;
} else {
// Add new folder with error status
currentFolders.push(errorFolderConfig);
}
this.fmdmService.updateFolders(currentFolders);
// Perform resource cleanup on operation failure
// Pass the original error so we can detect environment errors and preserve data
const errorObj = error instanceof Error ? error : new Error(String(error));
await this.performResourceCleanup(path, 'add_folder_failed', errorObj);
throw error;
}
}
/**
* Platform-aware path comparison helper
* Returns true if paths refer to the same folder (case-insensitive on Windows)
* Also normalizes path separators on Windows (\ vs /)
*/
private pathsEqual(path1: string, path2: string): boolean {
const isWindows = process.platform === 'win32';
if (isWindows) {
// Normalize both case and path separator on Windows
const normalized1 = path1.toLowerCase().replace(/\\/g, '/');
const normalized2 = path2.toLowerCase().replace(/\\/g, '/');
return normalized1 === normalized2;
}
return path1 === path2;
}
/**
* Normalize path for use as Map key
* Returns lowercase path with normalized separators on Windows for consistent Map key comparison
* On Unix, returns original path (case-sensitive filesystem)
*/
private normalizePathKey(path: string): string {
const isWindows = process.platform === 'win32';
return isWindows ? path.toLowerCase().replace(/\\/g, '/') : path;
}
/**
* Extract a readable folder name from a full path
* Returns the last component of the path using cross-platform path.basename
*/
private extractFolderName(folderPath: string): string {
return path.basename(folderPath) || folderPath;
}
/**
* Start lifecycle management for a folder already in FMDM (during daemon startup)
*/
private async startFolderLifecycle(path: string, model: string): Promise<void> {
try {
this.logger.info(`🔍 [FILE-WATCH-TRACE] startFolderLifecycle() called for ${path} with model ${model}`);
this.logger.debug(`[ORCHESTRATOR] Starting lifecycle for FMDM folder ${path} with model ${model}`);
// Check if model is available through FMDM
const fmdm = this.fmdmService.getFMDM();
const modelInfo = fmdm.curatedModels.find(m => m.id === model);
if (!modelInfo) {
throw new Error(`Model ${model} not found in curated models`);
}
// Queue will handle model download/loading if needed
this.logger.info(`🔍 [FILE-WATCH-TRACE] Model ${model} check complete, calling startFolderScanning()`);
await this.startFolderScanning(path, model, false); // Don't save to config during startup
} catch (error) {
const errorObj = error instanceof Error ? error : new Error(String(error));
this.logger.error(`[ORCHESTRATOR] Failed to start lifecycle for folder ${path}`, errorObj);
// Update FMDM with error status
this.fmdmService.updateFolderStatus(path, 'error', {
message: errorObj.message,
type: 'error'
});
throw errorObj;
}
}
/**
* Execute the actual folder addition operation
* Simplified: Just add to FMDM and queue for processing
* The queue will handle model downloading/loading sequentially
*/
private async executeAddFolder(path: string, model: string): Promise<void> {
try {
// Normalize path key for consistent Map operations
const pathKey = this.normalizePathKey(path);
this.logger.debug(`[ORCHESTRATOR] Adding folder to queue: ${path} with model ${model}`);
// Validate model exists in curated models
const fmdm = this.fmdmService.getFMDM();
const modelInfo = fmdm.curatedModels.find(m => m.id === model);
if (!modelInfo) {
throw new Error(`Model ${model} not found in curated models`);
}
// Add folder to FMDM as pending
this.addFolderToFMDM(path, model, 'pending');
// Emit activity event for folder added
this.activityService?.emit({
type: 'indexing',
level: 'info',
message: `Folder added: ${this.extractFolderName(path)}`,
userInitiated: true,
details: [path]
});
// Start folder scanning (queue will handle model download/load if needed)
await this.startFolderScanning(path, model);
} catch (error) {
const errorObj = error instanceof Error ? error : new Error(String(error));
this.logger.error(`[ORCHESTRATOR] Failed to add folder ${path}`, errorObj);
// Update FMDM with error status and message
// Folder already added to FMDM as 'pending' at line 356
// Model validation occurs at line 347
this.fmdmService.updateFolderStatus(path, 'error', {
message: errorObj.message,
type: 'error'
});
throw errorObj;
}
}
/**
* Start folder scanning after model is available
*/
private async startFolderScanning(path: string, model: string, saveToConfig: boolean = true): Promise<void> {
try {
// Normalize path key for consistent Map operations
const pathKey = this.normalizePathKey(path);
this.logger.info(`🔍 [FILE-WATCH-TRACE] startFolderScanning() called for ${path}`);
// Get the actual model dimensions from curated models
const modelDimension = getModelDimensions(model);
this.logger.debug(`[ORCHESTRATOR] Model ${model} has dimension ${modelDimension}`);
// Create SQLite storage for this folder
const storage = new SQLiteVecStorage({
folderPath: path,
modelName: model,
modelDimension: modelDimension,
logger: this.logger
});
// Ensure .folder-mcp directory exists before creating FileStateService
const fs = await import('fs');
const folderMcpDir = `${path}/.folder-mcp`;
if (!fs.existsSync(folderMcpDir)) {
fs.mkdirSync(folderMcpDir, { recursive: true });
this.logger.debug(`[ORCHESTRATOR] Created .folder-mcp directory: ${folderMcpDir}`);
}
// We need to ensure the database is initialized before sharing the connection
// The DatabaseManager in storage initializes on first use, so we need to trigger it
// We'll call getDatabaseManager which will initialize if needed
const dbManager = storage.getDatabaseManager();
await dbManager.initialize(); // This ensures database and tables are created
// Create per-folder FileStateService using the SAME database connection as embeddings
// This is critical to avoid WAL isolation issues
const database = dbManager.getDatabase();
const folderFileStateService = new FileStateService(database, this.logger);
// Reset any files stuck in PROCESSING state from previous interrupted runs
// This allows them to be re-indexed on daemon restart
const resetCount = await folderFileStateService.resetProcessingFiles();
if (resetCount > 0) {
this.logger.info(`[ORCHESTRATOR] Reset ${resetCount} files from PROCESSING to PENDING for ${path}`);
}
// Get max concurrent files from ONNX configuration
const maxConcurrentFiles = await this.onnxConfiguration.getMaxConcurrentFiles();
this.logger.debug(`[ORCHESTRATOR] Using maxConcurrentFiles=${maxConcurrentFiles} from ONNX configuration`);
// Use factory function to create folder lifecycle manager
const folderManager = createFolderLifecycleService(
`folder-${Date.now()}`, // Generate unique ID
path,
this.indexingOrchestrator,
this.fileSystemService,
storage,
folderFileStateService, // Use per-folder service instead of global
this.logger,
model, // Pass the model parameter
maxConcurrentFiles // Pass the configured max concurrent files
);
// Subscribe to manager events
this.logger.info(`🔍 [FILE-WATCH-TRACE] Calling subscribeFolderEvents() for ${path}`);
this.subscribeFolderEvents(path, folderManager);
// Store manager and storage for periodic sync (use normalized path key)
this.folderManagers.set(pathKey, folderManager);
this.folderStorages.set(pathKey, storage);
// Update FMDM for initial pending state
this.updateFMDM();
// Add folder to the sequential indexing queue instead of starting directly
// The queue will handle model loading/unloading and sequential processing
await this.folderIndexingQueue.addFolder(path, model, folderManager);
// Note: File watching will be started when the folder completes indexing
// This is handled in the subscribeFolderEvents method
// Save folder to configuration for persistence across daemon restarts (if requested)
if (saveToConfig) {
try {
// Use ConfigurationComponent's addFolder method instead of direct get/set
await this.configService.addFolder(path, model);
this.logger.info(`[ORCHESTRATOR] Saved folder to configuration: ${path}`);
} catch (error) {
// Check if it's a duplicate folder error (already exists)
const errorMessage = error instanceof Error ? error.message : String(error);
if (errorMessage.includes('already exists')) {
this.logger.debug(`[ORCHESTRATOR] Folder already in configuration: ${path}`);
} else {
this.logger.warn(`[ORCHESTRATOR] Failed to save folder to configuration: ${path}`, error as Error);
}
// Don't fail the entire operation if config save fails
}
} else {
this.logger.debug(`[ORCHESTRATOR] Skipping config save for startup folder: ${path}`);
}
this.logger.info(`[ORCHESTRATOR] Added folder to monitoring: ${path}`);
} catch (error) {
this.logger.error(`[ORCHESTRATOR] Failed to start folder scanning: ${path}`, error as Error);
// Check if this is a Python prerequisite error and format appropriately for FMDM
let errorMessage = error instanceof Error ? error.message : String(error);
// Handle Python prerequisite errors specifically
if (errorMessage.includes('Python 3.8+ required for')) {
// Error message is already formatted correctly from PythonEmbeddingService
this.logger.warn(`[ORCHESTRATOR] Python prerequisite error for ${path}: ${errorMessage}`);
} else if (errorMessage.includes('Python embedding dependencies not available')) {
// Error message is already formatted correctly from PythonEmbeddingService
this.logger.warn(`[ORCHESTRATOR] Python dependency error for ${path}: ${errorMessage}`);
} else if (errorMessage.includes('Python process failed to start') || errorMessage.includes('Failed to start Python process')) {
// Generic Python process failure - enhance with model information
try {
const { getModelMetadata } = require('../../interfaces/tui-ink/models/modelMetadata.js');
const metadata = getModelMetadata(model);
const modelDisplayName = metadata?.displayName || model;
errorMessage = `Python 3.8+ required for ${modelDisplayName}`;
} catch {
// Fallback if metadata is not available
errorMessage = `Python 3.8+ required for ${model}`;
}
this.logger.warn(`[ORCHESTRATOR] Enhanced Python error message for ${path}: ${errorMessage}`);
}
// Create error folder config for tracking
const errorFolderConfig: FolderConfig = {
path: path,
model: model,
status: 'error',
progress: 0,
notification: this.createErrorNotification(errorMessage)
};
// Update FMDM directly with the error folder state
// We need to add it directly since it may not exist in FMDM yet
// Get current folders from FMDM (not just from managers)
const currentFMDM = this.fmdmService.getFMDM();
const currentFolders = [...currentFMDM.folders];
// Check if folder already exists in current folders
const existingIndex = currentFolders.findIndex(f => this.pathsEqual(f.path, path));
if (existingIndex >= 0) {
// Update existing folder
currentFolders[existingIndex] = errorFolderConfig;
} else {
// Add new folder with error status
currentFolders.push(errorFolderConfig);
}
this.fmdmService.updateFolders(currentFolders);
// Perform resource cleanup on execution failure
// Pass the original error so we can detect environment errors and preserve data
const errorObj = error instanceof Error ? error : new Error(String(error));
await this.performResourceCleanup(path, 'add_folder_execution_failed', errorObj);
throw error;
}
}
/**
* Add folder to FMDM with initial status
* This ensures the folder appears in FMDM before any status updates
*/
private addFolderToFMDM(path: string, model: string, status: FolderIndexingStatus): void {
// Get current FMDM folders
const currentFMDM = this.fmdmService.getFMDM();
const currentFolders = [...currentFMDM.folders];
// Check if folder already exists (case-insensitive on Windows, case-sensitive on Unix)
const isWindows = process.platform === 'win32';
// Normalize path for comparison:
// 1. On Windows: lowercase for case-insensitive comparison
// 2. On Windows: replace backslashes with forward slashes for consistent separator
// 3. On Unix: keep original (case-sensitive filesystem)
const normalizedPath = isWindows
? path.toLowerCase().replace(/\\/g, '/')
: path;
const existingIndex = currentFolders.findIndex(f => {
const folderPathToCompare = isWindows
? f.path.toLowerCase().replace(/\\/g, '/')
: f.path;
return folderPathToCompare === normalizedPath;
});
if (existingIndex >= 0) {
// Update existing folder but preserve original path casing
// On Windows, C:\Users\Test and c:\users\test are the same folder
// We keep the original casing to maintain consistency
const originalPath = currentFolders[existingIndex]!.path;
currentFolders[existingIndex] = {
path: originalPath, // Preserve original path casing
model,
status,
progress: 0
};
this.logger.debug(`[ORCHESTRATOR] Updated existing folder in FMDM (preserved casing): ${originalPath}`);
} else {
// Add new folder with provided path
currentFolders.push({
path,
model,
status,
progress: 0
});
this.logger.debug(`[ORCHESTRATOR] Added new folder to FMDM: ${path}`);
}
// Update FMDM with the new folder list
this.fmdmService.updateFolders(currentFolders);
}
/**
* PUBLIC: Add folder to FMDM immediately with 'pending' status
* Called synchronously by WebSocket handler to prevent race conditions
* This ensures folder exists in FMDM before any async operations begin
*/
addFolderToFMDMImmediate(path: string, model: string): void {
this.addFolderToFMDM(path, model, 'pending');
}
async removeFolder(folderPath: string): Promise<void> {
// First, try to remove from the queue if it's pending
const removedFromQueue = this.folderIndexingQueue.removeFolder(folderPath);
if (removedFromQueue) {
this.logger.info(`[ORCHESTRATOR] Removed folder from indexing queue: ${folderPath}`);
}
// Use normalized path key for Map lookup
const pathKey = this.normalizePathKey(folderPath);
const manager = this.folderManagers.get(pathKey);
// IMMEDIATELY remove from folderManagers to prevent race conditions
// This ensures getCurrentFolderConfigs() won't include this folder in FMDM updates
// during async cleanup operations
if (manager) {
this.folderManagers.delete(pathKey);
this.folderStorages.delete(pathKey);
this.logger.info(`[ORCHESTRATOR] Immediately removed folder from tracking: ${folderPath}`);
// CRITICAL: Remove folder from FMDM DIRECTLY before calling updateFMDM()
// This prevents getCurrentFolderConfigs() from re-adding it via the preservation logic
// (which preserves FMDM folders without managers - but we want this one GONE)
const currentFMDM = this.fmdmService.getFMDM();
const filteredFolders = currentFMDM.folders.filter(f => !this.pathsEqual(f.path, folderPath));
this.logger.debug(`[ORCHESTRATOR] Removing folder from FMDM directly: ${folderPath} (${currentFMDM.folders.length} -> ${filteredFolders.length} folders)`);
this.fmdmService.updateFolders(filteredFolders);
// Now updateFMDM won't re-add it since it's gone from FMDM
this.updateFMDM();
}
if (!manager) {
this.logger.warn(`No manager found for folder: ${folderPath}`);
// Even without a manager, we need to:
// 1. Remove from configuration
// 2. Update FMDM to reflect removal
// This handles error state folders or folders that failed to start
} else {
await manager.stop();
}
// Only do Windows delay and file watching cleanup if there was a manager
if (manager) {
// On Windows, add a small delay to ensure database connections are fully released
// This prevents "EBUSY: resource busy or locked" errors when deleting the .folder-mcp directory
const isWindows = process.platform === 'win32';
if (isWindows) {
this.logger.debug(`[ORCHESTRATOR] Windows detected - waiting for database locks to be released...`);
await new Promise(resolve => setTimeout(resolve, 2000)); // 2000ms delay for Windows file lock release
}
// Stop file watching if it was started
if (this.monitoringOrchestrator) {
try {
await this.monitoringOrchestrator.stopFileWatching(folderPath);
this.logger.info(`Stopped file watching for removed folder: ${folderPath}`);
} catch (error) {
this.logger.warn(`Failed to stop file watching for ${folderPath}`, error as Error);
}
}
}
// Clean up .folder-mcp directory and its contents (even without manager)
try {
const folderMcpPath = `${folderPath}/.folder-mcp`;
const fs = await import('fs');
// Check if .folder-mcp directory exists
if (fs.existsSync(folderMcpPath)) {
this.logger.info(`Cleaning up .folder-mcp directory: ${folderMcpPath}`);
// Remove the entire .folder-mcp directory and its contents
await fs.promises.rm(folderMcpPath, { recursive: true, force: true });
this.logger.info(`Successfully removed .folder-mcp directory: ${folderMcpPath}`);
} else {
this.logger.debug(`No .folder-mcp directory found at: ${folderMcpPath}`);
}
} catch (error) {
this.logger.error(`Failed to clean up .folder-mcp directory for ${folderPath}:`, error as Error);
// Don't fail the entire removal process if cleanup fails
}
// Remove from configuration (even without manager)
try {
// Use ConfigurationComponent's removeFolder method
await this.configService.removeFolder(folderPath);
this.logger.info(`[ORCHESTRATOR] Removed folder from configuration: ${folderPath}`);
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
if (errorMessage.includes('not found')) {
this.logger.debug(`[ORCHESTRATOR] Folder not in configuration: ${folderPath}`);
} else {
this.logger.warn(`[ORCHESTRATOR] Failed to remove folder from configuration: ${folderPath}`, error as Error);
}
// Don't fail the entire operation if config removal fails
}
// For error state folders without managers, we need to explicitly remove from FMDM
// (Manager was already removed immediately at the start of removeFolder)
if (!manager) {
this.logger.debug(`[ORCHESTRATOR-REMOVE] No manager to delete, removing error folder from FMDM directly`);
const currentFMDM = this.fmdmService.getFMDM();
const filteredFolders = currentFMDM.folders.filter(f => !this.pathsEqual(f.path, folderPath));
this.fmdmService.updateFolders(filteredFolders);
}
// Final FMDM update after all cleanup
this.logger.debug(`[ORCHESTRATOR-REMOVE] Final updateFMDM after folder removal complete`);
this.updateFMDM();
// Emit activity event for folder removed
this.activityService?.emit({
type: 'indexing',
level: 'info',
message: `Folder removed: ${this.extractFolderName(folderPath)}`,
userInitiated: true,
details: [folderPath]
});
this.logger.info(`Removed folder from monitoring: ${folderPath}`);
}
getManager(folderPath: string): IFolderLifecycleManager | undefined {
const pathKey = this.normalizePathKey(folderPath);
return this.folderManagers.get(pathKey);
}
/**
* Get the model download manager (for setting cache callbacks)
*/
getModelDownloadManager(): IModelDownloadManager {
return this.modelDownloadManager;
}
/**
* Get the folder indexing queue (for MCP search operations with keep-alive)
*/
getQueue(): FolderIndexingQueue {
return this.folderIndexingQueue;
}
/**
* Record connection event (no-op after telemetry removal)
*/
recordConnection(duration?: number, isError = false): void {
// No-op: telemetry removed
}
/**
* Record query performance (no-op after telemetry removal)
*/
recordQuery(durationMs: number, cacheHit = false): void {
// No-op: telemetry removed
}
/**
* Get performance telemetry statistics (no-op after telemetry removal)
*/
getTelemetryStatistics(): any {
return null; // Telemetry removed
}
async startAll(): Promise<void> {
const startupStartTime = Date.now();
this.logger.info('Starting all configured folders...');
// Wait for Python to be initialized if any GPU folders are configured
if (this.pythonInitializationPromise) {
this.logger.info('Waiting for Python embedding service to initialize...');
try {
await this.pythonInitializationPromise;
this.logger.info('Python embedding service ready');
} catch (error) {
this.logger.warn('Python embedding service initialization failed, GPU models will not be available', error as Error);
}
}
// Get current FMDM state to see which folders are already loaded
const currentFMDM = this.fmdmService.getFMDM();
const fmdmFolders = currentFMDM.folders.map(f => f.path);
this.logger.debug(`FMDM already contains ${fmdmFolders.length} folders: ${fmdmFolders.join(', ')}`);
// Start lifecycle management for folders already in FMDM
if (fmdmFolders.length > 0) {
this.logger.info(`Starting lifecycle management for ${fmdmFolders.length} folders already in FMDM`);
for (const folderConfig of currentFMDM.folders) {
try {
// Skip folders that already have managers (shouldn't happen but safety check)
const pathKey = this.normalizePathKey(folderConfig.path);
if (this.folderManagers.has(pathKey)) {
this.logger.debug(`Manager already exists for ${folderConfig.path}, skipping`);
continue;
}
this.logger.info(`Starting lifecycle for FMDM folder: ${folderConfig.path} with model: ${folderConfig.model}`);
// Start folder lifecycle without adding to config (it's already there)
await this.startFolderLifecycle(folderConfig.path, folderConfig.model);
} catch (error) {
this.logger.error(`Failed to start lifecycle for FMDM folder ${folderConfig.path}:`, error as Error);
// Continue with other folders even if one fails
}
}
this.logger.info(`Completed starting lifecycle for ${fmdmFolders.length} FMDM folders`);
} else {
this.logger.info('No folders found in FMDM to start lifecycle management');
}
// Log startup performance telemetry
const startupDuration = Date.now() - startupStartTime;
this.logger.info('Orchestrator startup completed', {
startupDurationMs: startupDuration,
foldersManaged: this.folderManagers.size,
telemetryEnabled: false,
memoryMonitorEnabled: false
});
}
async stopAll(): Promise<void> {
const shutdownStartTime = Date.now();
this.logger.info(`Stopping all ${this.folderManagers.size} folder managers`);
// Stop folder validation timer
this.stopFolderValidation();
// Stop periodic filesystem sync
this.stopPeriodicSync();
// Stop the folder indexing queue first
// This will stop current processing and clear the queue
try {
this.logger.info('[ORCHESTRATOR] Stopping folder indexing queue');
await this.folderIndexingQueue.stop();
this.logger.info('[ORCHESTRATOR] Folder indexing queue stopped');
} catch (error) {
this.logger.error('[ORCHESTRATOR] Error stopping folder indexing queue:', error instanceof Error ? error : new Error(String(error)));
}
// Memory monitoring and telemetry removed
// Shutdown resource manager first to stop accepting new operations
try {
this.logger.info('[ORCHESTRATOR] Shutting down resource manager');
await this.resourceManager.shutdown(false);
this.logger.info('[ORCHESTRATOR] Resource manager shutdown complete');
} catch (error) {
this.logger.error('[ORCHESTRATOR] Error shutting down resource manager:', error instanceof Error ? error : new Error(String(error)));
}
for (const [path, manager] of this.folderManagers) {
try {
await manager.stop();
// Stop file watching if it was started
if (this.monitoringOrchestrator) {
try {
await this.monitoringOrchestrator.stopFileWatching(path);
} catch (error) {
this.logger.warn(`Failed to stop file watching for ${path}`, error as Error);
}
}
} catch (error) {
this.logger.error(`Error stopping manager for ${path}`, error as Error);
}
}
this.folderManagers.clear();
// Clear the model factory cache to ensure clean shutdown
try {
this.logger.info('[ORCHESTRATOR] Clearing model factory cache');
this.modelFactory.clearCache();
this.logger.info('[ORCHESTRATOR] Model factory cache cleared');
} catch (error) {
this.logger.error('[ORCHESTRATOR] Error clearing model factory cache:', error instanceof Error ? error : new Error(String(error)));
}
// Log shutdown performance telemetry
const shutdownDuration = Date.now() - shutdownStartTime;
this.logger.info('Orchestrator shutdown completed', {
shutdownDurationMs: shutdownDuration,
previouslyManagedFolders: 0, // folderManagers was cleared
telemetryWasEnabled: false,
memoryMonitorWasEnabled: false
});
}
/**
* Subscribe to events from a folder manager
*/
private subscribeFolderEvents(folderPath: string, manager: IFolderLifecycleManager): void {
this.logger.info(`🔍 [FILE-WATCH-TRACE] subscribeFolderEvents() setting up event listeners for ${folderPath}`);
// Listen for state changes
manager.on('stateChange', (state) => {
this.logger.debug(`[ORCHESTRATOR] Folder ${folderPath} state changed to: ${state.status}`);
this.onFolderStateChange(folderPath, state);
});
// Listen for scan completion
manager.on('scanComplete', (state) => {
this.logger.debug(`[ORCHESTRATOR] Folder ${folderPath} scan complete with ${state.fileEmbeddingTasks?.length || 0} tasks`);
this.onScanComplete(folderPath, manager, state);
});
// Listen for index completion
manager.on('indexComplete', (state) => {
this.logger.info(`🔍 [FILE-WATCH-TRACE] indexComplete event fired for ${folderPath}`);
this.logger.debug(`[ORCHESTRATOR] Folder ${folderPath} indexing complete`);
this.onIndexComplete(folderPath, state);
});
// Listen for changes detected
manager.on('changesDetected', () => {
this.logger.debug(`[ORCHESTRATOR] Folder ${folderPath} detected changes`);
this.onChangesDetected(folderPath, manager);
});
// Listen for progress updates
manager.on('progressUpdate', (progress) => {
this.logger.debug(`[ORCHESTRATOR] Folder ${folderPath} progress: ${progress.percentage}%`);
this.fmdmService.updateFolderProgress(folderPath, progress.percentage);
});
// Listen for errors
manager.on('error', (error) => {
this.logger.error(`[ORCHESTRATOR] Folder ${folderPath} error:`, error);
this.onFolderError(folderPath, error);
});
}
/**
* Handle folder state change
*/
private onFolderStateChange(folderPath: string, state: any): void {
// Update FMDM whenever state changes
this.updateFMDM();
}
/**
* Handle scan completion - decide whether to start indexing
*/
private async onScanComplete(folderPath: string, manager: IFolderLifecycleManager, state: any): Promise<void> {
if (state.fileEmbeddingTasks && state.fileEmbeddingTasks.length > 0) {
// Has tasks, start indexing
this.logger.info(`[ORCHESTRATOR] Starting indexing for ${folderPath} with ${state.fileEmbeddingTasks.length} tasks`);
await manager.startIndexing();
} else {
// No tasks, already in active state
this.logger.info(`[ORCHESTRATOR] No tasks for ${folderPath}, already active`);
// Check for Windows performance issues when folder becomes active
await this.checkWindowsPerformanceForFolder(folderPath, state.model);
}
// Update FMDM when indexing starts or folder becomes active
this.updateFMDM();
}
/**
* Helper to create error notification from error message
*/
private createErrorNotification(message: string): { message: string; type: 'error' } {
return { message, type: 'error' };
}
/**
* Check Windows performance issues for a folder that uses Python models
*/
private async checkWindowsPerformanceForFolder(folderPath: string, model?: string): Promise<void> {
if (!model) {
return; // No model information available
}
try {
const performanceResult = await this.windowsPerformanceService.detectPerformanceIssues(model);
if (performanceResult.shouldShowWarning && performanceResult.warningMessage) {
this.logger.info(`[ORCHESTRATOR] Windows performance warning for ${folderPath}: ${performanceResult.warningMessage}`);
// Set notification in FMDM for this folder
this.fmdmService.updateFolderNotification(folderPath, {
message: performanceResult.warningMessage,
type: 'warning'
});
}
} catch (error) {
// Don't fail the folder activation if performance check fails
this.logger.debug(`[ORCHESTRATOR] Windows performance check failed for ${folderPath}`, { error: error instanceof Error ? error.message : String(error) });
}
}
/**
* Handle index completion - start file watching when folder becomes active
*/
private async onIndexComplete(folderPath: string, state: any): Promise<void> {
this.logger.info(`🔍 [FILE-WATCH-TRACE] onIndexComplete() called for ${folderPath}`);
this.logger.info(`[ORCHESTRATOR] Indexing complete for ${folderPath}, starting file watching`);
// Create and emit FMDM informational message for active state
if (state.indexingStats) {
const { fileCount, indexingTimeSeconds } = state.indexingStats;
const infoMessage = `▶ ${folderPath} [active] i ${fileCount} files indexed • indexing time ${indexingTimeSeconds}s`;
this.logger.info(`[ORCHESTRATOR] ${infoMessage}`);
// Update folder status to 'active' with permanent completion notification
this.fmdmService.updateFolderStatus(folderPath, 'active', {
message: `${fileCount} files indexed • indexing time ${indexingTimeSeconds}s`,
type: 'info'
});
} else {
this.logger.debug(`[ORCHESTRATOR] No indexing statistics available for ${folderPath}`);
}
// Check for Windows performance issues when folder becomes active after indexing
await this.checkWindowsPerformanceForFolder(folderPath, state.model);
// Only start file watching if not already watching
// This prevents duplicate watcher creation on re-indexing events
if (this.monitoringOrchestrator) {
const watchStatus = await this.monitoringOrchestrator.getWatchingStatus(folderPath);
if (watchStatus.isActive) {
this.logger.debug(`[ORCHESTRATOR] File watching already active for ${folderPath}, skipping`);
this.updateFMDM();
return;
}
}
await this.startFileWatchingForFolder(folderPath);
// Update FMDM
this.updateFMDM();
}
/**
* Start file watching for a specific folder
*/
private async startFileWatchingForFolder(folderPath: string): Promise<void> {
try {
this.logger.info(`🔍 [FILE-WATCH-TRACE] startFileWatchingForFolder() called for ${folderPath}`);
// Create MonitoringOrchestrator if we don't have one
if (!this.monitoringOrchestrator) {
this.logger.info(`🔍 [FILE-WATCH-TRACE] Creating new MonitoringOrchestrator instance`);
this.logger.debug(`[ORCHESTRATOR] Creating MonitoringOrchestrator for file watching`);
const { MonitoringOrchestrator } = await import('../../application/monitoring/orchestrator.js');
// Create dummy services for MonitoringOrchestrator - we only need file watching
const dummyFileParsingService = {
parseFile: async () => ({ success: true, chunks: [] }),
getSupportedFormats: () => ['txt', 'md', 'pdf']
};
const dummyCacheService = {
getCacheStatus: async () => ({ status: 'ready' }),
clearCache: async () => true
};
const dummyConfigService = this.configService;
const dummyIncrementalIndexer = {
indexChanges: async () => ({
success: true,
filesProcessed: 0,
chunksGenerated: 0,
embeddingsCreated: 0,
errors: []
})
};
this.monitoringOrchestrator = new MonitoringOrchestrator(
dummyFileParsingService as any,
dummyCacheService as any,
this.logger,
dummyConfigService as any,
dummyIncrementalIndexer as any
);
// Set up change detection callback
this.monitoringOrchestrator.setChangeDetectionCallback((folderPath: string, changeCount: number) => {
this.logger.info(`[ORCHESTRATOR] File changes detected in ${folderPath} (${changeCount} changes)`);
const pathKey = this.normalizePathKey(folderPath);
const manager = this.folderManagers.get(pathKey);
if (manager) {
// Directly trigger the change detection handling
this.onChangesDetected(folderPath, manager);
} else {
this.logger.warn(`[ORCHESTRATOR] No manager found for changed folder: ${folderPath}`);
}
});
}
// Start file watching for this folder
const watchingOptions = {
debounceMs: 2000, // 2 second debounce
enableBatchProcessing: true,
batchSize: 10,
includeFileTypes: getSupportedExtensions(),
excludePatterns: [
'**/node_modules/**',
'**/.git/**',
'**/.folder-mcp/**'
]
};
this.logger.info(`🔍 [FILE-WATCH-TRACE] Calling monitoringOrchestrator.startFileWatching() for ${folderPath}`);
const watchResult = await this.monitoringOrchestrator.startFileWatching(folderPath, watchingOptions);
this.logger.info(`🔍 [FILE-WATCH-TRACE] startFileWatching() returned: success=${watchResult.success}`);
if (watchResult.success) {
this.logger.info(`[ORCHESTRATOR] File watching started successfully for ${folderPath}`, {
watchId: watchResult.watchId,
startedAt: watchResult.startedAt
});
} else {
this.logger.error(`[ORCHESTRATOR] Failed to start file watching for ${folderPath}: ${watchResult.error}`);
}
} catch (error) {
this.logger.error(`[ORCHESTRATOR] Error starting file watching for ${folderPath}:`, error instanceof Error ? error : new Error(String(error)));
}
}
/**
* Handle changes detected - re-queue folder for full indexing
*/
private async onChangesDetected(folderPath: string, manager: IFolderLifecycleManager): Promise<void> {
this.logger.info(`[ORCHESTRATOR] File changes detected in ${folderPath}`);
// Get the folder configuration from FMDM
const currentFMDM = this.fmdmService.getFMDM();
const folderConfig = currentFMDM.folders.find(f => this.pathsEqual(f.path, folderPath));
if (!folderConfig) {
this.logger.error(`[ORCHESTRATOR] Cannot re-queue ${folderPath} - folder not found in FMDM`);
return;
}
// Check if folder just completed indexing (within last 5 seconds)
// This prevents re-queuing due to file watcher events triggered by the indexing process itself
const managerState = manager.getState();
const lastCompletionTime = managerState.lastIndexingCompletedAt;
if (lastCompletionTime) {
const timeSinceCompletion = Date.now() - lastCompletionTime;
const GRACE_PERIOD_MS = 5000; // 5 seconds
if (timeSinceCompletion < GRACE_PERIOD_MS) {
this.logger.info(
`[ORCHESTRATOR] Ignoring re-queue request for ${folderPath} - ` +
`just completed indexing ${timeSinceCompletion}ms ago (within ${GRACE_PERIOD_MS}ms grace period)`
);
return;
}
}
// Emit activity event for folder change detected
this.activityService?.emit({
type: 'indexing',
level: 'info',
message: `Folder changed, initiating re-indexing: ${this.extractFolderName(folderPath)}`,
userInitiated: false,
details: [folderPath]
});
// Re-queue the folder for full indexing (same as daemon restart behavior)
try {
this.logger.info(`[ORCHESTRATOR] Re-queuing folder ${folderPath} for indexing due to file changes`);
// Reset the manager to pending state to allow re-processing
manager.reset();
// Add folder to the indexing queue with the existing manager - this will trigger full indexing
await this.folderIndexingQueue.addFolder(
folderPath,
folderConfig.model,
manager,
'normal'
);
this.logger.info(`[ORCHESTRATOR] Successfully re-queued ${folderPath} for full indexing`);
} catch (error) {
this.logger.error(`[ORCHESTRATOR] Failed to re-queue ${folderPath} for indexing:`, error instanceof Error ? error : new Error(String(error)));
// Don't throw here - just log the error as this is a background operation
}
}
/**
* Handle folder error
*/
private onFolderError(folderPath: string, error: Error): void {
this.logger.error(`[ORCHESTRATOR] Folder error for ${folderPath}:`, error);
// Check if this is a Python prerequisite error and format appropriately for FMDM
let errorMessage = error.message;
// Handle Python prerequisite errors specifically
if (errorMessage.includes('Python 3.8+ required for')) {
// Error message is already formatted correctly from PythonEmbeddingService
this.logger.warn(`[ORCHESTRATOR] Python prerequisite error for ${folderPath}: ${errorMessage}`);
} else if (errorMessage.includes('Python embedding dependencies not available')) {
// Error message is already formatted correctly from PythonEmbeddingService
this.logger.warn(`[ORCHESTRATOR] Python dependency error for ${folderPath}: ${errorMessage}`);
} else if (errorMessage.includes('Python process failed to start') || errorMessage.includes('Failed to start Python process')) {
// Generic Python process failure - enhance with model information
const pathKey = this.normalizePathKey(folderPath);
const manager = this.folderManagers.get(pathKey);
if (manager) {
const state = manager.getState();
const model = state.model || 'unknown';
try {
const { getModelMetadata } = require('../../interfaces/tui-ink/models/modelMetadata.js');
const metadata = getModelMetadata(model);
const modelDisplayName = metadata?.displayName || model;
errorMessage = `Python 3.8+ required for ${modelDisplayName}`;
} catch {
// Fallback if metadata is not available
errorMessage = `Python 3.8+ required for ${model}`;
}
this.logger.warn(`[ORCHESTRATOR] Enhanced Python error message for ${folderPath}: ${errorMessage}`);
}
}
// Use FMDM service to update folder status with specific error message
const pathKey = this.normalizePathKey(folderPath);
const manager = this.folderManagers.get(pathKey);
if (manager) {
const state = manager.getState();
const model = state.model || 'unknown';
// Create error folder config with enhanced error message
const errorFolderConfig: FolderConfig = {
path: folderPath,
model: model,
status: 'error',
progress: 0,
notification: this.createErrorNotification(errorMessage)
};
// Error state is tracked in the folder manager itself
}
// Update FMDM with error state
this.updateFMDM();
}
/**
* Get current folder configs for FMDM
*/
private getCurrentFolderConfigs(): FolderConfig[] {
const folders: FolderConfig[] = [];
// Get current FMDM state to preserve existing notifications
const currentFMDM = this.fmdmService.getFMDM();
// Add folders with managers
for (const [path, manager] of this.folderManagers) {
const state = manager.getState();
const folderConfig: FolderConfig = {
path,
model: state.model || 'unknown',
status: state.status,
...(state.errorMessage && { notification: this.createErrorNotification(state.errorMessage) })
};
// Add indexing progress (for indexing phase and completed active folders)
if (state.status === 'indexing') {
folderConfig.progress = state.progress?.percentage;
// Add progress message as notification for better UX
if (state.progressMessage) {
folderConfig.notification = {
message: state.progressMessage,
type: 'info'
};
}
} else if (state.status === 'active') {
folderConfig.progress = 100; // Active folders are 100% complete
// Preserve completion notifications for active folders from current FMDM
const existingFolder = currentFMDM.folders.find(f => this.pathsEqual(f.path, path));
if (existingFolder?.notification) {
// Only preserve completion notifications (contain "files indexed")
// Clear progress notifications (contain "processing" or "in progress")
if (existingFolder.notification.message.includes('files indexed')) {
folderConfig.notification = existingFolder.notification;
}
}
} else {
// Clear progress notification for other states
if (folderConfig.notification?.type === 'info' && !state.errorMessage) {
delete folderConfig.notification;
}
}
// Add scanning progress (only for scanning phase)
if (state.status === 'scanning' && state.scanningProgress) {
folderConfig.scanningProgress = {
phase: state.scanningProgress.phase,
processedFiles: state.scanningProgress.processedFiles,
totalFiles: state.scanningProgress.totalFiles,
percentage: state.scanningProgress.percentage,
};
}
folders.push(folderConfig);
}
// CRITICAL FIX: Preserve folders from FMDM that don't have managers yet
// This prevents race conditions where folders are removed from FMDM before their managers are created
// Folders can exist in FMDM during pending/downloading-model states before managers are initialized
// Create a Set of folder paths already in the array to prevent duplicates
// IMPORTANT: Normalize to lowercase on Windows only (case-insensitive), keep original on Unix (case-sensitive)
const isWindows = process.platform === 'win32';
const existingPaths = new Set(folders.map(f => isWindows ? f.path.toLowerCase() : f.path));
for (const fmdmFolder of currentFMDM.folders) {
// Normalize path to lowercase on Windows only for case-insensitive comparison
const normalizedPath = isWindows ? fmdmFolder.path.toLowerCase() : fmdmFolder.path;
const isInExistingPaths = existingPaths.has(normalizedPath);
const pathKey = this.normalizePathKey(fmdmFolder.path);
const hasManager = this.folderManagers.has(pathKey);
// Skip if this folder is already in the array (deduplication)
if (isInExistingPaths) {
continue;
}
// Check if this folder already has a manager
if (!hasManager) {
// This folder is in FMDM but doesn't have a manager yet - preserve it
folders.push(fmdmFolder);
existingPaths.add(normalizedPath); // Add normalized path to prevent future duplicates
}
}
// No need to add errorFolders separately - error state is tracked in folderManagers
return folders;
}
/**
* Update FMDM with current state of all folders
*/
private updateFMDM(): void {
const folders = this.getCurrentFolderConfigs();
this.logger.debug(`[ORCHESTRATOR-FMDM] Updating FMDM with ${folders.length} folders`);
// Update FMDM with all folder states
this.fmdmService.updateFolders(folders);
}
/**
* Start periodic folder validation to detect deleted folders
*/
private startFolderValidation(): void {
this.logger.debug('[ORCHESTRATOR] Starting periodic folder validation');
this.folderValidationTimer = setInterval(async () => {
await this.validateAllFolders();
}, 30000); // Check every 30 seconds
}
/**
* Stop folder validation timer
*/
private stopFolderValidation(): void {
if (this.folderValidationTimer) {
clearInterval(this.folderValidationTimer);
delete this.folderValidationTimer;
this.logger.debug('[ORCHESTRATOR] Stopped folder validation timer');
}
}
/**
* Start periodic filesystem sync
*/
private startPeriodicSync(): void {
this.logger.info('[ORCHESTRATOR] Starting periodic filesystem sync');
this.periodicSyncService.start(async () => {
this.logger.info(`[PERIODIC-SYNC] Executing periodic sync for ${this.folderManagers.size} folders`);
// Sync all folders
for (const [folderPath, manager] of this.folderManagers) {
const storage = this.folderStorages.get(folderPath);
if (storage) {
await this.periodicSyncService.syncFolder(folderPath, manager, storage);
}
}
this.logger.info('[PERIODIC-SYNC] Periodic sync cycle completed');
});
}
/**
* Stop periodic filesystem sync
*/
private stopPeriodicSync(): void {
this.periodicSyncService.stop();
this.logger.debug('[ORCHESTRATOR] Stopped periodic filesystem sync');
}
/**
* Validate all monitored folders still exist
*/
private async validateAllFolders(): Promise<void> {
const foldersToMarkError: string[] = [];
for (const [folderPath, manager] of this.folderManagers) {
try {
const fs = await import('fs');
// Check if folder still exists
if (!fs.existsSync(folderPath)) {
this.logger.warn(`[ORCHESTRATOR] Monitored folder no longer exists: ${folderPath}`);
foldersToMarkError.push(folderPath);
}
} catch (error) {
this.logger.error(`[ORCHESTRATOR] Error validating folder ${folderPath}:`, error instanceof Error ? error : new Error(String(error)));
// If we can't validate, assume folder is problematic and mark with error
foldersToMarkError.push(folderPath);
}
}
// Mark non-existent folders with error status instead of removing them
for (const folderPath of foldersToMarkError) {
this.logger.info(`[ORCHESTRATOR] Marking deleted folder with error status: ${folderPath}`);
await this.markFolderAsError(folderPath, 'Folder no longer exists');
}
}
/**
* Mark folder as error and stop its lifecycle manager but keep it in tracking
*/
private async markFolderAsError(folderPath: string, errorMessage: string): Promise<void> {
const pathKey = this.normalizePathKey(folderPath);
const manager = this.folderManagers.get(pathKey);
if (!manager) {
this.logger.debug(`[ORCHESTRATOR] No manager found for folder error marking: ${folderPath}`);
return;
}
try {
// Stop the manager to halt any ongoing processes
await manager.stop();
this.logger.info(`[ORCHESTRATOR] Stopped lifecycle manager for error folder: ${folderPath}`);
// Stop file watching if it was started
if (this.monitoringOrchestrator) {
try {
await this.monitoringOrchestrator.stopFileWatching(folderPath);
this.logger.info(`[ORCHESTRATOR] Stopped file watching for error folder: ${folderPath}`);
} catch (error) {
this.logger.warn(`[ORCHESTRATOR] Failed to stop file watching for ${folderPath}`, error as Error);
}
}
// Remove from active managers but keep in error tracking
this.folderManagers.delete(folderPath);
// Get folder config from configuration to preserve model info
let folderConfig: FolderConfig;
try {
const existingConfig = await this.configService.getFolder(folderPath);
folderConfig = {
path: folderPath,
model: existingConfig?.model || getDefaultModelId(), // Fallback to dynamic default model
status: 'error',
notification: this.createErrorNotification(errorMessage)
};
} catch (error) {
// If we can't get the config, create a minimal one
folderConfig = {
path: folderPath,
model: getDefaultModelId(), // Dynamic default model
status: 'error',
notification: this.createErrorNotification(errorMessage)
};
}
// Update FMDM to show the folder with error status
this.updateFMDM();
this.logger.info(`[ORCHESTRATOR] Marked folder as error: ${folderPath} (${errorMessage})`);
} catch (error) {
this.logger.error(`[ORCHESTRATOR] Error marking folder as error ${folderPath}:`, error instanceof Error ? error : new Error(String(error)));
}
}
/**
* Internal folder removal with optional error message
*/
private async removeFolderInternal(folderPath: string, reason?: string): Promise<void> {
const pathKey = this.normalizePathKey(folderPath);
const manager = this.folderManagers.get(pathKey);
if (!manager) {
this.logger.debug(`[ORCHESTRATOR] No manager found for folder removal: ${folderPath}`);
return;
}
try {
// Stop the manager
await manager.stop();
// On Windows, add a small delay to ensure database connections are fully released
// This prevents "EBUSY: resource busy or locked" errors when deleting the .folder-mcp directory
const isWindows = process.platform === 'win32';
if (isWindows) {
this.logger.debug(`[ORCHESTRATOR] Windows detected - waiting for database locks to be released...`);
await new Promise(resolve => setTimeout(resolve, 500)); // 500ms delay
}
// Stop file watching if it was started
if (this.monitoringOrchestrator) {
try {
await this.monitoringOrchestrator.stopFileWatching(folderPath);
this.logger.info(`[ORCHESTRATOR] Stopped file watching for removed folder: ${folderPath}`);
} catch (error) {
this.logger.warn(`[ORCHESTRATOR] Failed to stop file watching for ${folderPath}`, error as Error);
}
}
// Remove from configuration (but don't fail if it's not there)
try {
await this.configService.removeFolder(folderPath);
this.logger.info(`[ORCHESTRATOR] Removed folder from configuration: ${folderPath}`);
} catch (error) {
this.logger.debug(`[ORCHESTRATOR] Folder not in configuration during cleanup: ${folderPath}`);
}
// Remove from our tracking
this.folderManagers.delete(folderPath);
// Update FMDM to remove the folder
this.updateFMDM();
this.logger.info(`[ORCHESTRATOR] Successfully removed folder: ${folderPath}${reason ? ` (${reason})` : ''}`);
} catch (error) {
this.logger.error(`[ORCHESTRATOR] Error during folder cleanup for ${folderPath}:`, error instanceof Error ? error : new Error(String(error)));
}
}
/**
* Get intelligent memory monitoring statistics (removed)
*/
getIntelligentMemoryStatistics(): {
baselineEstablished: boolean;
samplesCollected: number;
currentHeapUsedMB: number;
currentHeapUtilization: number;
baselineDeviation?: number;
monitoringDuration: number;
} {
// Memory monitoring removed
return {
baselineEstablished: false,
samplesCollected: 0,
currentHeapUsedMB: 0,
currentHeapUtilization: 0,
monitoringDuration: 0
};
}
/**
* Get current memory statistics for external monitoring
*/
getMemoryStatistics(): {
process: NodeJS.MemoryUsage;
rssMemoryMB: number;
resourceManager: ResourceStats;
folders: {
managed: number;
error: number;
};
} {
const memUsage = process.memoryUsage();
const rssMemoryMB = memUsage.rss / (1024 * 1024); // Use RSS for actual memory footprint
const resourceStats = this.resourceManager.getStats();
return {
process: memUsage,
rssMemoryMB: rssMemoryMB,
resourceManager: resourceStats,
folders: {
managed: this.folderManagers.size,
error: 0
}
};
}
/**
* Perform resource cleanup when operations fail
* This ensures that partially completed operations don't leave the system in an inconsistent state
*
* CRITICAL: For environment errors (native module mismatch, Python issues, etc.):
* - DO NOT delete the .folder-mcp directory (preserve indexed data!)
* - DO NOT remove from configuration (allow retry after environment fix)
* - Only clean up transient resources (managers, watchers, pending operations)
*
* @param folderPath Path to the folder being cleaned up
* @param reason Reason for cleanup
* @param originalError Optional - the error that triggered cleanup (used to detect environment errors)
*/
private async performResourceCleanup(folderPath: string, reason: string, originalError?: Error): Promise<void> {
// Check if this is an environment error that should preserve data
const preserveData = originalError ? isEnvironmentError(originalError) : false;
if (preserveData) {
this.logger.warn(`[ORCHESTRATOR] Environment error detected for ${folderPath} - preserving .folder-mcp directory and configuration`);
this.logger.warn(`[ORCHESTRATOR] To fix: ${getEnvironmentErrorMessage(originalError!)}`);
}
this.logger.info(`[ORCHESTRATOR] Performing resource cleanup for ${folderPath} (reason: ${reason}, preserveData: ${preserveData})`);
try {
// 1. Force resource manager to cancel any pending operations for this folder
if (this.resourceManager) {
try {
const operationId = `add-folder-${folderPath}`;
const cancelled = await this.resourceManager.cancelOperation(operationId);
if (cancelled) {
this.logger.info(`[ORCHESTRATOR] Cancelled pending resource manager operation: ${operationId}`);
}
// Also try to cancel scan operations
const scanOperationId = `scan-changes-${folderPath}`;
const scanCancelled = await this.resourceManager.cancelOperation(scanOperationId);
if (scanCancelled) {
this.logger.info(`[ORCHESTRATOR] Cancelled pending scan operation: ${scanOperationId}`);
}
} catch (error) {
this.logger.warn(`[ORCHESTRATOR] Error cancelling resource manager operations for ${folderPath}:`, error instanceof Error ? error : new Error(String(error)));
}
}
// 2. Stop and clean up any partially created folder manager
const pathKey = this.normalizePathKey(folderPath);
if (this.folderManagers.has(pathKey)) {
try {
const manager = this.folderManagers.get(pathKey);
if (manager) {
this.logger.info(`[ORCHESTRATOR] Stopping partially created folder manager for ${folderPath}`);
await manager.stop();
this.logger.info(`[ORCHESTRATOR] Folder manager stopped successfully`);
// On Windows, add a small delay to ensure database connections are fully released
const isWindows = process.platform === 'win32';
if (isWindows) {
this.logger.debug(`[ORCHESTRATOR] Windows detected - waiting for database locks to be released during cleanup...`);
await new Promise(resolve => setTimeout(resolve, 500)); // 500ms delay
}
}
this.folderManagers.delete(pathKey);
} catch (error) {
this.logger.warn(`[ORCHESTRATOR] Error stopping folder manager during cleanup for ${folderPath}:`, error instanceof Error ? error : new Error(String(error)));
}
}
// 3. Clean up file watching if it was started
if (this.monitoringOrchestrator) {
try {
await this.monitoringOrchestrator.stopFileWatching(folderPath);
this.logger.info(`[ORCHESTRATOR] File watching stopped for ${folderPath} during cleanup`);
} catch (error) {
// This is expected if file watching wasn't started yet
this.logger.debug(`[ORCHESTRATOR] File watching cleanup for ${folderPath}: ${error instanceof Error ? error.message : String(error)}`);
}
}
// 4. Clean up .folder-mcp directory ONLY if NOT an environment error
// CRITICAL: Environment errors should NOT delete indexed data!
if (!preserveData) {
try {
const folderMcpPath = `${folderPath}/.folder-mcp`;
const fsModule = await import('fs');
if (fsModule.existsSync(folderMcpPath)) {
this.logger.info(`[ORCHESTRATOR] Cleaning up .folder-mcp directory during cleanup: ${folderMcpPath}`);
await fsModule.promises.rm(folderMcpPath, { recursive: true, force: true });
this.logger.info(`[ORCHESTRATOR] Successfully cleaned up .folder-mcp directory`);
}
} catch (error) {
this.logger.warn(`[ORCHESTRATOR] Error cleaning up .folder-mcp directory for ${folderPath}:`, error instanceof Error ? error : new Error(String(error)));
}
} else {
this.logger.info(`[ORCHESTRATOR] Preserving .folder-mcp directory for ${folderPath} (environment error - data will be available after fix)`);
}
// 5. Remove from configuration ONLY if NOT an environment error
// CRITICAL: Environment errors should keep folder in config for retry after fix
if (!preserveData) {
try {
await this.configService.removeFolder(folderPath);
this.logger.info(`[ORCHESTRATOR] Removed ${folderPath} from configuration during cleanup`);
} catch (error) {
// This is expected if folder wasn't added to config yet
this.logger.debug(`[ORCHESTRATOR] Configuration cleanup for ${folderPath}: ${error instanceof Error ? error.message : String(error)}`);
}
} else {
this.logger.info(`[ORCHESTRATOR] Preserving ${folderPath} in configuration (environment error - will retry after fix)`);
}
// 6. Folder removal from managers handled above
// 7. Force garbage collection if available to free up memory
if (global.gc) {
this.logger.debug(`[ORCHESTRATOR] Triggering garbage collection after resource cleanup for ${folderPath}`);
global.gc();
}
// 8. Update FMDM to remove any partially created folder entries
// BUT: Don't remove error folders that were intentionally added to FMDM
// Check if the folder is already in FMDM with error status
const currentFMDM = this.fmdmService.getFMDM();
const folderInFMDM = currentFMDM.folders.find(f => f.path === folderPath);
if (!folderInFMDM || folderInFMDM.status !== 'error') {
// Only update FMDM if folder is not in error state
// (error state folders should remain visible to the user)
this.updateFMDM();
} else {
this.logger.debug(`[ORCHESTRATOR] Keeping error folder ${folderPath} in FMDM during cleanup`);
}
// 9. Log final resource statistics after cleanup
if (this.resourceManager) {
const stats = this.resourceManager.getStats();
this.logger.info(`[ORCHESTRATOR] Resource cleanup completed for ${folderPath}`, {
reason,
preserveData,
activeOperations: stats.activeOperations,
queuedOperations: stats.queuedOperations,
memoryUsedMB: Math.round(stats.memoryUsedMB)
});
}
} catch (error) {
this.logger.error(`[ORCHESTRATOR] Error during resource cleanup for ${folderPath}:`, error instanceof Error ? error : new Error(String(error)));
// Don't throw - cleanup should not fail the parent operation
}
}
/**
* Setup event handlers for the folder indexing queue
*/
private setupQueueEventHandlers(): void {
// Handle queue events for FMDM updates
this.folderIndexingQueue.on('queue:added', (folder) => {
this.logger.debug(`[ORCHESTRATOR] Folder added to queue: ${folder.folderPath}`);
this.updateFMDM();
});
this.folderIndexingQueue.on('queue:started', (folder) => {
this.logger.info(`[ORCHESTRATOR] Started processing folder: ${folder.folderPath}`);
this.updateFMDM();
});
this.folderIndexingQueue.on('queue:model-downloading', (folder, modelId, progress) => {
this.logger.debug(`[ORCHESTRATOR] Model downloading for ${folder.folderPath}: ${modelId} - ${progress}%`);
// Update FMDM with downloading status and progress
this.fmdmService.updateFolderStatus(folder.folderPath, 'downloading-model', {
message: `Downloading model ${modelId}: ${progress}%`,
type: 'info'
});
// Also update the folder's downloadProgress field for TUI display
this.fmdmService.updateModelDownloadStatus(modelId, 'downloading', progress);
// Emit activity event for model download progress
this.activityService?.emit({
type: 'model',
level: 'info',
message: `Downloading: ${modelId} (${progress}%)`,
progress,
userInitiated: false,
details: [`Folder: ${this.extractFolderName(folder.folderPath)}`]
});
});
this.folderIndexingQueue.on('queue:model-loading', (folder, progress) => {
this.logger.debug(`[ORCHESTRATOR] Model loading for ${folder.folderPath}: ${progress.stage}`);
// Update FMDM with model loading progress
if (progress.stage === 'downloading' && progress.progress !== undefined) {
this.fmdmService.updateFolderStatus(folder.folderPath, 'downloading-model', {
message: `Downloading model: ${progress.progress}%`,
type: 'info'
});
} else if (progress.stage === 'loading_model') {
// Model download complete, now loading into memory
this.fmdmService.updateFolderStatus(folder.folderPath, 'loading-model', {
message: `Loading model into memory${progress.progress ? `: ${progress.progress}%` : '...'}`,
type: 'info'
});
}
});
this.folderIndexingQueue.on('queue:model-loaded', (folder, modelId) => {
this.logger.info(`[ORCHESTRATOR] Model loaded for ${folder.folderPath}: ${modelId}`);
this.updateFMDM();
// Emit activity event for model ready
this.activityService?.emit({
type: 'model',
level: 'success',
message: `Model ready: ${modelId}`,
userInitiated: false
});
});
this.folderIndexingQueue.on('queue:progress', (folder, progress) => {
this.logger.debug(`[ORCHESTRATOR] Indexing progress for ${folder.folderPath}: ${progress.percentage}%`);
// Update FMDM with current progress
this.fmdmService.updateFolderProgress(folder.folderPath, progress.percentage);
this.updateFMDM();
// Emit activity event for indexing progress
// Use correlationId to group progress updates for same folder
// Include useful details about current progress
const details: string[] = [];
// Line 1: Model being used (always show)
const modelMeta = getModelMetadata(folder.modelId);
const modelName = modelMeta?.displayName || folder.modelId;
details.push(`Model: ${modelName}`);
// Line 2: File progress - use "Scanning..." if count not yet available
if (progress.totalFiles > 0) {
details.push(`Files: ${progress.processedFiles}/${progress.totalFiles}`);
} else {
details.push('Scanning files...');
}
// Line 3: Chunk progress if available (embedding phase)
if (progress.totalChunks > 0) {
details.push(`Chunks: ${progress.processedChunks}/${progress.totalChunks}`);
} else if (progress.processedFiles > 0) {
details.push('Preparing chunks...');
}
this.activityService?.emit({
type: 'indexing',
level: 'info',
correlationId: `indexing-${folder.folderPath}`,
message: `Indexing: ${folder.folderPath}`,
progress: progress.percentage,
userInitiated: false,
details
});
});
this.folderIndexingQueue.on('queue:completed', async (folder) => {
this.logger.info(`[ORCHESTRATOR] Completed processing folder: ${folder.folderPath}`);
// Get the manager's state which should include indexingStats
const pathKey = this.normalizePathKey(folder.folderPath);
const manager = this.folderManagers.get(pathKey);
if (manager) {
const state = manager.getState();
// Update FMDM with file count statistics if available
// This is important for re-queued folders to show updated file counts
if ((state as any).indexingStats) {
const { fileCount, indexingTimeSeconds } = (state as any).indexingStats;
const infoMessage = `▶ ${folder.folderPath} [active] i ${fileCount} files indexed • indexing time ${indexingTimeSeconds}s`;
this.logger.info(`[ORCHESTRATOR] Updating FMDM after re-indexing: ${infoMessage}`);
// Update folder status to 'active' with file count notification
this.fmdmService.updateFolderStatus(folder.folderPath, 'active', {
message: `${fileCount} files indexed • indexing time ${indexingTimeSeconds}s`,
type: 'info'
});
// Emit activity event for indexing complete with stats
// Use same correlationId as progress events so TUI replaces the in-progress event
this.activityService?.emit({
type: 'indexing',
level: 'success',
correlationId: `indexing-${folder.folderPath}`,
message: `Indexed: ${folder.folderPath}`,
progress: 100,
userInitiated: false,
details: [`Files: ${fileCount}`, `Duration: ${indexingTimeSeconds}s`]
});
} else {
this.logger.debug(`[ORCHESTRATOR] No indexing statistics available for ${folder.folderPath}`);
// Emit activity event without stats
// Use same correlationId as progress events so TUI replaces the in-progress event
this.activityService?.emit({
type: 'indexing',
level: 'success',
correlationId: `indexing-${folder.folderPath}`,
message: `Indexed: ${folder.folderPath}`,
progress: 100,
userInitiated: false
});
}
}
this.updateFMDM();
});
this.folderIndexingQueue.on('queue:failed', (folder, error) => {
this.logger.error(`[ORCHESTRATOR] Failed to process folder ${folder.folderPath}:`, error);
// Update FMDM with error status (this happens on each retry attempt)
this.fmdmService.updateFolderStatus(folder.folderPath, 'error', {
message: error.message,
type: 'error'
});
// Emit activity event for indexing failure
this.activityService?.emit({
type: 'error',
level: 'error',
message: `Failed: ${this.extractFolderName(folder.folderPath)}`,
userInitiated: false,
details: [error.message]
});
});
this.folderIndexingQueue.on('queue:permanently-failed', (folder, error) => {
this.logger.error(`[ORCHESTRATOR] Folder ${folder.folderPath} permanently failed after all retry attempts`);
// Update the folder manager's state to error using the proper method
const pathKey = this.normalizePathKey(folder.folderPath);
const manager = this.folderManagers.get(pathKey);
if (manager) {
// Call handleError to properly transition to error state
(manager as any).handleError?.(error, 'Indexing failed after 3 attempts');
}
// Update FMDM with permanent error status
// The updateFMDM call will get the error state from the manager
this.updateFMDM();
});
this.folderIndexingQueue.on('queue:empty', () => {
this.logger.debug('[ORCHESTRATOR] Indexing queue is empty');
});
this.folderIndexingQueue.on('queue:model-unloaded', (modelId) => {
this.logger.debug(`[ORCHESTRATOR] Model unloaded by queue: ${modelId}`);
// Notify the registry that the model was unloaded
const registry = PythonEmbeddingServiceRegistry.getInstance();
registry.notifyModelUnloaded();
});
}
/**
* Initialize Python embedding service for model downloads
* Uses the same factory pattern as the daemon's model cache checker
*/
private async initializePythonEmbeddingService(): Promise<void> {
try {
this.logger.debug('[ORCHESTRATOR] Initializing Python embedding service in idle state...');
// Import factory function (same as daemon uses)
const { createPythonEmbeddingService } = await import('../factories/model-factories.js');
// Create Python embedding service WITHOUT a model - starts in idle state
// The service will load models on demand when needed for each folder
const pythonService = await createPythonEmbeddingService({
// No modelName - Python starts empty in idle state
batchSize: 32,
maxSequenceLength: 512
});
// Wire the service to the model download manager
this.modelDownloadManager.setPythonEmbeddingService(pythonService);
this.logger.info('[ORCHESTRATOR] Python embedding service initialized in idle state, ready for on-demand model loading');
} catch (error) {
this.logger.error('[ORCHESTRATOR] Failed to initialize Python embedding service for downloads:', error instanceof Error ? error : new Error(String(error)));
// Don't throw - this is not a fatal error, just means model downloads won't work
}
}
/**
* Initialize ONNX downloader for CPU model downloads
*/
private async initializeONNXDownloader(): Promise<void> {
try {
this.logger.debug('[ORCHESTRATOR] Initializing ONNX downloader for CPU model downloads...');
// Import the ONNX downloader
const { ONNXDownloader } = await import('../../infrastructure/embeddings/onnx/onnx-downloader.js');
// Create ONNX downloader with default cache directory
const onnxDownloader = new ONNXDownloader({
cacheDirectory: path.join(os.homedir(), '.cache', 'folder-mcp', 'onnx-models')
});
// Wire the downloader to the model download manager
this.modelDownloadManager.setONNXDownloader(onnxDownloader);
this.logger.info('[ORCHESTRATOR] ONNX downloader wired to model download manager');
} catch (error) {
this.logger.error('[ORCHESTRATOR] Failed to initialize ONNX downloader:', error instanceof Error ? error : new Error(String(error)));
// Don't throw - this is not a fatal error, just means CPU model downloads won't work
}
}
}
