sync_portfolio
Synchronize your local portfolio elements with GitHub using configurable modes: additive for safe additions, mirror for exact matching, or backup for data protection. Always preview changes with dry-run first.
Instructions
Sync ALL elements in your local portfolio with your GitHub repository. By default uses 'additive' mode which only adds new items and never deletes (safest). Use 'mirror' mode for exact sync (with deletion confirmations). Use 'backup' mode to treat GitHub as backup source. ALWAYS run with dry_run:true first to preview changes. For individual elements, use 'portfolio_element_manager' instead.
Input Schema
TableJSON Schema
| Name | Required | Description | Default |
|---|---|---|---|
| direction | No | Sync direction: 'push' (upload to GitHub), 'pull' (download from GitHub), or 'both' (bidirectional sync). Defaults to 'push'. | |
| mode | No | Sync mode: 'additive' (default, only adds new items, never deletes), 'mirror' (makes exact match, requires confirmation for deletions), 'backup' (GitHub as backup, only pulls missing items locally). Defaults to 'additive' for safety. | |
| force | No | Whether to force sync even if there are conflicts. Use with caution as this may overwrite changes. In 'mirror' mode, this skips deletion confirmations. | |
| dry_run | No | Show what would be synced without actually performing the sync. RECOMMENDED to run with dry_run:true first to preview changes. | |
| confirm_deletions | No | In 'mirror' mode, require explicit confirmation for each deletion. Defaults to true unless force:true is set. |
Implementation Reference
- src/server/ServerSetup.ts:69-70 (registration)Registers the portfolio tools group, which includes the sync_portfolio tool via getPortfolioToolsthis.toolRegistry.registerMany(getPortfolioTools(instance));
- Defines the input schema, description, and registration handler for the sync_portfolio MCP tool. The handler delegates to server.syncPortfolio(options)tool: { name: "sync_portfolio", description: "Sync ALL elements in your local portfolio with your GitHub repository. By default uses 'additive' mode which only adds new items and never deletes (safest). Use 'mirror' mode for exact sync (with deletion confirmations). Use 'backup' mode to treat GitHub as backup source. ALWAYS run with dry_run:true first to preview changes. For individual elements, use 'portfolio_element_manager' instead.", inputSchema: { type: "object", properties: { direction: { type: "string", enum: ["push", "pull", "both"], description: "Sync direction: 'push' (upload to GitHub), 'pull' (download from GitHub), or 'both' (bidirectional sync). Defaults to 'push'.", }, mode: { type: "string", enum: ["additive", "mirror", "backup"], description: "Sync mode: 'additive' (default, only adds new items, never deletes), 'mirror' (makes exact match, requires confirmation for deletions), 'backup' (GitHub as backup, only pulls missing items locally). Defaults to 'additive' for safety.", }, force: { type: "boolean", description: "Whether to force sync even if there are conflicts. Use with caution as this may overwrite changes. In 'mirror' mode, this skips deletion confirmations.", }, dry_run: { type: "boolean", description: "Show what would be synced without actually performing the sync. RECOMMENDED to run with dry_run:true first to preview changes.", }, confirm_deletions: { type: "boolean", description: "In 'mirror' mode, require explicit confirmation for each deletion. Defaults to true unless force:true is set.", }, }, }, }, handler: (args: SyncPortfolioArgs) => server.syncPortfolio({ direction: args?.direction || 'push', mode: args?.mode || 'additive', force: args?.force || false, dryRun: args?.dry_run || false, confirmDeletions: args?.confirm_deletions !== false // Default true unless explicitly false }) },
- Primary handler implementing pull sync functionality specifically for sync_portfolio tool. Matches exact parameters (direction, mode, force, dryRun, confirmDeletions). Handles GitHub to local sync with safety features./** * PortfolioPullHandler - Handles pulling portfolio elements from GitHub * * This handler implements the pull functionality for sync_portfolio, * enabling users to download their portfolio from GitHub to local storage. * Supports multiple sync modes (additive, mirror, backup) and dry-run. */ import { PortfolioRepoManager } from '../portfolio/PortfolioRepoManager.js'; import { GitHubPortfolioIndexer } from '../portfolio/GitHubPortfolioIndexer.js'; import { PortfolioManager } from '../portfolio/PortfolioManager.js'; import { PortfolioIndexManager } from '../portfolio/PortfolioIndexManager.js'; import { ElementType } from '../portfolio/types.js'; import { logger } from '../utils/logger.js'; import { PortfolioSyncComparer, SyncMode, SyncAction } from '../sync/PortfolioSyncComparer.js'; import { PortfolioDownloader } from '../sync/PortfolioDownloader.js'; import { UnicodeValidator } from '../security/validators/unicodeValidator.js'; import { SecurityMonitor } from '../security/securityMonitor.js'; import { getPortfolioRepositoryName } from '../config/portfolioConfig.js'; import * as fs from 'fs/promises'; import * as path from 'path'; export interface PullOptions { direction: string; mode?: string; force?: boolean; dryRun?: boolean; confirmDeletions?: boolean; } export interface PullResult { content: Array<{ type: string; text: string; }>; } export class PortfolioPullHandler { private portfolioRepoManager: PortfolioRepoManager; private githubIndexer: GitHubPortfolioIndexer; private portfolioManager: PortfolioManager; private indexManager: PortfolioIndexManager; private syncComparer: PortfolioSyncComparer; private downloader: PortfolioDownloader; constructor() { this.portfolioRepoManager = new PortfolioRepoManager(getPortfolioRepositoryName()); this.githubIndexer = GitHubPortfolioIndexer.getInstance(); this.portfolioManager = PortfolioManager.getInstance(); this.indexManager = PortfolioIndexManager.getInstance(); this.syncComparer = new PortfolioSyncComparer(); this.downloader = new PortfolioDownloader(); } /** * Execute the pull operation from GitHub to local portfolio */ async executePull(options: PullOptions, personaIndicator: string): Promise<PullResult> { try { logger.info('Starting portfolio pull operation', { options }); // Step 1: Validate sync mode const syncMode = this.validateSyncMode(options.mode); // Step 2: Fetch GitHub portfolio index const progressMessages: string[] = []; progressMessages.push('š Fetching portfolio from GitHub...'); const githubIndex = await this.githubIndexer.getIndex(true); if (!githubIndex || githubIndex.totalElements === 0) { return { content: [{ type: "text", text: `${personaIndicator}ā ļø No elements found in GitHub portfolio. Nothing to pull.` }] }; } progressMessages.push(`š Found ${githubIndex.totalElements} elements on GitHub`); // Step 3: Get local portfolio state await this.indexManager.rebuildIndex(); const localElements = await this.getAllLocalElements(); progressMessages.push(`š Found ${this.countElements(localElements)} local elements`); // Step 4: Compare and determine sync actions const syncActions = this.syncComparer.compareElements( githubIndex.elements, localElements, syncMode ); // Step 5: Handle dry-run mode if (options.dryRun) { return this.formatDryRunResults(syncActions, progressMessages, personaIndicator); } // Step 6: Check for deletions requiring confirmation if (syncActions.toDelete.length > 0 && syncMode === 'mirror' && !options.force && options.confirmDeletions !== false) { return { content: [{ type: "text", text: `${personaIndicator}ā ļø Pull operation would delete ${syncActions.toDelete.length} local elements.\n\n` + `Elements to delete:\n${syncActions.toDelete.map(a => ` - ${a.name}`).join('\n')}\n\n` + `To proceed, run with \`force: true\` or \`confirmDeletions: false\`` }] }; } // Step 7: Execute sync actions const results = await this.executeSyncActions( syncActions, githubIndex.username, githubIndex.repository, progressMessages ); // Step 8: Return success summary return { content: [{ type: "text", text: `${personaIndicator}ā **Portfolio Pull Complete**\n\n` + progressMessages.join('\n') + '\n\n' + `**Summary:**\n` + ` š„ Added: ${results.added}\n` + ` š Updated: ${results.updated}\n` + ` š Skipped: ${results.skipped}\n` + (results.deleted > 0 ? ` šļø Deleted: ${results.deleted}\n` : '') + `\nYour local portfolio is now synchronized with GitHub!` }] }; } catch (error) { logger.error('Portfolio pull failed', { error }); return { content: [{ type: "text", text: `${personaIndicator}ā Failed to pull portfolio: ${error instanceof Error ? error.message : String(error)}` }] }; } } /** * Validate and normalize sync mode * SECURITY FIX: Added Unicode normalization to prevent homograph attacks */ private validateSyncMode(mode?: string): SyncMode { const validModes: SyncMode[] = ['additive', 'mirror', 'backup']; // SECURITY FIX: Normalize Unicode to prevent homograph attacks const normalizedMode = mode ? UnicodeValidator.normalize(mode).normalizedContent : 'additive'; const syncMode = normalizedMode.toLowerCase() as SyncMode; if (!validModes.includes(syncMode)) { throw new Error(`Invalid sync mode: ${mode}. Valid modes are: ${validModes.join(', ')}`); } return syncMode; } /** * Get all local elements organized by type */ private async getAllLocalElements(): Promise<Map<ElementType, any[]>> { const elements = new Map<ElementType, any[]>(); const elementTypes = Object.values(ElementType); for (const type of elementTypes) { const typeElements = await this.indexManager.getElementsByType(type); if (typeElements.length > 0) { elements.set(type, typeElements); } } return elements; } /** * Count total elements in a map */ private countElements(elements: Map<ElementType, any[]>): number { let count = 0; for (const typeElements of elements.values()) { count += typeElements.length; } return count; } /** * Format dry-run results for display */ private formatDryRunResults( syncActions: { toAdd: SyncAction[], toUpdate: SyncAction[], toDelete: SyncAction[], toSkip: SyncAction[] }, progressMessages: string[], personaIndicator: string ): PullResult { const lines = [ `${personaIndicator}š **Dry Run Results**`, '', ...progressMessages, '', '**Planned Actions:**' ]; if (syncActions.toAdd.length > 0) { lines.push(`\nš„ **To Add (${syncActions.toAdd.length}):**`); syncActions.toAdd.forEach(action => { lines.push(` - ${action.type}/${action.name}`); }); } if (syncActions.toUpdate.length > 0) { lines.push(`\nš **To Update (${syncActions.toUpdate.length}):**`); syncActions.toUpdate.forEach(action => { lines.push(` - ${action.type}/${action.name}`); }); } if (syncActions.toDelete.length > 0) { lines.push(`\nšļø **To Delete (${syncActions.toDelete.length}):**`); syncActions.toDelete.forEach(action => { lines.push(` - ${action.type}/${action.name}`); }); } if (syncActions.toSkip.length > 0) { lines.push(`\nš **To Skip (${syncActions.toSkip.length}):**`); syncActions.toSkip.forEach(action => { lines.push(` - ${action.type}/${action.name} (${action.reason})`); }); } lines.push('', 'Run without `dryRun: true` to execute these changes.'); return { content: [{ type: "text", text: lines.join('\n') }] }; } /** * Execute the sync actions */ private async executeSyncActions( syncActions: { toAdd: SyncAction[], toUpdate: SyncAction[], toDelete: SyncAction[], toSkip: SyncAction[] }, username: string, repository: string, progressMessages: string[] ): Promise<{ added: number, updated: number, deleted: number, skipped: number }> { const results = { added: 0, updated: 0, deleted: 0, skipped: syncActions.toSkip.length }; // PERFORMANCE: Process downloads in parallel batches for improved speed const BATCH_SIZE = 5; // Process 5 downloads at a time to avoid rate limiting // Helper function to process a batch of actions const processBatch = async (actions: SyncAction[], operation: string) => { const results = await Promise.allSettled( actions.map(async (action) => { progressMessages.push(`${operation}: ${action.type}/${action.name}`); await this.downloadAndSaveElement(action, username, repository); return action; }) ); return results.map((result, index) => ({ action: actions[index], success: result.status === 'fulfilled', error: result.status === 'rejected' ? result.reason : null })); }; // Process additions in batches for (let i = 0; i < syncActions.toAdd.length; i += BATCH_SIZE) { const batch = syncActions.toAdd.slice(i, i + BATCH_SIZE); const batchResults = await processBatch(batch, 'š„ Downloading'); for (const result of batchResults) { if (result.success) { results.added++; } else { logger.error(`Failed to add ${result.action.type}/${result.action.name}`, { error: result.error }); progressMessages.push(`ā Failed to add: ${result.action.type}/${result.action.name}`); } } } // Process updates in batches for (let i = 0; i < syncActions.toUpdate.length; i += BATCH_SIZE) { const batch = syncActions.toUpdate.slice(i, i + BATCH_SIZE); const batchResults = await processBatch(batch, 'š Updating'); for (const result of batchResults) { if (result.success) { results.updated++; } else { logger.error(`Failed to update ${result.action.type}/${result.action.name}`, { error: result.error }); progressMessages.push(`ā Failed to update: ${result.action.type}/${result.action.name}`); } } } // Process deletions for (const action of syncActions.toDelete) { try { progressMessages.push(`šļø Deleting: ${action.type}/${action.name}`); await this.deleteLocalElement(action); results.deleted++; } catch (error) { logger.error(`Failed to delete ${action.type}/${action.name}`, { error }); progressMessages.push(`ā Failed to delete: ${action.type}/${action.name}`); } } // PERFORMANCE: Batch rebuild index after all operations complete if (results.added > 0 || results.updated > 0 || results.deleted > 0) { progressMessages.push('š Rebuilding index...'); await this.indexManager.rebuildIndex(); } return results; } /** * Download element from GitHub and save locally * SECURITY: Added audit logging for GitHub operations */ private async downloadAndSaveElement( action: SyncAction, username: string, repository: string ): Promise<void> { // Set up the repo manager with the correct context this.portfolioRepoManager.setToken(await this.getGitHubToken()); // SECURITY: Log the download operation for audit trail SecurityMonitor.logSecurityEvent({ type: 'PORTFOLIO_FETCH_SUCCESS', severity: 'LOW', source: 'PortfolioPullHandler.downloadAndSaveElement', details: `Downloading element: ${action.type}/${action.name} from ${username}/${repository}` }); // Download the element content const elementData = await this.downloader.downloadFromGitHub( this.portfolioRepoManager, action.path, username, repository ); // Save to local portfolio const elementDir = this.portfolioManager.getElementDir(action.type); const fileName = path.basename(action.path); const filePath = path.join(elementDir, fileName); await fs.writeFile(filePath, elementData.content, 'utf-8'); // SECURITY: Log successful save for audit trail SecurityMonitor.logSecurityEvent({ type: 'ELEMENT_CREATED', severity: 'LOW', source: 'PortfolioPullHandler.downloadAndSaveElement', details: `Saved element to: ${action.type}/${fileName}` }); // PERFORMANCE: Skip individual index rebuild - will batch rebuild after all operations } /** * Delete local element * SECURITY: Added audit logging for deletion operations */ private async deleteLocalElement(action: SyncAction): Promise<void> { const elementDir = this.portfolioManager.getElementDir(action.type); // Use the original filename from the path to preserve extension const fileName = path.basename(action.path) || `${action.name}.md`; const filePath = path.join(elementDir, fileName); // SECURITY: Log deletion attempt for audit trail SecurityMonitor.logSecurityEvent({ type: 'ELEMENT_DELETED', severity: 'MEDIUM', source: 'PortfolioPullHandler.deleteLocalElement', details: `Attempting to delete: ${action.type}/${fileName}` }); try { await fs.unlink(filePath); // PERFORMANCE: Skip individual index rebuild - will batch rebuild after all operations // SECURITY: Log successful deletion SecurityMonitor.logSecurityEvent({ type: 'ELEMENT_DELETED', severity: 'MEDIUM', source: 'PortfolioPullHandler.deleteLocalElement', details: `Successfully deleted: ${action.type}/${fileName}` }); } catch (error: any) { if (error.code !== 'ENOENT') { throw error; } // File already doesn't exist, that's fine } } /** * Get GitHub token from auth manager */ private async getGitHubToken(): Promise<string> { // This should use the same token management as the rest of the system const { TokenManager } = await import('../security/tokenManager.js'); const token = await TokenManager.getGitHubTokenAsync(); if (!token) { throw new Error('GitHub authentication required. Please run setup_github_auth first.'); } return token; } }
- src/handlers/SyncHandlerV2.ts:29-278 (handler)Secondary sync handler for sync_portfolio MCP tool, handles individual/bulk element operations (download/upload/compare) via PortfolioSyncManagerexport class SyncHandler { private syncManager: PortfolioSyncManager; private configManager: ConfigManager; constructor() { this.syncManager = new PortfolioSyncManager(); this.configManager = ConfigManager.getInstance(); } /** * Handle portfolio sync operations */ async handleSyncOperation(options: SyncOperationOptions, indicator: string = '') { try { await this.configManager.initialize(); // Check if sync is enabled (allow list-remote and compare even when disabled) const syncEnabled = this.configManager.getSetting('sync.enabled'); const readOnlyOperations = ['list-remote', 'compare']; if (!syncEnabled && !readOnlyOperations.includes(options.operation)) { return { content: [{ type: "text", text: `${indicator}ā ļø **Sync is Disabled**\n\n` + `Portfolio sync is currently disabled for privacy.\n\n` + `To enable sync:\n` + `\`dollhouse_config action: "set", setting: "sync.enabled", value: true\`\n\n` + `You can still use \`list-remote\` and \`compare\` to view differences.` }] }; } // Map our operation to PortfolioSyncManager's SyncOperation format const syncOp: SyncOperation = { operation: this.mapOperation(options.operation), element_name: options.element_name, element_type: options.element_type || options.filter?.type, // Use filter.type if element_type not provided bulk: options.operation.includes('bulk'), show_diff: options.operation === 'compare', force: options.options?.force, confirm: options.options?.force || options.options?.dry_run === false // force implies confirm, dry_run=false means confirm }; // Call the unified handleSyncOperation method const result = await this.syncManager.handleSyncOperation(syncOp); // Format the result based on the operation type return this.formatResult(result, options, indicator); } catch (error) { const sanitizedError = SecureErrorHandler.sanitizeError(error); return { content: [{ type: "text", text: `${indicator}ā Sync operation failed: ${sanitizedError.message}` }] }; } } private mapOperation(operation: string): 'download' | 'upload' | 'compare' | 'list-remote' { switch (operation) { case 'list-remote': return 'list-remote'; case 'download': case 'bulk-download': return 'download'; case 'upload': case 'bulk-upload': return 'upload'; case 'compare': return 'compare'; default: return 'list-remote'; } } private formatResult(result: SyncResult, options: SyncOperationOptions, indicator: string) { if (!result.success) { return { content: [{ type: "text", text: `${indicator}ā ${result.message}` }] }; } switch (options.operation) { case 'list-remote': return this.formatListResult(result, indicator); case 'download': case 'bulk-download': return this.formatDownloadResult(result, options, indicator); case 'upload': case 'bulk-upload': return this.formatUploadResult(result, options, indicator); case 'compare': return this.formatCompareResult(result, options, indicator); default: return { content: [{ type: "text", text: `${indicator}ā ${result.message}` }] }; } } private formatListResult(result: SyncResult, indicator: string) { if (!result.elements || result.elements.length === 0) { return { content: [{ type: "text", text: `${indicator}š **GitHub Portfolio is Empty**\n\n` + `No elements found in your GitHub portfolio.\n\n` + `Upload elements using:\n` + `\`sync_portfolio operation: "upload", element_name: "name", element_type: "type"\`` }] }; } let text = `${indicator}š **GitHub Portfolio Contents**\n\n`; text += `Found ${result.elements.length} elements:\n\n`; // Group by type const byType: Record<string, any[]> = {}; for (const element of result.elements) { if (!byType[element.type]) { byType[element.type] = []; } byType[element.type].push(element); } for (const [type, elements] of Object.entries(byType)) { text += `**${type}** (${elements.length}):\n`; for (const element of elements) { text += ` ā¢ ${element.name}`; if (element.remoteVersion) { text += ` v${element.remoteVersion}`; } if (element.status) { text += ` (${element.status})`; } text += '\n'; } text += '\n'; } return { content: [{ type: "text", text }] }; } private formatDownloadResult(result: SyncResult, options: SyncOperationOptions, indicator: string) { if (options.operation === 'bulk-download') { const elements = result.elements || []; const downloaded = elements.filter(e => e.action === 'download').length; const skipped = elements.filter(e => e.action === 'skip').length; return { content: [{ type: "text", text: `${indicator}ā **Bulk Download Complete**\n\n` + `Downloaded: ${downloaded} elements\n` + `Skipped: ${skipped} elements\n\n` + result.message }] }; } return { content: [{ type: "text", text: `${indicator}ā **Element Downloaded**\n\n` + `Element: ${options.element_name} (${options.element_type})\n\n` + result.message }] }; } private formatUploadResult(result: SyncResult, options: SyncOperationOptions, indicator: string) { if (options.operation === 'bulk-upload') { const elements = result.elements || []; const uploaded = elements.filter(e => e.action === 'upload').length; const skipped = elements.filter(e => e.action === 'skip').length; return { content: [{ type: "text", text: `${indicator}ā **Bulk Upload Complete**\n\n` + `Uploaded: ${uploaded} elements\n` + `Skipped: ${skipped} elements\n\n` + result.message }] }; } return { content: [{ type: "text", text: `${indicator}ā **Element Uploaded**\n\n` + `Element: ${options.element_name} (${options.element_type})\n\n` + result.message }] }; } private formatCompareResult(result: SyncResult, options: SyncOperationOptions, indicator: string) { let text = `${indicator}š **Version Comparison**\n\n`; text += `Element: ${options.element_name} (${options.element_type})\n\n`; if (result.data) { // If we have detailed comparison data const data = result.data; if (data.local) { text += `**Local Version**: ${data.local.version}\n`; text += ` Modified: ${new Date(data.local.timestamp).toLocaleString()}\n`; } else { text += `**Local Version**: Not found\n`; } if (data.remote) { text += `\n**Remote Version**: ${data.remote.version}\n`; text += ` Modified: ${new Date(data.remote.timestamp).toLocaleString()}\n`; } else { text += `\n**Remote Version**: Not found\n`; } if (data.diff) { text += `\n**Differences**:\n${data.diff}`; } } text += `\n\n${result.message}`; return { content: [{ type: "text", text }] }; } }
- Core helper managing low-level sync operations (download, upload, compare, list) called by handlers with privacy/safety checks.export class PortfolioSyncManager { private configManager: ConfigManager; private portfolioManager: PortfolioManager; private repoManager: PortfolioRepoManager; private indexer: GitHubPortfolioIndexer; constructor() { this.configManager = ConfigManager.getInstance(); this.portfolioManager = PortfolioManager.getInstance(); this.repoManager = new PortfolioRepoManager(getPortfolioRepositoryName()); this.indexer = GitHubPortfolioIndexer.getInstance(); } /** * Main handler for sync operations */ public async handleSyncOperation(params: SyncOperation): Promise<SyncResult> { try { // Check if sync is enabled in config const config = this.configManager.getConfig(); if (!config.sync.enabled && params.operation !== 'list-remote') { return { success: false, message: 'Sync is disabled. Enable it with: dollhouse_config --action update --setting sync.enabled --value true' }; } // Check bulk permissions if (params.bulk) { const bulkAllowed = this.isBulkOperationAllowed(params.operation, config); if (!bulkAllowed.allowed) { return { success: false, message: bulkAllowed.message }; } } // Handle operations switch (params.operation) { case 'list-remote': return await this.listRemoteElements(params.element_type); case 'download': if (params.bulk) { return await this.bulkDownload(params.element_type, params.confirm); } else if (params.element_name) { return await this.downloadElement( params.element_name, params.element_type!, params.version, params.force ); } else { return { success: false, message: 'Element name required for individual download' }; } case 'upload': if (params.bulk) { return await this.bulkUpload(params.element_type, params.confirm); } else if (params.element_name) { return await this.uploadElement( params.element_name, params.element_type!, params.confirm ); } else { return { success: false, message: 'Element name required for individual upload' }; } case 'compare': if (params.element_name && params.element_type) { return await this.compareVersions( params.element_name, params.element_type, params.show_diff ); } else { return { success: false, message: 'Element name and type required for comparison' }; } default: return { success: false, message: `Unknown operation: ${params.operation}` }; } } catch (error) { logger.error('Sync operation failed', { operation: params.operation, error: error instanceof Error ? error.message : String(error) }); return { success: false, message: `Sync operation failed: ${error instanceof Error ? error.message : String(error)}` }; } } /** * Check if bulk operation is allowed */ private isBulkOperationAllowed(operation: string, config: DollhouseConfig): { allowed: boolean; message: string } { if (operation === 'download' && !config.sync.bulk.download_enabled) { return { allowed: false, message: 'Bulk download is disabled. Enable with: dollhouse_config --action update --setting sync.bulk.download_enabled --value true' }; } if (operation === 'upload' && !config.sync.bulk.upload_enabled) { return { allowed: false, message: 'Bulk upload is disabled. Enable with: dollhouse_config --action update --setting sync.bulk.upload_enabled --value true' }; } return { allowed: true, message: '' }; } /** * List elements available in GitHub portfolio */ private async listRemoteElements(filterType?: ElementType): Promise<SyncResult> { try { // Get GitHub token const token = await TokenManager.getGitHubTokenAsync(); if (!token) { return { success: false, message: 'GitHub authentication required. Use setup_github_auth first.' }; } this.repoManager.setToken(token); // Get index of GitHub portfolio const index = await this.indexer.getIndex(); if (!index || index.totalElements === 0) { return { success: true, message: 'No elements found in GitHub portfolio', elements: [] }; } // Format elements for display const elements: SyncElementInfo[] = []; for (const [type, entries] of index.elements) { // Skip if filtering by type and this isn't the requested type if (filterType && type !== filterType) { continue; } for (const entry of entries) { elements.push({ name: entry.name, type: type, remoteVersion: entry.version, status: 'unchanged', action: 'download' }); } } return { success: true, message: `Found ${elements.length} elements in GitHub portfolio`, elements }; } catch (error) { return { success: false, message: `Failed to list remote elements: ${error instanceof Error ? error.message : String(error)}` }; } } /** * Download a specific element from GitHub */ private async downloadElement( elementName: string, elementType: ElementType, version?: string, force?: boolean ): Promise<SyncResult> { try { const config = this.configManager.getConfig(); // Validate element name const validation = UnicodeValidator.normalize(elementName); if (!validation.isValid) { return { success: false, message: `Invalid element name: ${validation.detectedIssues?.[0] || 'unknown error'}` }; } // Get token and set it const token = await TokenManager.getGitHubTokenAsync(); if (!token) { return { success: false, message: 'GitHub authentication required' }; } this.repoManager.setToken(token); // Get GitHub index const index = await this.indexer.getIndex(); // Find the element - first try exact match, then fuzzy match const entries = index.elements.get(elementType) || []; let entry = entries.find(e => e.name === elementName); // If exact match not found, try fuzzy matching if (!entry) { // Try case-insensitive exact match first entry = entries.find(e => e.name.toLowerCase() === elementName.toLowerCase()); // If still not found, try fuzzy matching if (!entry) { const fuzzyMatch = this.findFuzzyMatch(elementName, entries); if (fuzzyMatch) { logger.info(`Fuzzy match found: '${elementName}' matched to '${fuzzyMatch.name}'`); entry = fuzzyMatch; } } } if (!entry) { // Generate helpful suggestions const suggestions = this.getSuggestions(elementName, entries); const suggestionText = suggestions.length > 0 ? `\n\nDid you mean one of these?\n${suggestions.map(s => ` ā¢ ${s.name}`).join('\n')}` : ''; return { success: false, message: `Element '${elementName}' (${elementType}) not found in GitHub portfolio${suggestionText}` }; } // Check for local conflicts const localPath = this.portfolioManager.getElementPath(elementType, `${elementName}.md`); let hasLocalVersion = false; let localContent: string | null = null; try { localContent = await fs.readFile(localPath, 'utf-8'); hasLocalVersion = true; } catch { // No local version exists } // Download the element const response = await fetch(entry.downloadUrl, { headers: { 'Authorization': `Bearer ${token}`, 'Accept': 'application/vnd.github.v3.raw' } }); if (!response.ok) { throw new Error(`Failed to download: ${response.statusText}`); } const remoteContent = await response.text(); // Validate content security const validationResult = ContentValidator.validateAndSanitize(remoteContent); if (!validationResult.isValid && validationResult.severity === 'critical') { return { success: false, message: `Security issue detected in remote content: ${validationResult.detectedPatterns?.join(', ')}` }; } // Check if content is different if (hasLocalVersion && localContent) { const localHash = createHash('sha256').update(localContent).digest('hex'); const remoteHash = createHash('sha256').update(remoteContent).digest('hex'); if (localHash === remoteHash) { return { success: true, message: `Element '${elementName}' is already up to date` }; } // Show confirmation for overwrite unless force flag is set if (config.sync.individual.require_confirmation && !force) { const diff = await this.generateDiff(localContent, remoteContent); return { success: false, message: `Local version exists. Please confirm download will overwrite:\n\n${diff}\n\nTo proceed, use --force flag`, data: { requiresConfirmation: true } }; } } // Save the element await fs.mkdir(path.dirname(localPath), { recursive: true }); await fs.writeFile(localPath, remoteContent, 'utf-8'); logger.info('Element downloaded from GitHub', { element: elementName, type: elementType, version: entry.version }); return { success: true, message: `Successfully downloaded '${elementName}' (${elementType}) from GitHub portfolio` }; } catch (error) { return { success: false, message: `Failed to download element: ${error instanceof Error ? error.message : String(error)}` }; } } /** * Upload a specific element to GitHub */ private async uploadElement( elementName: string, elementType: ElementType, confirm?: boolean ): Promise<SyncResult> { try { const config = this.configManager.getConfig(); // Check for local element const localPath = this.portfolioManager.getElementPath(elementType, `${elementName}.md`); let content: string; try { content = await fs.readFile(localPath, 'utf-8'); } catch { return { success: false, message: `Element '${elementName}' (${elementType}) not found locally` }; } // Check privacy metadata const parsed = SecureYamlParser.parse(content, { maxYamlSize: 64 * 1024, validateContent: false, validateFields: false }); if (parsed.data?.privacy?.local_only === true) { return { success: false, message: `Element '${elementName}' is marked as local-only and cannot be uploaded` }; } // Validate content security const validationResult = ContentValidator.validateAndSanitize(content); if (!validationResult.isValid && validationResult.severity === 'critical') { return { success: false, message: `Security issue detected: ${validationResult.detectedPatterns?.join(', ')}` }; } // Scan for sensitive content if configured if (config.sync.privacy.scan_for_secrets) { logger.debug('Scanning for secrets before upload'); // Implement actual secret scanning const secretPatterns = [ /api[_-]?key\s*[:=]\s*['"][^'"]+['"]/gi, /secret\s*[:=]\s*['"][^'"]+['"]/gi, /password\s*[:=]\s*['"][^'"]+['"]/gi, /token\s*[:=]\s*['"][^'"]+['"]/gi, /private[_-]?key\s*[:=]\s*['"][^'"]+['"]/gi ]; for (const pattern of secretPatterns) { if (pattern.test(content)) { return { success: false, message: `Potential secret detected in content. Please review and remove sensitive information before uploading.` }; } } } // Get confirmation if required (unless already confirmed) if (config.sync.individual.require_confirmation && !confirm) { return { success: false, message: `Please confirm upload of '${elementName}' (${elementType}) to GitHub.\n\nContent preview:\n${content.substring(0, 500)}...\n\nTo proceed, use --confirm flag`, data: { requiresConfirmation: true } }; } // Get token and validate const token = await TokenManager.getGitHubTokenAsync(); if (!token) { return { success: false, message: 'GitHub authentication required' }; } // Create a PortfolioElement for the adapter (fixes Issue #913) // Using PortfolioElementAdapter instead of incomplete IElement implementation const portfolioElement = { type: elementType, metadata: { name: elementName, description: parsed.data?.description || '', author: parsed.data?.author || 'unknown', created: parsed.data?.created || new Date().toISOString(), updated: new Date().toISOString(), version: parsed.data?.version || '1.0.0', tags: parsed.data?.tags || [] }, content: content }; // Use PortfolioElementAdapter to properly implement IElement interface const adapter = new PortfolioElementAdapter(portfolioElement); // Use PortfolioRepoManager to upload this.repoManager.setToken(token); // DEBUG: Log upload attempt logger.debug('[BULK_SYNC_DEBUG] Upload element attempt', { elementName, elementType, hasToken: !!token, tokenPrefix: token ? token.substring(0, 10) + '...' : 'none', adapterHasMetadata: !!(adapter && adapter.metadata), timestamp: new Date().toISOString() }); try { const url = await this.repoManager.saveElement(adapter, true); // consent is true since we've already checked logger.info('Element uploaded to GitHub', { element: elementName, type: elementType, url }); return { success: true, message: `Successfully uploaded '${elementName}' (${elementType}) to GitHub portfolio`, data: { url } }; } catch (uploadError) { // Handle specific errors if (uploadError instanceof Error && uploadError.message.includes('repository does not exist')) { return { success: false, message: `GitHub portfolio repository not found. Please initialize it first using init_portfolio tool.` }; } throw uploadError; } } catch (error) { return { success: false, message: `Failed to upload element: ${error instanceof Error ? error.message : String(error)}` }; } } /** * Compare local and remote versions */ private async compareVersions( elementName: string, elementType: ElementType, showDiff?: boolean ): Promise<SyncResult> { try { // Get local version const localPath = this.portfolioManager.getElementPath(elementType, `${elementName}.md`); let localContent: string | null = null; let localVersion: VersionInfo | null = null; try { localContent = await fs.readFile(localPath, 'utf-8'); const parsed = SecureYamlParser.parse(localContent, { maxYamlSize: 64 * 1024, validateContent: false, validateFields: false }); localVersion = { version: parsed.data?.version || '1.0.0', timestamp: new Date(parsed.data?.updated || parsed.data?.created || Date.now()), author: parsed.data?.author || 'unknown', hash: createHash('sha256').update(localContent).digest('hex'), size: Buffer.byteLength(localContent), source: 'local' }; } catch { // No local version } // Get remote version const token = await TokenManager.getGitHubTokenAsync(); if (!token) { return { success: false, message: 'GitHub authentication required' }; } const index = await this.indexer.getIndex(); const entries = index.elements.get(elementType) || []; const entry = entries.find(e => e.name === elementName); let remoteVersion: VersionInfo | null = null; let remoteContent: string | null = null; if (entry) { const response = await fetch(entry.downloadUrl, { headers: { 'Authorization': `Bearer ${token}`, 'Accept': 'application/vnd.github.v3.raw' } }); if (response.ok) { remoteContent = await response.text(); remoteVersion = { version: entry.version || '1.0.0', timestamp: entry.lastModified, author: entry.author || 'unknown', hash: createHash('sha256').update(remoteContent).digest('hex'), size: entry.size, source: 'remote' }; } } // Build comparison result const result: any = { element: elementName, type: elementType, local: localVersion, remote: remoteVersion }; if (localVersion && remoteVersion) { result.status = localVersion.hash === remoteVersion.hash ? 'identical' : 'different'; if (showDiff && localContent && remoteContent && result.status === 'different') { result.diff = await this.generateDiff(localContent, remoteContent); } } else if (localVersion && !remoteVersion) { result.status = 'local-only'; } else if (!localVersion && remoteVersion) { result.status = 'remote-only'; } else { result.status = 'not-found'; } return { success: true, message: `Version comparison for '${elementName}' (${elementType})`, data: result }; } catch (error) { return { success: false, message: `Failed to compare versions: ${error instanceof Error ? error.message : String(error)}` }; } } /** * Bulk download elements */ private async bulkDownload(elementType?: ElementType, confirm?: boolean): Promise<SyncResult> { const config = this.configManager.getConfig(); if (!config.sync.bulk.download_enabled) { return { success: false, message: 'Bulk download is not enabled in configuration' }; } // Get list of remote elements const remoteResult = await this.listRemoteElements(); if (!remoteResult.success || !remoteResult.elements) { return remoteResult; } // Filter by type if specified let elementsToDownload = remoteResult.elements; if (elementType) { elementsToDownload = elementsToDownload.filter(e => e.type === elementType); } if (elementsToDownload.length === 0) { return { success: true, message: 'No elements to download', elements: [] }; } // Show preview if required (unless already confirmed) if (config.sync.bulk.require_preview && !confirm) { return { success: false, message: `Bulk download preview:\n\n${elementsToDownload.length} elements will be downloaded:\n${elementsToDownload.map(e => `- ${e.name} (${e.type})`).join('\n')}\n\nTo proceed, use --confirm flag`, data: { requiresConfirmation: true }, elements: elementsToDownload }; } // Perform actual bulk download const results = { downloaded: [] as string[], skipped: [] as string[], failed: [] as { name: string; error: string }[] }; for (const element of elementsToDownload) { try { const result = await this.downloadElement(element.name, element.type, undefined, true); // force=true to skip individual confirmations if (result.success) { results.downloaded.push(element.name); } else if (result.message?.includes('already up to date')) { results.skipped.push(element.name); } else { results.failed.push({ name: element.name, error: result.message || 'Unknown error' }); } } catch (error) { results.failed.push({ name: element.name, error: error instanceof Error ? error.message : String(error) }); } } // Build summary message let message = `Bulk download complete:\n`; message += `- Downloaded: ${results.downloaded.length} elements\n`; message += `- Skipped (up to date): ${results.skipped.length} elements\n`; message += `- Failed: ${results.failed.length} elements`; if (results.failed.length > 0) { message += `\n\nFailed downloads:\n${results.failed.map(f => `- ${f.name}: ${f.error}`).join('\n')}`; } return { success: results.failed.length === 0, message, data: results }; } /** * Bulk upload elements */ private async bulkUpload(elementType?: ElementType, confirm?: boolean): Promise<SyncResult> { const config = this.configManager.getConfig(); if (!config.sync.bulk.upload_enabled) { return { success: false, message: 'Bulk upload is not enabled in configuration' }; } // Get list of local elements const types = elementType ? [elementType] : [ ElementType.PERSONA, ElementType.SKILL, ElementType.TEMPLATE, ElementType.AGENT, ElementType.MEMORY, ElementType.ENSEMBLE ]; const localElements: { name: string; type: ElementType; path: string }[] = []; for (const type of types) { const dir = this.portfolioManager.getElementDir(type); try { const files = await fs.readdir(dir); for (const file of files) { if (file.endsWith('.md')) { localElements.push({ name: file.replace('.md', ''), type, path: path.join(dir, file) }); } } } catch (error) { // Directory may not exist yet logger.debug(`Directory for ${type} does not exist yet`); } } if (localElements.length === 0) { return { success: true, message: 'No local elements to upload', elements: [] }; } // Show preview if required (unless already confirmed) if (config.sync.bulk.require_preview && !confirm) { // Convert to SyncElementInfo format for preview const previewElements: SyncElementInfo[] = localElements.map(e => ({ name: e.name, type: e.type, status: 'local-only' as const, action: 'upload' as const })); return { success: false, message: `Bulk upload preview:\n\n${localElements.length} elements will be uploaded:\n${localElements.map(e => `- ${e.name} (${e.type})`).join('\n')}\n\nTo proceed, use --confirm flag`, data: { requiresConfirmation: true }, elements: previewElements }; } // Perform actual bulk upload const results = { uploaded: [] as string[], skipped: [] as string[], failed: [] as { name: string; error: string }[] }; for (const element of localElements) { try { const result = await this.uploadElement(element.name, element.type, true); // confirm=true to skip individual confirmations if (result.success) { results.uploaded.push(element.name); } else if (result.message?.includes('local-only')) { results.skipped.push(element.name); } else { results.failed.push({ name: element.name, error: result.message || 'Unknown error' }); } } catch (error) { results.failed.push({ name: element.name, error: error instanceof Error ? error.message : String(error) }); } } // Build summary message let message = `Bulk upload complete:\n`; message += `- Uploaded: ${results.uploaded.length} elements\n`; message += `- Skipped (local-only): ${results.skipped.length} elements\n`; message += `- Failed: ${results.failed.length} elements`; if (results.failed.length > 0) { message += `\n\nFailed uploads:\n${results.failed.map(f => `- ${f.name}: ${f.error}`).join('\n')}`; } return { success: results.failed.length === 0, message, data: results }; } /** * Generate diff between two content versions */ private async generateDiff(local: string, remote: string): Promise<string> { // Simple line-based diff for now const localLines = local.split('\n'); const remoteLines = remote.split('\n'); let diff = ''; const maxLines = Math.max(localLines.length, remoteLines.length); for (let i = 0; i < maxLines && i < 10; i++) { // Show first 10 lines of diff const localLine = localLines[i] || ''; const remoteLine = remoteLines[i] || ''; if (localLine !== remoteLine) { if (localLine && !remoteLine) { diff += `- ${localLine}\n`; } else if (!localLine && remoteLine) { diff += `+ ${remoteLine}\n`; } else { diff += `- ${localLine}\n`; diff += `+ ${remoteLine}\n`; } } } if (maxLines > 10) { diff += `\n... ${maxLines - 10} more lines ...`; } return diff || 'No differences found'; } /** * Find a fuzzy match for an element name */ private findFuzzyMatch(searchName: string, entries: GitHubIndexEntry[]): GitHubIndexEntry | null { const search = searchName.toLowerCase().replaceAll(/[-_]/g, ''); let bestMatch: typeof entries[0] | null = null; let bestScore = 0; for (const entry of entries) { // Normalize the entry name for comparison const normalized = entry.name.toLowerCase().replaceAll(/[-_]/g, ''); // Calculate similarity score const score = this.calculateSimilarity(search, normalized); if (score > bestScore && score > 0.5) { // Minimum threshold of 0.5 bestScore = score; bestMatch = entry; } } return bestMatch; } /** * Get suggestions for similar element names */ private getSuggestions(searchName: string, entries: GitHubIndexEntry[]): Array<{name: string}> { const search = searchName.toLowerCase().replaceAll(/[-_]/g, ''); const scored: Array<{entry: typeof entries[0]; score: number}> = []; for (const entry of entries) { const normalized = entry.name.toLowerCase().replaceAll(/[-_]/g, ''); const score = this.calculateSimilarity(search, normalized); if (score > 0.3) { // Lower threshold for suggestions scored.push({ entry, score }); } } // Sort by score and return top 5 return scored .sort((a, b) => b.score - a.score) .slice(0, 5) .map(s => ({ name: s.entry.name })); } /** * Calculate similarity between two strings * Returns a score between 0 and 1 */ private calculateSimilarity(a: string, b: string): number { // Exact match if (a === b) return 1.0; // One contains the other if (a.includes(b) || b.includes(a)) return 0.8; // Calculate word overlap const wordsA = a.split(/[^a-z0-9]+/); const wordsB = b.split(/[^a-z0-9]+/); let matches = 0; for (const wordA of wordsA) { if (wordA && wordsB.some(wordB => wordB === wordA)) { matches++; } } if (matches > 0) { const overlap = (matches * 2) / (wordsA.length + wordsB.length); return Math.max(0.6, overlap); // At least 0.6 for any word match } // Check for partial matches for (const wordA of wordsA) { for (const wordB of wordsB) { if (wordA.length > 3 && wordB.length > 3) { if (wordA.includes(wordB) || wordB.includes(wordA)) { return 0.5; } } } } // No significant similarity return 0; } }