documcp

Instructions

Implementation Reference

• src/memory/index.ts:680-750 (schema)

  Tool input schema definition for memory_import_advanced, including all advanced options for format, mode, validation, conflict resolution, backup, transformations, etc.
  { name: "memory_import_advanced", description: "Advanced memory import with validation and conflict resolution", inputSchema: { type: "object", properties: { inputPath: { type: "string", description: "Input file path" }, options: { type: "object", properties: { format: { type: "string", enum: [ "json", "jsonl", "csv", "xml", "yaml", "sqlite", "archive", ], default: "json", }, mode: { type: "string", enum: ["merge", "replace", "append", "update"], default: "merge", }, validation: { type: "string", enum: ["strict", "loose", "none"], default: "strict", }, conflictResolution: { type: "string", enum: ["skip", "overwrite", "merge", "rename"], default: "skip", }, backup: { type: "boolean", default: true }, dryRun: { type: "boolean", default: false }, mapping: { type: "object", description: "Field mapping for different schemas", }, transformation: { type: "object", properties: { enabled: { type: "boolean", default: false }, rules: { type: "array", items: { type: "object", properties: { field: { type: "string" }, operation: { type: "string", enum: ["convert", "transform", "validate"], }, params: { type: "object" }, }, }, }, }, }, }, }, }, required: ["inputPath"], }, },
• src/memory/export-import.ts:350-442 (handler)

  Core handler function implementing the advanced memory import logic: format detection, data loading/parsing, validation, backup creation, conflict resolution (skip/overwrite/merge/rename), dry-run support, transformations, and detailed result reporting with ImportResult type.
  async importMemories( inputPath: string, options: Partial<ImportOptions> = {}, ): Promise<ImportResult> { const defaultOptions: ImportOptions = { format: "json", mode: "merge", validation: "strict", conflictResolution: "skip", backup: true, dryRun: false, }; const activeOptions = { ...defaultOptions, ...options }; const startTime = Date.now(); this.emit("import_started", { inputPath, options: activeOptions }); try { // Create backup if requested if (activeOptions.backup && !activeOptions.dryRun) { await this.createBackup(); } // Detect and verify format const detectedFormat = await this.detectFormat(inputPath); if (detectedFormat !== activeOptions.format) { this.emit("format_mismatch", { detected: detectedFormat, specified: activeOptions.format, }); } // Load and parse import data const importData = await this.loadImportData(inputPath, activeOptions); // Validate import data const validationResult = await this.validateImportData( importData, activeOptions, ); if ( validationResult.invalid > 0 && activeOptions.validation === "strict" ) { throw new Error( `Validation failed: ${validationResult.invalid} invalid entries`, ); } // Process import data const result = await this.processImportData(importData, activeOptions); this.emit("import_completed", { result, duration: Date.now() - startTime, }); return result; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); this.emit("import_error", { error: errorMessage }); return { success: false, processed: 0, imported: 0, skipped: 0, errors: 1, errorDetails: [errorMessage], conflicts: 0, validation: { valid: 0, invalid: 0, warnings: [], }, summary: { newEntries: 0, updatedEntries: 0, duplicateEntries: 0, failedEntries: 0, }, metadata: { importedAt: new Date(), source: inputPath, format: activeOptions.format, mode: activeOptions.mode, }, }; } }
• src/memory/export-import.ts:40-56 (schema)

  TypeScript interface defining the input options structure for advanced memory import, matching the tool's inputSchema exactly.
  export interface ImportOptions { format: "json" | "jsonl" | "csv" | "xml" | "yaml" | "sqlite" | "archive"; mode: "merge" | "replace" | "append" | "update"; validation: "strict" | "loose" | "none"; conflictResolution: "skip" | "overwrite" | "merge" | "rename"; backup: boolean; dryRun: boolean; mapping?: Record<string, string>; // Field mapping for different schemas transformation?: { enabled: boolean; rules: Array<{ field: string; operation: "convert" | "transform" | "validate"; params: any; }>; }; }
• src/memory/export-import.ts:554-1265 (helper)

  Key helper method that processes the loaded data: applies transformations/mappings, handles conflicts per resolution strategy, performs storage operations (store/update), tracks statistics, imports learning/KG data if present. Called by the main handler.
  * Get supported formats */ getSupportedFormats(): { export: string[]; import: string[]; compression: string[]; encryption: string[]; } { return { export: ["json", "jsonl", "csv", "xml", "yaml", "sqlite", "archive"], import: ["json", "jsonl", "csv", "xml", "yaml", "sqlite", "archive"], compression: ["gzip", "zip", "none"], encryption: ["aes-256-gcm", "aes-192-gcm", "aes-128-gcm"], }; } /** * Validate export/import compatibility */ async validateCompatibility( sourcePath: string, _targetSystem: string = "DocuMCP", ): Promise<{ compatible: boolean; issues: string[]; recommendations: string[]; migrationRequired: boolean; }> { try { const format = await this.detectFormat(sourcePath); const sampleData = await this.loadSampleData(sourcePath, format); const issues: string[] = []; const recommendations: string[] = []; let compatible = true; let migrationRequired = false; // Check format compatibility if (!this.getSupportedFormats().import.includes(format)) { issues.push(`Unsupported format: ${format}`); compatible = false; } // Check schema compatibility const schemaIssues = this.validateSchema(sampleData); if (schemaIssues.length > 0) { issues.push(...schemaIssues); migrationRequired = true; } // Check data integrity const integrityIssues = this.validateDataIntegrity(sampleData); if (integrityIssues.length > 0) { issues.push(...integrityIssues); recommendations.push("Consider data cleaning before import"); } // Generate recommendations if (migrationRequired) { recommendations.push("Create migration plan for schema transformation"); } if (format === "csv") { recommendations.push( "Consider using JSON or JSONL for better data preservation", ); } return { compatible, issues, recommendations, migrationRequired, }; } catch (error) { return { compatible: false, issues: [error instanceof Error ? error.message : String(error)], recommendations: ["Verify file format and accessibility"], migrationRequired: false, }; } } /** * Private helper methods */ private async getFilteredEntries( filters?: ExportOptions["filters"], ): Promise<MemoryEntry[]> { let entries = await this.storage.getAll(); if (!filters) return entries; // Apply type filter if (filters.types && filters.types.length > 0) { entries = entries.filter((entry) => filters.types!.includes(entry.type)); } // Apply date range filter if (filters.dateRange) { entries = entries.filter((entry) => { const entryDate = new Date(entry.timestamp); return ( entryDate >= filters.dateRange!.start && entryDate <= filters.dateRange!.end ); }); } // Apply project filter if (filters.projects && filters.projects.length > 0) { entries = entries.filter((entry) => filters.projects!.some( (project) => entry.data.projectPath?.includes(project) || entry.data.projectId === project, ), ); } // Apply tags filter if (filters.tags && filters.tags.length > 0) { entries = entries.filter( (entry) => entry.tags?.some((tag) => filters.tags!.includes(tag)), ); } // Apply outcomes filter if (filters.outcomes && filters.outcomes.length > 0) { entries = entries.filter( (entry) => filters.outcomes!.includes(entry.data.outcome) || (entry.data.success === true && filters.outcomes!.includes("success")) || (entry.data.success === false && filters.outcomes!.includes("failure")), ); } return entries; } private async prepareExportData( entries: MemoryEntry[], options: ExportOptions, ): Promise<any> { const exportData: any = { metadata: { version: this.version, exportedAt: new Date().toISOString(), source: "DocuMCP Memory System", entries: entries.length, options: { includeMetadata: options.includeMetadata, includeLearning: options.includeLearning, includeKnowledgeGraph: options.includeKnowledgeGraph, }, }, memories: entries, }; // Include learning data if requested if (options.includeLearning) { const patterns = await this.learningSystem.getPatterns(); exportData.learning = { patterns, statistics: await this.learningSystem.getStatistics(), }; } // Include knowledge graph if requested if (options.includeKnowledgeGraph) { const nodes = await this.knowledgeGraph.getAllNodes(); const edges = await this.knowledgeGraph.getAllEdges(); exportData.knowledgeGraph = { nodes, edges, statistics: await this.knowledgeGraph.getStatistics(), }; } return exportData; } private applyAnonymization( data: any, anonymizeOptions: { fields: string[]; method: string }, ): void { const anonymizeValue = (value: any, method: string): any => { if (typeof value !== "string") return value; switch (method) { case "hash": return this.hashValue(value); case "remove": return null; case "pseudonymize": return this.pseudonymizeValue(value); default: return value; } }; const anonymizeObject = (obj: any): void => { for (const [key, value] of Object.entries(obj)) { if (anonymizeOptions.fields.includes(key)) { obj[key] = anonymizeValue(value, anonymizeOptions.method); } else if (typeof value === "object" && value !== null) { anonymizeObject(value); } } }; anonymizeObject(data); } private hashValue(value: string): string { // Simple hash - in production, use a proper cryptographic hash let hash = 0; for (let i = 0; i < value.length; i++) { const char = value.charCodeAt(i); hash = (hash << 5) - hash + char; hash = hash & hash; } return `hash_${Math.abs(hash).toString(36)}`; } private pseudonymizeValue(_value: string): string { // Simple pseudonymization - in production, use proper techniques const prefixes = ["user", "project", "system", "item"]; const suffix = Math.random().toString(36).substr(2, 8); const prefix = prefixes[Math.floor(Math.random() * prefixes.length)]; return `${prefix}_${suffix}`; } private async exportToJSON( outputPath: string, data: any, _options: ExportOptions, ): Promise<string> { const jsonData = JSON.stringify(data, null, 2); // Handle compression-aware file paths (e.g., file.json.gz) let filePath = outputPath; if (!outputPath.includes(".json")) { filePath = `${outputPath}.json`; } await fs.writeFile(filePath, jsonData, "utf8"); return filePath; } private async exportToJSONL( outputPath: string, data: any, _options: ExportOptions, ): Promise<string> { const filePath = outputPath.endsWith(".jsonl") ? outputPath : `${outputPath}.jsonl`; return new Promise((resolve, reject) => { const writeStream = createWriteStream(filePath); writeStream.on("error", (error) => { reject(error); }); writeStream.on("finish", () => { resolve(filePath); }); // Write metadata as first line writeStream.write(JSON.stringify(data.metadata) + "\n"); // Write each memory entry as a separate line for (const entry of data.memories) { writeStream.write(JSON.stringify(entry) + "\n"); } // Write learning data if included if (data.learning) { writeStream.write( JSON.stringify({ type: "learning", data: data.learning }) + "\n", ); } // Write knowledge graph if included if (data.knowledgeGraph) { writeStream.write( JSON.stringify({ type: "knowledgeGraph", data: data.knowledgeGraph, }) + "\n", ); } writeStream.end(); }); } private async exportToCSV( outputPath: string, data: any, _options: ExportOptions, ): Promise<string> { const filePath = outputPath.endsWith(".csv") ? outputPath : `${outputPath}.csv`; // Flatten memory entries for CSV format const flattenedEntries = data.memories.map((entry: MemoryEntry) => ({ id: entry.id, timestamp: entry.timestamp, type: entry.type, projectPath: entry.data.projectPath || "", projectId: entry.data.projectId || "", language: entry.data.language || "", framework: entry.data.framework || "", outcome: entry.data.outcome || "", success: entry.data.success || false, tags: entry.tags?.join(";") || "", metadata: JSON.stringify(entry.metadata || {}), })); // Generate CSV headers const headers = Object.keys(flattenedEntries[0] || {}); const csvLines = [headers.join(",")]; // Generate CSV rows for (const entry of flattenedEntries) { const row = headers.map((header) => { const value = entry[header as keyof typeof entry]; const stringValue = typeof value === "string" ? value : JSON.stringify(value); return `"${stringValue.replace(/"/g, '""')}"`; }); csvLines.push(row.join(",")); } await fs.writeFile(filePath, csvLines.join("\n"), "utf8"); return filePath; } private async exportToXML( outputPath: string, data: any, _options: ExportOptions, ): Promise<string> { const filePath = outputPath.endsWith(".xml") ? outputPath : `${outputPath}.xml`; const xmlData = this.convertToXML(data); await fs.writeFile(filePath, xmlData, "utf8"); return filePath; } private async exportToYAML( outputPath: string, data: any, _options: ExportOptions, ): Promise<string> { const filePath = outputPath.endsWith(".yaml") ? outputPath : `${outputPath}.yaml`; // Simple YAML conversion - in production, use a proper YAML library const yamlData = this.convertToYAML(data); await fs.writeFile(filePath, yamlData, "utf8"); return filePath; } private async exportToSQLite( _outputPath: string, _data: any, _options: ExportOptions, ): Promise<string> { // This would require a SQLite library like better-sqlite3 // For now, throw an error indicating additional dependencies needed throw new Error( "SQLite export requires additional dependencies (better-sqlite3)", ); } private async exportToArchive( outputPath: string, data: any, options: ExportOptions, ): Promise<string> { const archivePath = outputPath.endsWith(".tar") ? outputPath : `${outputPath}.tar`; // Create archive metadata const metadata: ArchiveMetadata = { version: this.version, created: new Date(), source: "DocuMCP Memory System", description: "Complete memory system export archive", manifest: { files: [], total: { files: 0, size: 0, entries: data.memories.length }, }, options, }; // This would require archiving capabilities // For now, create multiple files and reference them in metadata const baseDir = archivePath.replace(".tar", ""); await fs.mkdir(baseDir, { recursive: true }); // Export memories as JSON const memoriesPath = `${baseDir}/memories.json`; await this.exportToJSON(memoriesPath, { memories: data.memories }, options); metadata.manifest.files.push({ name: "memories.json", type: "memories", size: (await fs.stat(memoriesPath)).size, checksum: "sha256-placeholder", entries: data.memories.length, }); // Export learning data if included if (data.learning) { const learningPath = `${baseDir}/learning.json`; await this.exportToJSON(learningPath, data.learning, options); metadata.manifest.files.push({ name: "learning.json", type: "learning", size: (await fs.stat(learningPath)).size, checksum: "sha256-placeholder", }); } // Export knowledge graph if included if (data.knowledgeGraph) { const kgPath = `${baseDir}/knowledge-graph.json`; await this.exportToJSON(kgPath, data.knowledgeGraph, options); metadata.manifest.files.push({ name: "knowledge-graph.json", type: "knowledge-graph", size: (await fs.stat(kgPath)).size, checksum: "sha256-placeholder", }); } // Write metadata const metadataPath = `${baseDir}/metadata.json`; await this.exportToJSON(metadataPath, metadata, options); return baseDir; } private async applyCompression( filePath: string, compression: string, targetPath?: string, ): Promise<string> { if (compression === "gzip") { const compressedPath = targetPath || `${filePath}.gz`; const content = await fs.readFile(filePath, "utf8"); // Simple mock compression - just add a header and write the content await fs.writeFile(compressedPath, `GZIP_HEADER\n${content}`, "utf8"); // Clean up temp file if we used one if (targetPath && targetPath !== filePath) { await fs.unlink(filePath); } return compressedPath; } // For other compression types or 'none', return original path this.emit("compression_skipped", { reason: "Not implemented", compression, }); return filePath; } private async applyEncryption( filePath: string, encryption: any, ): Promise<string> { // This would require encryption capabilities // For now, return the original path this.emit("encryption_skipped", { reason: "Not implemented", encryption }); return filePath; } private getIncludedComponents(options: ExportOptions): string[] { const components = ["memories"]; if (options.includeMetadata) components.push("metadata"); if (options.includeLearning) components.push("learning"); if (options.includeKnowledgeGraph) components.push("knowledge-graph"); return components; } private async detectFormat(filePath: string): Promise<string> { const extension = filePath.split(".").pop()?.toLowerCase(); switch (extension) { case "json": return "json"; case "jsonl": return "jsonl"; case "csv": return "csv"; case "xml": return "xml"; case "yaml": case "yml": return "yaml"; case "db": case "sqlite": return "sqlite"; case "tar": case "zip": return "archive"; default: { // Try to detect by content const content = await fs.readFile(filePath, "utf8"); if (content.trim().startsWith("{") || content.trim().startsWith("[")) { return "json"; } if (content.includes("<?xml")) { return "xml"; } return "unknown"; } } } private async loadImportData( filePath: string, options: ImportOptions, ): Promise<any> { switch (options.format) { case "json": return JSON.parse(await fs.readFile(filePath, "utf8")); case "jsonl": return this.loadJSONLData(filePath); case "csv": return this.loadCSVData(filePath); case "xml": return this.loadXMLData(filePath); case "yaml": return this.loadYAMLData(filePath); default: throw new Error(`Unsupported import format: ${options.format}`); } } private async loadJSONLData(filePath: string): Promise<any> { const content = await fs.readFile(filePath, "utf8"); const lines = content.trim().split("\n"); const data: any = { memories: [], learning: null, knowledgeGraph: null }; for (const line of lines) { const parsed = JSON.parse(line); if (parsed.type === "learning") { data.learning = parsed.data; } else if (parsed.type === "knowledgeGraph") { data.knowledgeGraph = parsed.data; } else if (parsed.version) { data.metadata = parsed; } else { data.memories.push(parsed); } } return data; } private async loadCSVData(filePath: string): Promise<any> { const content = await fs.readFile(filePath, "utf8"); const lines = content.trim().split("\n"); const headers = lines[0].split(",").map((h) => h.replace(/"/g, "")); const memories = []; for (let i = 1; i < lines.length; i++) { const values = this.parseCSVLine(lines[i]); const entry: any = {}; for (let j = 0; j < headers.length; j++) { const header = headers[j]; const value = values[j]; // Parse special fields if (header === "tags") { entry.tags = value ? value.split(";") : []; } else if (header === "metadata") { try { entry.metadata = JSON.parse(value); } catch { entry.metadata = {}; } } else if (header === "success") { entry.data = entry.data || {}; entry.data.success = value === "true"; } else if ( [ "projectPath", "projectId", "language", "framework", "outcome", ].includes(header) ) { entry.data = entry.data || {}; entry.data[header] = value; } else { entry[header] = value; } } memories.push(entry); } return { memories }; } private parseCSVLine(line: string): string[] { const values: string[] = []; let current = ""; let inQuotes = false; for (let i = 0; i < line.length; i++) { const char = line[i]; if (char === '"') { if (inQuotes && line[i + 1] === '"') { current += '"'; i++; } else { inQuotes = !inQuotes; } } else if (char === "," && !inQuotes) { values.push(current); current = ""; } else { current += char; } } values.push(current); return values; } private async loadXMLData(_filePath: string): Promise<any> { // This would require an XML parser throw new Error("XML import requires additional dependencies (xml2js)"); } private async loadYAMLData(_filePath: string): Promise<any> { // This would require a YAML parser throw new Error("YAML import requires additional dependencies (js-yaml)"); } private async validateImportData( data: any, options: ImportOptions, ): Promise<{ valid: number; invalid: number; warnings: string[] }> { const result = { valid: 0, invalid: 0, warnings: [] as string[] }; if (!data.memories || !Array.isArray(data.memories)) { result.warnings.push("No memories array found in import data"); return result; } for (const entry of data.memories) { if (this.validateMemoryEntry(entry, options.validation)) { result.valid++; } else { result.invalid++; } } return result; } private validateMemoryEntry(entry: any, validation: string): boolean { // Check for completely missing or null required fields if ( !entry.id || !entry.timestamp || entry.type === null || entry.type === undefined || entry.data === null ) { return false; // These are invalid regardless of validation level } if (!entry.type) { return validation !== "strict"; } if (validation === "strict") { return Boolean(entry.data && typeof entry.data === "object"); } // For loose validation, still require data to be defined (not null) if (validation === "loose" && entry.data === null) { return false; } return true; } private async processImportData(