contextual-retrieval.ts•25.2 kB
/**
* Contextual Memory Retrieval System for DocuMCP
* Implements Issue #49: Contextual Memory Retrieval
*
* Provides intelligent, context-aware memory retrieval using semantic similarity,
* temporal relevance, and user intent analysis for enhanced recommendation accuracy.
*/
import { MemoryManager } from "./manager.js";
import { MemoryEntry } from "./storage.js";
import { KnowledgeGraph } from "./knowledge-graph.js";
export interface RetrievalContext {
currentProject?: {
path: string;
language: string;
framework?: string;
domain?: string;
size?: "small" | "medium" | "large";
};
userIntent?: {
action: "analyze" | "recommend" | "deploy" | "troubleshoot" | "learn";
urgency: "low" | "medium" | "high";
experience: "novice" | "intermediate" | "expert";
};
sessionContext?: {
recentActions: string[];
focusAreas: string[];
timeConstraints?: number; // minutes
};
temporalContext?: {
timeRange?: { start: string; end: string };
recency: "recent" | "all" | "historical";
seasonality?: boolean;
};
}
export interface SemanticEmbedding {
vector: number[];
metadata: {
source: string;
confidence: number;
generatedAt: string;
};
}
export interface ContextualMatch {
memory: MemoryEntry;
relevanceScore: number;
contextualFactors: {
semantic: number;
temporal: number;
structural: number;
intentional: number;
};
reasoning: string[];
confidence: number;
}
export interface RetrievalResult {
matches: ContextualMatch[];
metadata: {
queryContext: RetrievalContext;
totalCandidates: number;
processingTime: number;
fallbackUsed: boolean;
};
insights: {
patterns: string[];
recommendations: string[];
gaps: string[];
};
}
export class ContextualMemoryRetrieval {
private memoryManager: MemoryManager;
private knowledgeGraph: KnowledgeGraph;
private embeddingCache: Map<string, SemanticEmbedding>;
private readonly maxCacheSize = 1000;
private readonly similarityThreshold = 0.6;
constructor(memoryManager: MemoryManager, knowledgeGraph: KnowledgeGraph) {
this.memoryManager = memoryManager;
this.knowledgeGraph = knowledgeGraph;
this.embeddingCache = new Map();
}
/**
* Retrieve contextually relevant memories
*/
async retrieve(
query: string,
context: RetrievalContext,
options?: {
maxResults?: number;
minRelevance?: number;
includeReasoning?: boolean;
},
): Promise<RetrievalResult> {
const startTime = Date.now();
const maxResults = options?.maxResults || 10;
const minRelevance = options?.minRelevance || 0.3;
// Get candidate memories based on basic filtering
const candidates = await this.getCandidateMemories(query, context);
// Score and rank candidates
const scoredMatches = await this.scoreAndRankCandidates(
candidates,
query,
context,
);
// Filter by relevance threshold
const relevantMatches = scoredMatches
.filter((match) => match.relevanceScore >= minRelevance)
.slice(0, maxResults);
// Generate insights from matches
const insights = await this.generateInsights(relevantMatches, context);
const processingTime = Date.now() - startTime;
return {
matches: relevantMatches,
metadata: {
queryContext: context,
totalCandidates: candidates.length,
processingTime,
fallbackUsed: relevantMatches.length === 0 && candidates.length > 0,
},
insights,
};
}
/**
* Get candidate memories using multiple retrieval strategies
*/
private async getCandidateMemories(
query: string,
context: RetrievalContext,
): Promise<MemoryEntry[]> {
const candidates = new Map<string, MemoryEntry>();
// Strategy 1: Text-based search
const textMatches = await this.memoryManager.search(query, {
sortBy: "timestamp",
});
textMatches.forEach((memory) => candidates.set(memory.id, memory));
// Strategy 2: Context-based filtering
if (context.currentProject) {
const contextMatches = await this.getContextBasedCandidates(
context.currentProject,
);
contextMatches.forEach((memory) => candidates.set(memory.id, memory));
}
// Strategy 3: Intent-based retrieval
if (context.userIntent) {
const intentMatches = await this.getIntentBasedCandidates(
context.userIntent,
);
intentMatches.forEach((memory) => candidates.set(memory.id, memory));
}
// Strategy 4: Temporal filtering
if (context.temporalContext) {
const temporalMatches = await this.getTemporalCandidates(
context.temporalContext,
);
temporalMatches.forEach((memory) => candidates.set(memory.id, memory));
}
// Strategy 5: Knowledge graph traversal
const graphMatches = await this.getGraphBasedCandidates(query, context);
graphMatches.forEach((memory) => candidates.set(memory.id, memory));
return Array.from(candidates.values());
}
/**
* Get candidates based on current project context
*/
private async getContextBasedCandidates(
project: NonNullable<RetrievalContext["currentProject"]>,
): Promise<MemoryEntry[]> {
const searchCriteria = [];
// Language-based search
searchCriteria.push(
this.memoryManager
.search("", { sortBy: "timestamp" })
.then((memories) =>
memories.filter(
(m) =>
m.data.language?.primary === project.language ||
m.metadata.tags?.includes(project.language),
),
),
);
// Framework-based search
if (project.framework) {
searchCriteria.push(
this.memoryManager
.search("", { sortBy: "timestamp" })
.then((memories) =>
memories.filter(
(m) =>
m.data.framework?.name === project.framework ||
(project.framework &&
m.metadata.tags?.includes(project.framework)),
),
),
);
}
// Project size similarity
if (project.size) {
searchCriteria.push(
this.memoryManager
.search("", { sortBy: "timestamp" })
.then((memories) =>
memories.filter(
(m) =>
this.categorizeProjectSize(m.data.stats?.files || 0) ===
project.size,
),
),
);
}
const results = await Promise.all(searchCriteria);
const allMatches = results.flat();
// Deduplicate
const unique = new Map<string, MemoryEntry>();
allMatches.forEach((memory) => unique.set(memory.id, memory));
return Array.from(unique.values());
}
/**
* Get candidates based on user intent
*/
private async getIntentBasedCandidates(
intent: NonNullable<RetrievalContext["userIntent"]>,
): Promise<MemoryEntry[]> {
const intentTypeMap = {
analyze: ["analysis", "evaluation", "assessment"],
recommend: ["recommendation", "suggestion", "advice"],
deploy: ["deployment", "publish", "release"],
troubleshoot: ["error", "issue", "problem", "debug"],
learn: ["tutorial", "guide", "example", "pattern"],
};
const searchTerms = intentTypeMap[intent.action] || [intent.action];
const searches = searchTerms.map((term) =>
this.memoryManager.search(term, { sortBy: "timestamp" }),
);
const results = await Promise.all(searches);
const allMatches = results.flat();
// Filter by experience level
return allMatches.filter((memory) => {
if (intent.experience === "novice") {
return (
!memory.metadata.tags?.includes("advanced") &&
!memory.metadata.tags?.includes("expert")
);
} else if (intent.experience === "expert") {
return (
memory.metadata.tags?.includes("advanced") ||
memory.metadata.tags?.includes("expert") ||
memory.data.complexity === "complex"
);
}
return true; // intermediate gets all
});
}
/**
* Get candidates based on temporal context
*/
private async getTemporalCandidates(
temporal: NonNullable<RetrievalContext["temporalContext"]>,
): Promise<MemoryEntry[]> {
const searchOptions: any = { sortBy: "timestamp" };
if (temporal.timeRange) {
// Use memory manager's built-in time filtering
const allMemories = await this.memoryManager.search("", searchOptions);
return allMemories.filter((memory) => {
const memoryTime = new Date(memory.timestamp);
const start = new Date(temporal.timeRange!.start);
const end = new Date(temporal.timeRange!.end);
return memoryTime >= start && memoryTime <= end;
});
}
if (temporal.recency === "recent") {
const cutoff = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); // Last 7 days
const allMemories = await this.memoryManager.search("", searchOptions);
return allMemories.filter(
(memory) => new Date(memory.timestamp) > cutoff,
);
}
if (temporal.recency === "historical") {
const cutoff = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000); // Older than 90 days
const allMemories = await this.memoryManager.search("", searchOptions);
return allMemories.filter(
(memory) => new Date(memory.timestamp) < cutoff,
);
}
return this.memoryManager.search("", searchOptions);
}
/**
* Get candidates using knowledge graph traversal
*/
private async getGraphBasedCandidates(
query: string,
context: RetrievalContext,
): Promise<MemoryEntry[]> {
if (!context.currentProject) return [];
// Find relevant nodes in the knowledge graph
const graphQuery = {
nodeTypes: ["project", "technology"],
properties: context.currentProject.language
? {
language: context.currentProject.language,
}
: undefined,
maxDepth: 2,
};
const graphResult = this.knowledgeGraph.query(graphQuery);
const relevantNodeIds = graphResult.nodes.map((node) => node.id);
// Find memories associated with these nodes
const memories: MemoryEntry[] = [];
const allMemories = await this.memoryManager.search("", {
sortBy: "timestamp",
});
for (const memory of allMemories) {
const projectNodeId = `project:${memory.metadata.projectId}`;
const techNodeId = memory.metadata.ssg
? `tech:${memory.metadata.ssg}`
: null;
if (
relevantNodeIds.includes(projectNodeId) ||
(techNodeId && relevantNodeIds.includes(techNodeId))
) {
memories.push(memory);
}
}
return memories;
}
/**
* Score and rank candidates based on contextual relevance
*/
private async scoreAndRankCandidates(
candidates: MemoryEntry[],
query: string,
context: RetrievalContext,
): Promise<ContextualMatch[]> {
const matches: ContextualMatch[] = [];
for (const memory of candidates) {
const contextualFactors = await this.calculateContextualFactors(
memory,
query,
context,
);
const relevanceScore = this.calculateOverallRelevance(contextualFactors);
const reasoning = this.generateReasoning(
memory,
contextualFactors,
context,
);
const confidence = this.calculateConfidence(contextualFactors, memory);
matches.push({
memory,
relevanceScore,
contextualFactors,
reasoning,
confidence,
});
}
// Sort by relevance score (descending)
return matches.sort((a, b) => b.relevanceScore - a.relevanceScore);
}
/**
* Calculate contextual factors for scoring
*/
private async calculateContextualFactors(
memory: MemoryEntry,
query: string,
context: RetrievalContext,
): Promise<ContextualMatch["contextualFactors"]> {
const semantic = await this.calculateSemanticSimilarity(memory, query);
const temporal = await this.calculateTemporalRelevance(memory, context);
const structural = this.calculateStructuralRelevance(memory, context);
const intentional = this.calculateIntentionalRelevance(memory, context);
return { semantic, temporal, structural, intentional };
}
/**
* Calculate semantic similarity using simple text matching
* (In a production system, this would use embeddings)
*/
private async calculateSemanticSimilarity(
memory: MemoryEntry,
query: string,
): Promise<number> {
const queryTerms = query.toLowerCase().split(/\s+/);
const memoryText = JSON.stringify(memory.data).toLowerCase();
const metadataText = JSON.stringify(memory.metadata).toLowerCase();
let matches = 0;
for (const term of queryTerms) {
if (memoryText.includes(term) || metadataText.includes(term)) {
matches++;
}
}
return queryTerms.length > 0 ? matches / queryTerms.length : 0;
}
/**
* Calculate temporal relevance based on recency and context
*/
private calculateTemporalRelevance(
memory: MemoryEntry,
context: RetrievalContext,
): Promise<number> {
const memoryDate = new Date(memory.timestamp);
const now = new Date();
const daysSince =
(now.getTime() - memoryDate.getTime()) / (1000 * 60 * 60 * 24);
// Base score decreases with age
let score = Math.exp(-daysSince / 30); // Half-life of 30 days
// Boost for explicit temporal preferences
if (context.temporalContext?.recency === "recent" && daysSince <= 7) {
score *= 1.5;
} else if (
context.temporalContext?.recency === "historical" &&
daysSince >= 90
) {
score *= 1.3;
}
// Consider time constraints
if (context.sessionContext?.timeConstraints) {
const urgencyMultiplier =
context.userIntent?.urgency === "high" ? 1.2 : 1.0;
score *= urgencyMultiplier;
}
return Promise.resolve(Math.min(score, 1.0));
}
/**
* Calculate structural relevance based on project similarity
*/
private calculateStructuralRelevance(
memory: MemoryEntry,
context: RetrievalContext,
): number {
if (!context.currentProject) return 0.5; // Neutral when no project context
let score = 0;
let factors = 0;
// Language match
if (memory.data.language?.primary === context.currentProject.language) {
score += 0.4;
}
factors++;
// Framework match
if (
context.currentProject.framework &&
memory.data.framework?.name === context.currentProject.framework
) {
score += 0.3;
}
factors++;
// Size similarity
if (context.currentProject.size) {
const memorySize = this.categorizeProjectSize(
memory.data.stats?.files || 0,
);
if (memorySize === context.currentProject.size) {
score += 0.2;
}
}
factors++;
// Type relevance
if (
memory.type === "analysis" &&
context.userIntent?.action === "analyze"
) {
score += 0.1;
} else if (
memory.type === "recommendation" &&
context.userIntent?.action === "recommend"
) {
score += 0.1;
}
factors++;
return factors > 0 ? score / factors : 0;
}
/**
* Calculate intentional relevance based on user intent
*/
private calculateIntentionalRelevance(
memory: MemoryEntry,
context: RetrievalContext,
): number {
if (!context.userIntent) return 0.5; // Neutral when no intent
let score = 0;
// Action alignment
const actionTypeMap = {
analyze: ["analysis", "evaluation"],
recommend: ["recommendation"],
deploy: ["deployment"],
troubleshoot: ["deployment", "configuration"],
learn: ["analysis", "recommendation"],
};
const relevantTypes = actionTypeMap[context.userIntent.action] || [];
if (relevantTypes.includes(memory.type)) {
score += 0.5;
}
// Experience level alignment
if (context.userIntent.experience === "novice") {
// Prefer simpler, more successful cases
if (
memory.data.status === "success" ||
memory.data.complexity !== "complex"
) {
score += 0.3;
}
} else if (context.userIntent.experience === "expert") {
// Prefer complex or edge cases
if (
memory.data.complexity === "complex" ||
memory.metadata.tags?.includes("advanced")
) {
score += 0.3;
}
}
// Urgency consideration
if (context.userIntent.urgency === "high") {
// Prefer recent, successful cases
const daysSince =
(Date.now() - new Date(memory.timestamp).getTime()) /
(1000 * 60 * 60 * 24);
if (daysSince <= 7 && memory.data.status === "success") {
score += 0.2;
}
}
return Math.min(score, 1.0);
}
/**
* Calculate overall relevance score
*/
private calculateOverallRelevance(
factors: ContextualMatch["contextualFactors"],
): number {
// Weighted combination of factors
const weights = {
semantic: 0.3,
temporal: 0.2,
structural: 0.3,
intentional: 0.2,
};
return (
factors.semantic * weights.semantic +
factors.temporal * weights.temporal +
factors.structural * weights.structural +
factors.intentional * weights.intentional
);
}
/**
* Generate reasoning for why a memory was selected
*/
private generateReasoning(
memory: MemoryEntry,
factors: ContextualMatch["contextualFactors"],
context: RetrievalContext,
): string[] {
const reasoning: string[] = [];
if (factors.semantic > 0.7) {
reasoning.push("High semantic similarity to query");
}
if (factors.temporal > 0.8) {
reasoning.push("Recently relevant information");
}
if (factors.structural > 0.6) {
reasoning.push(
`Similar project structure (${
memory.data.language?.primary || "unknown"
})`,
);
}
if (factors.intentional > 0.7) {
reasoning.push(
`Matches user intent for ${
context.userIntent?.action || "general"
} action`,
);
}
if (
memory.data.status === "success" &&
context.userIntent?.urgency === "high"
) {
reasoning.push("Proven successful approach for urgent needs");
}
if (memory.metadata.ssg && context.currentProject?.framework) {
reasoning.push(
`Experience with ${memory.metadata.ssg} for similar projects`,
);
}
return reasoning.length > 0 ? reasoning : ["General relevance to query"];
}
/**
* Calculate confidence in the match
*/
private calculateConfidence(
factors: ContextualMatch["contextualFactors"],
memory: MemoryEntry,
): number {
let confidence = (factors.semantic + factors.structural) / 2;
// Boost confidence for successful outcomes
if (memory.data.status === "success") {
confidence *= 1.2;
}
// Boost confidence for recent data
const daysSince =
(Date.now() - new Date(memory.timestamp).getTime()) /
(1000 * 60 * 60 * 24);
if (daysSince <= 30) {
confidence *= 1.1;
}
// Boost confidence for rich metadata
if (memory.metadata.tags && memory.metadata.tags.length > 2) {
confidence *= 1.05;
}
return Math.min(confidence, 1.0);
}
/**
* Generate insights from retrieved matches
*/
private async generateInsights(
matches: ContextualMatch[],
context: RetrievalContext,
): Promise<RetrievalResult["insights"]> {
const patterns: string[] = [];
const recommendations: string[] = [];
const gaps: string[] = [];
if (matches.length === 0) {
gaps.push("No relevant memories found for current context");
recommendations.push(
"Consider expanding search criteria or building more experience",
);
return { patterns, recommendations, gaps };
}
// Analyze patterns in successful matches
const successfulMatches = matches.filter(
(m) => m.memory.data.status === "success" && m.relevanceScore > 0.6,
);
if (successfulMatches.length >= 2) {
// Find common SSGs
const ssgs = new Map<string, number>();
successfulMatches.forEach((match) => {
if (match.memory.metadata.ssg) {
ssgs.set(
match.memory.metadata.ssg,
(ssgs.get(match.memory.metadata.ssg) || 0) + 1,
);
}
});
if (ssgs.size > 0) {
const topSSG = Array.from(ssgs.entries()).sort(
([, a], [, b]) => b - a,
)[0];
patterns.push(
`${topSSG[0]} appears in ${topSSG[1]} successful similar projects`,
);
recommendations.push(
`Consider ${topSSG[0]} based on successful precedents`,
);
}
// Find common success factors
const commonFactors = this.findCommonSuccessFactors(successfulMatches);
patterns.push(...commonFactors);
}
// Identify gaps
if (
context.userIntent?.action === "deploy" &&
matches.filter((m) => m.memory.type === "deployment").length === 0
) {
gaps.push("Limited deployment experience for similar projects");
recommendations.push(
"Proceed cautiously with deployment and document the process",
);
}
if (
context.userIntent?.experience === "novice" &&
matches.every((m) => m.confidence < 0.7)
) {
gaps.push("Limited beginner-friendly resources for this context");
recommendations.push(
"Consider consulting documentation or seeking expert guidance",
);
}
return { patterns, recommendations, gaps };
}
/**
* Find common success factors across matches
*/
private findCommonSuccessFactors(matches: ContextualMatch[]): string[] {
const factors: string[] = [];
const hasTests = matches.filter(
(m) => m.memory.data.testing?.hasTests,
).length;
if (hasTests / matches.length > 0.7) {
factors.push("Projects with testing have higher success rates");
}
const hasCI = matches.filter((m) => m.memory.data.ci?.hasCI).length;
if (hasCI / matches.length > 0.6) {
factors.push("CI/CD adoption correlates with deployment success");
}
const simpleProjects = matches.filter(
(m) => m.memory.data.complexity !== "complex",
).length;
if (simpleProjects / matches.length > 0.8) {
factors.push("Simpler project structures show more reliable outcomes");
}
return factors;
}
/**
* Categorize project size for comparison
*/
private categorizeProjectSize(
fileCount: number,
): "small" | "medium" | "large" {
if (fileCount < 50) return "small";
if (fileCount < 200) return "medium";
return "large";
}
/**
* Get contextual suggestions for improving retrieval
*/
async getSuggestions(context: RetrievalContext): Promise<{
queryImprovements: string[];
contextEnhancements: string[];
learningOpportunities: string[];
}> {
const suggestions = {
queryImprovements: [] as string[],
contextEnhancements: [] as string[],
learningOpportunities: [] as string[],
};
// Analyze current context completeness
if (!context.currentProject) {
suggestions.contextEnhancements.push(
"Provide current project information for better matches",
);
}
if (!context.userIntent) {
suggestions.contextEnhancements.push(
"Specify your intent (analyze, recommend, deploy, etc.) for targeted results",
);
}
if (!context.temporalContext) {
suggestions.contextEnhancements.push(
"Set temporal preferences (recent vs. historical) for relevance",
);
}
// Analyze retrieval patterns
const recentSearches = await this.memoryManager.search("search", {
sortBy: "timestamp",
});
if (recentSearches.length < 5) {
suggestions.learningOpportunities.push(
"System will improve with more usage and data",
);
}
// Check for data gaps
if (context.currentProject?.language) {
const languageMemories = await this.memoryManager.search(
context.currentProject.language,
);
if (languageMemories.length < 3) {
suggestions.learningOpportunities.push(
`More experience needed with ${context.currentProject.language} projects`,
);
}
}
return suggestions;
}
/**
* Clear embedding cache
*/
clearCache(): void {
this.embeddingCache.clear();
}
/**
* Get retrieval statistics
*/
getStatistics(): {
cacheSize: number;
cacheHitRate: number;
averageRetrievalTime: number;
commonContextTypes: Record<string, number>;
} {
// This would track actual usage statistics in a real implementation
return {
cacheSize: this.embeddingCache.size,
cacheHitRate: 0.85, // Placeholder
averageRetrievalTime: 150, // ms
commonContextTypes: {
project_analysis: 45,
ssg_recommendation: 38,
deployment_troubleshooting: 12,
learning_assistance: 5,
},
};
}
}
export default ContextualMemoryRetrieval;