Smart Tree - ST

//! Context Gathering System for Smart Tree //! //! This module searches across AI tool directories (~/.claude, ~/.windsurf, ~/.cursor, etc.) //! to gather project-related context and convert it into M8 format for processing. pub mod collab_session; pub mod cross_session; pub mod partnership; pub mod temporal; use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; use walkdir::WalkDir; // TODO: Use proper M8 wave format when available // use crate::mem8::wave::{MemoryWave, WaveGrid, FrequencyBand, SensorType}; // use crate::mem8::format::M8Writer; /// AI tool directories to search for context pub const AI_TOOL_DIRS: &[&str] = &[ ".claude", ".windsurf", ".cursor", ".continue", ".github/copilot", ".vscode", ".idea", ".zed", ]; /// File extensions that contain context information pub const CONTEXT_EXTENSIONS: &[&str] = &[ "json", "jsonl", "xml", "yaml", "yml", "toml", "md", "txt", "log", "conf", "config", "env", "settings", ]; /// Context gathering configuration #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GatherConfig { /// Directories to search (relative to home) pub search_dirs: Vec<String>, /// Additional custom directories pub custom_dirs: Vec<PathBuf>, /// File extensions to include pub extensions: Vec<String>, /// Project identifiers to look for pub project_identifiers: Vec<String>, /// Maximum file size to process (in bytes) pub max_file_size: usize, /// Enable recursive search in subdirectories pub recursive: bool, /// Privacy mode - redact sensitive information pub privacy_mode: bool, } impl Default for GatherConfig { fn default() -> Self { Self { search_dirs: AI_TOOL_DIRS.iter().map(|s| s.to_string()).collect(), custom_dirs: vec![], extensions: CONTEXT_EXTENSIONS.iter().map(|s| s.to_string()).collect(), project_identifiers: vec![], max_file_size: 10 * 1024 * 1024, // 10MB recursive: true, privacy_mode: true, } } } /// Represents gathered context from various sources #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GatheredContext { pub source_path: PathBuf, pub ai_tool: String, pub content_type: ContextType, pub content: ContextContent, pub metadata: HashMap<String, String>, pub relevance_score: f32, pub timestamp: chrono::DateTime<chrono::Utc>, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] pub enum ContextType { ChatHistory, ProjectSettings, CodeSnippets, Documentation, Configuration, SearchHistory, Bookmarks, CustomPrompts, ModelPreferences, WorkspaceState, } #[derive(Debug, Clone, Serialize, Deserialize)] pub enum ContextContent { Text(String), Json(serde_json::Value), Xml(String), Binary(Vec<u8>), } /// Main context gatherer pub struct ContextGatherer { config: GatherConfig, project_path: PathBuf, gathered_contexts: Vec<GatheredContext>, session_tracker: collab_session::CollaborativeSessionTracker, cross_session_bridge: cross_session::CrossSessionBridge, } impl ContextGatherer { pub fn new(project_path: PathBuf, config: GatherConfig) -> Self { Self { config, project_path, gathered_contexts: Vec::new(), session_tracker: collab_session::CollaborativeSessionTracker::new(), cross_session_bridge: cross_session::CrossSessionBridge::new(), } } /// Gather context from all configured sources pub fn gather_all(&mut self) -> Result<()> { let home_dir = dirs::home_dir().context("Failed to get home directory")?; // Clone to avoid borrow issues let search_dirs = self.config.search_dirs.clone(); let custom_dirs = self.config.custom_dirs.clone(); // Search AI tool directories for dir_name in search_dirs { let search_path = home_dir.join(&dir_name); if search_path.exists() { println!("🔍 Scanning {}", search_path.display()); self.scan_directory(&search_path, &dir_name)?; } } // Search custom directories for custom_dir in custom_dirs { if custom_dir.exists() { println!("🔍 Scanning custom: {}", custom_dir.display()); let tool_name = custom_dir .file_name() .and_then(|n| n.to_str()) .unwrap_or("custom"); self.scan_directory(&custom_dir, tool_name)?; } } // Post-process and score relevance self.score_relevance(); // Analyze for cross-domain patterns let patterns = self .cross_session_bridge .analyze_for_patterns(&self.gathered_contexts); if !patterns.is_empty() { println!("🔗 Found {} cross-domain patterns", patterns.len()); } // Generate insights if we have enough data let insights = self.cross_session_bridge.generate_insights(0.3); if !insights.is_empty() { println!("💡 Generated {} cross-session insights", insights.len()); } Ok(()) } /// Scan a directory for context files fn scan_directory(&mut self, path: &Path, ai_tool: &str) -> Result<()> { let walker = if self.config.recursive { WalkDir::new(path).max_depth(5) } else { WalkDir::new(path).max_depth(1) }; for entry in walker.into_iter().filter_map(|e| e.ok()) { let path = entry.path(); // Skip if not a file if !path.is_file() { continue; } // Check file extension if let Some(ext) = path.extension().and_then(|e| e.to_str()) { if !self.config.extensions.contains(&ext.to_string()) { continue; } // Check file size if let Ok(metadata) = fs::metadata(path) { if metadata.len() as usize > self.config.max_file_size { continue; } } // Process the file if let Ok(context) = self.process_file(path, ai_tool) { if self.is_relevant(&context) { // Track collaborative sessions let _ = self.session_tracker.process_context(&context); self.gathered_contexts.push(context); } } } } Ok(()) } /// Process a single file and extract context fn process_file(&self, path: &Path, ai_tool: &str) -> Result<GatheredContext> { let content = fs::read_to_string(path).context("Failed to read file")?; let ext = path.extension().and_then(|e| e.to_str()).unwrap_or(""); let (content_type, content) = match ext { "json" => self.parse_json(&content, path)?, "jsonl" => self.parse_jsonl(&content, path)?, "xml" => self.parse_xml(&content)?, "yaml" | "yml" => self.parse_yaml(&content)?, "md" => (ContextType::Documentation, ContextContent::Text(content)), _ => (ContextType::Configuration, ContextContent::Text(content)), }; Ok(GatheredContext { source_path: path.to_path_buf(), ai_tool: ai_tool.to_string(), content_type, content, metadata: self.extract_metadata(path), relevance_score: 0.0, // Will be calculated later timestamp: chrono::Utc::now(), }) } /// Parse JSON content and determine its type fn parse_json(&self, content: &str, path: &Path) -> Result<(ContextType, ContextContent)> { let json: serde_json::Value = serde_json::from_str(content)?; // Detect content type based on structure and filename let content_type = if path.to_str().unwrap_or("").contains("chat") { ContextType::ChatHistory } else if path.to_str().unwrap_or("").contains("settings") { ContextType::ProjectSettings } else if json.get("messages").is_some() { ContextType::ChatHistory } else if json.get("workspace").is_some() { ContextType::WorkspaceState } else { ContextType::Configuration }; // Apply privacy redaction if needed let json = if self.config.privacy_mode { self.redact_sensitive_json(json) } else { json }; Ok((content_type, ContextContent::Json(json))) } /// Parse JSONL (JSON Lines) format fn parse_jsonl(&self, content: &str, path: &Path) -> Result<(ContextType, ContextContent)> { let mut lines = Vec::new(); for line in content.lines() { if let Ok(json) = serde_json::from_str::<serde_json::Value>(line) { lines.push(json); } } let content_type = if path.to_str().unwrap_or("").contains("history") { ContextType::SearchHistory } else { ContextType::ChatHistory }; Ok(( content_type, ContextContent::Json(serde_json::Value::Array(lines)), )) } /// Parse XML content fn parse_xml(&self, content: &str) -> Result<(ContextType, ContextContent)> { // For now, store as text - could add proper XML parsing later Ok(( ContextType::Configuration, ContextContent::Xml(content.to_string()), )) } /// Parse YAML content fn parse_yaml(&self, content: &str) -> Result<(ContextType, ContextContent)> { let yaml: serde_yaml::Value = serde_yaml::from_str(content)?; let json = serde_json::to_value(yaml)?; Ok((ContextType::Configuration, ContextContent::Json(json))) } /// Check if context is relevant to the current project fn is_relevant(&self, context: &GatheredContext) -> bool { let project_name = self .project_path .file_name() .and_then(|n| n.to_str()) .unwrap_or(""); // Check content for project references let content_str = match &context.content { ContextContent::Text(t) => t.clone(), ContextContent::Json(j) => j.to_string(), ContextContent::Xml(x) => x.clone(), ContextContent::Binary(_) => return false, }; // Look for project name or identifiers if content_str .to_lowercase() .contains(&project_name.to_lowercase()) { return true; } for identifier in &self.config.project_identifiers { if content_str.contains(identifier) { return true; } } // Check path references let project_path_str = self.project_path.to_string_lossy(); if content_str.contains(project_path_str.as_ref()) { return true; } false } /// Score relevance of gathered contexts fn score_relevance(&mut self) { let project_name = self .project_path .file_name() .and_then(|n| n.to_str()) .unwrap_or("") .to_lowercase(); for context in &mut self.gathered_contexts { let mut score = 0.0; // Score based on content type score += match context.content_type { ContextType::ChatHistory => 0.8, ContextType::ProjectSettings => 0.9, ContextType::CodeSnippets => 0.7, ContextType::Documentation => 0.6, ContextType::CustomPrompts => 0.8, _ => 0.5, }; // Score based on recency let age_days = (chrono::Utc::now() - context.timestamp).num_days(); if age_days < 7 { score += 0.3; } else if age_days < 30 { score += 0.2; } else if age_days < 90 { score += 0.1; } // Score based on project name mentions let content_str = match &context.content { ContextContent::Text(t) => t.clone(), ContextContent::Json(j) => j.to_string(), ContextContent::Xml(x) => x.clone(), ContextContent::Binary(_) => String::new(), }; let mentions = content_str.to_lowercase().matches(&project_name).count(); score += (mentions as f32 * 0.1).min(0.5); context.relevance_score = score.min(1.0); } // Sort by relevance self.gathered_contexts .sort_by(|a, b| b.relevance_score.partial_cmp(&a.relevance_score).unwrap()); } /// Extract metadata from file path fn extract_metadata(&self, path: &Path) -> HashMap<String, String> { let mut metadata = HashMap::new(); if let Some(parent) = path.parent() { metadata.insert( "parent_dir".to_string(), parent.to_string_lossy().to_string(), ); } if let Ok(file_metadata) = fs::metadata(path) { if let Ok(modified) = file_metadata.modified() { metadata.insert( "modified".to_string(), chrono::DateTime::<chrono::Utc>::from(modified).to_rfc3339(), ); } metadata.insert("size".to_string(), file_metadata.len().to_string()); } metadata } /// Redact sensitive information from JSON #[allow(clippy::only_used_in_recursion)] fn redact_sensitive_json(&self, mut json: serde_json::Value) -> serde_json::Value { if let Some(obj) = json.as_object_mut() { for (key, value) in obj.iter_mut() { if key.contains("key") || key.contains("token") || key.contains("secret") || key.contains("password") { *value = serde_json::Value::String("[REDACTED]".to_string()); } else if value.is_object() || value.is_array() { *value = self.redact_sensitive_json(value.clone()); } } } else if let Some(arr) = json.as_array_mut() { for value in arr.iter_mut() { *value = self.redact_sensitive_json(value.clone()); } } json } /// Convert gathered contexts to M8 format pub fn to_m8(&self) -> Result<Vec<u8>> { // For now, create a simple JSON representation // TODO: Implement proper M8 wave-based format let m8_data = serde_json::json!({ "version": "1.0", "type": "context_gather", "metadata": { "project_path": self.project_path, "total_contexts": self.gathered_contexts.len(), "timestamp": chrono::Utc::now().to_rfc3339(), "top_sources": self.get_top_sources(), }, "contexts": self.gathered_contexts.iter().map(|c| { serde_json::json!({ "path": c.source_path.to_string_lossy(), "tool": c.ai_tool, "type": format!("{:?}", c.content_type), "score": c.relevance_score, "preview": match &c.content { ContextContent::Text(t) => t.chars().take(100).collect::<String>(), ContextContent::Json(j) => j.to_string().chars().take(100).collect::<String>(), ContextContent::Xml(x) => x.chars().take(100).collect::<String>(), ContextContent::Binary(b) => format!("[Binary: {} bytes]", b.len()), } }) }).collect::<Vec<_>>() }); // Compress with zlib for efficiency use flate2::write::ZlibEncoder; use flate2::Compression; use std::io::Write; let json_bytes = serde_json::to_vec(&m8_data)?; let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); encoder.write_all(&json_bytes)?; encoder.finish().map_err(Into::into) } /// Get summary of top context sources fn get_top_sources(&self) -> HashMap<String, usize> { let mut sources = HashMap::new(); for context in &self.gathered_contexts { *sources.entry(context.ai_tool.clone()).or_insert(0) += 1; } sources } /// Get gathered contexts pub fn contexts(&self) -> &[GatheredContext] { &self.gathered_contexts } /// Save gathered contexts to JSON file pub fn save_json(&self, path: &Path) -> Result<()> { let json = serde_json::to_string_pretty(&self.gathered_contexts)?; fs::write(path, json)?; Ok(()) } /// Perform temporal analysis on gathered contexts pub fn analyze_temporal( &self, resolution: temporal::TemporalResolution, ) -> temporal::TemporalPatterns { let analyzer = temporal::TemporalContextAnalyzer::new(self.gathered_contexts.clone(), resolution); analyzer.detect_patterns() } /// Get temporal wave grid representation pub fn create_temporal_waves( &self, resolution: temporal::TemporalResolution, ) -> temporal::TemporalWaveGrid { let analyzer = temporal::TemporalContextAnalyzer::new(self.gathered_contexts.clone(), resolution); analyzer.create_temporal_waves() } /// Apply temporal decay to relevance scores pub fn apply_temporal_decay(&mut self, half_life_days: f32) { let mut analyzer = temporal::TemporalContextAnalyzer::new( self.gathered_contexts.clone(), temporal::TemporalResolution::Day, ); analyzer.apply_temporal_decay(half_life_days); // Update our contexts with decayed scores self.gathered_contexts = analyzer.contexts; } /// Analyze AI-human partnership patterns pub fn analyze_partnership(&self) -> partnership::PartnershipAnalysis { let analyzer = partnership::PartnershipAnalyzer::new(self.gathered_contexts.clone()); analyzer.analyze_partnership() } /// Get collaborative session tracker pub fn session_tracker(&self) -> &collab_session::CollaborativeSessionTracker { &self.session_tracker } /// Get mutable collaborative session tracker pub fn session_tracker_mut(&mut self) -> &mut collab_session::CollaborativeSessionTracker { &mut self.session_tracker } /// Anchor an important collaborative memory pub fn anchor_memory( &mut self, origin: collab_session::CollaborativeOrigin, anchor_type: collab_session::AnchorType, context: String, keywords: Vec<String>, ) -> Result<String> { self.session_tracker .anchor_memory(origin, anchor_type, context, keywords) } /// Find memories related to keywords pub fn find_relevant_memories(&self, keywords: &[String]) -> Vec<String> { self.session_tracker .find_relevant_anchors(keywords) .into_iter() .map(|anchor| { format!( "[{}] {}: {} (keywords: {})", anchor.timestamp.format("%Y-%m-%d"), match &anchor.anchor_type { collab_session::AnchorType::PatternInsight => "Pattern", collab_session::AnchorType::Solution => "Solution", collab_session::AnchorType::Breakthrough => "Breakthrough", collab_session::AnchorType::LearningMoment => "Learning", collab_session::AnchorType::SharedJoke => "Joke", collab_session::AnchorType::TechnicalPattern => "Tech Pattern", collab_session::AnchorType::ProcessImprovement => "Process", }, anchor.context, anchor.keywords.join(", ") ) }) .collect() } /// Get co-engagement heatmap pub fn get_co_engagement_heatmap(&self) -> collab_session::CoEngagementHeatmap { let sessions: Vec<_> = self .session_tracker .session_history .iter() .cloned() .collect(); collab_session::CoEngagementHeatmap::from_sessions(&sessions) } /// Get cross-domain patterns pub fn get_cross_domain_patterns(&self) -> Vec<&cross_session::CrossDomainPattern> { self.cross_session_bridge.get_patterns() } /// Get relevant insights for current project pub fn get_relevant_insights( &self, keywords: &[String], ) -> Vec<cross_session::CrossSessionInsight> { self.cross_session_bridge .suggest_relevant_insights(&self.project_path, keywords) } /// Invite a persona for consultation pub fn invite_persona( &self, context: &str, duration: u32, ) -> Option<cross_session::PersonaInvitation> { self.cross_session_bridge.invite_persona(context, duration) } } // TODO: Implement M8Writer extension when M8 format is available