//! Directory indexing.
//!
//! Walks directories to find and parse source files, building
//! the initial code graph.
use arbor_core::{parse_file, CodeNode};
use arbor_graph::{ArborGraph, GraphBuilder, GraphStore};
use ignore::WalkBuilder;
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::time::Instant;
use tracing::{debug, info, warn};
/// Result of indexing a directory.
pub struct IndexResult {
/// The built graph.
pub graph: ArborGraph,
/// Number of files processed (parsed fresh).
pub files_indexed: usize,
/// Number of files loaded from cache.
pub cache_hits: usize,
/// Number of nodes extracted.
pub nodes_extracted: usize,
/// Time taken in milliseconds.
pub duration_ms: u64,
/// Files that failed to parse.
pub errors: Vec<(String, String)>,
}
/// Options for directory indexing.
#[derive(Debug, Clone, Default)]
pub struct IndexOptions {
/// Follow symbolic links when walking directories.
pub follow_symlinks: bool,
/// Path to cache directory (e.g., `.arbor/cache`).
/// If None, caching is disabled.
pub cache_path: Option<PathBuf>,
}
/// Indexes a directory and returns the code graph.
///
/// This walks all source files, parses them, and builds the
/// relationship graph. It respects .gitignore patterns.
///
/// If `options.cache_path` is set, files are cached with their mtimes.
/// Only files with changed mtimes are re-parsed.
///
/// # Example
///
/// ```no_run
/// use arbor_watcher::{index_directory, IndexOptions};
/// use std::path::Path;
///
/// let result = index_directory(Path::new("./src"), IndexOptions::default()).unwrap();
/// println!("Indexed {} files, {} nodes", result.files_indexed, result.nodes_extracted);
/// ```
pub fn index_directory(root: &Path, options: IndexOptions) -> Result<IndexResult, std::io::Error> {
let start = Instant::now();
let mut builder = GraphBuilder::new();
let mut files_indexed = 0;
let mut cache_hits = 0;
let mut nodes_extracted = 0;
let mut errors = Vec::new();
info!("Starting index of {}", root.display());
// Open cache if configured
let store =
options
.cache_path
.as_ref()
.and_then(|path| match GraphStore::open_or_reset(path) {
Ok(s) => Some(s),
Err(e) => {
warn!("Failed to open cache: {}, proceeding without cache", e);
None
}
});
// Track files we've seen (for detecting deleted files)
let mut seen_files: HashSet<String> = HashSet::new();
// Walk the directory, respecting .gitignore
let walker = WalkBuilder::new(root)
.hidden(true) // Skip hidden files
.git_ignore(true) // Respect .gitignore
.git_global(true)
.git_exclude(true)
.follow_links(options.follow_symlinks)
.build();
for entry in walker.filter_map(Result::ok) {
let path = entry.path();
// Skip directories
if path.is_dir() {
continue;
}
// Check if it's a supported file type
let extension = match path.extension().and_then(|e| e.to_str()) {
Some(ext) => ext,
None => continue,
};
if !arbor_core::languages::is_supported(extension) {
continue;
}
let path_str = path.display().to_string();
seen_files.insert(path_str.clone());
// Check cache
if let Some(ref store) = store {
// Get file mtime
let current_mtime = match std::fs::metadata(path) {
Ok(meta) => meta
.modified()
.ok()
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| d.as_secs())
.unwrap_or(0),
Err(_) => 0,
};
// Check cached mtime
if let Ok(Some(cached_mtime)) = store.get_mtime(&path_str) {
if cached_mtime == current_mtime {
// File unchanged, load from cache
if let Ok(Some(cached_nodes)) = store.get_file_nodes(&path_str) {
debug!("Cache hit: {}", path.display());
nodes_extracted += cached_nodes.len();
cache_hits += 1;
builder.add_nodes(cached_nodes);
continue;
}
}
}
// Cache miss or stale, parse file
debug!("Parsing (cache miss): {}", path.display());
match parse_file(path) {
Ok(nodes) => {
nodes_extracted += nodes.len();
files_indexed += 1;
// Update cache
if let Err(e) = store.update_file(&path_str, &nodes, current_mtime) {
warn!("Failed to update cache for {}: {}", path_str, e);
}
builder.add_nodes(nodes);
}
Err(e) => {
warn!("Failed to parse {}: {}", path.display(), e);
errors.push((path_str, e.to_string()));
}
}
} else {
// No cache, parse directly
debug!("Parsing {}", path.display());
match parse_file(path) {
Ok(nodes) => {
nodes_extracted += nodes.len();
files_indexed += 1;
builder.add_nodes(nodes);
}
Err(e) => {
warn!("Failed to parse {}: {}", path.display(), e);
errors.push((path_str, e.to_string()));
}
}
}
}
// Handle deleted files: remove from cache any files that no longer exist
if let Some(ref store) = store {
if let Ok(cached_files) = store.list_cached_files() {
for cached_file in cached_files {
if !seen_files.contains(&cached_file) {
debug!("Removing deleted file from cache: {}", cached_file);
if let Err(e) = store.remove_file(&cached_file) {
warn!("Failed to remove {} from cache: {}", cached_file, e);
}
}
}
}
}
let graph = builder.build();
let duration = start.elapsed();
info!(
"Indexed {} files, {} cache hits ({} nodes) in {:?}",
files_indexed, cache_hits, nodes_extracted, duration
);
Ok(IndexResult {
graph,
files_indexed,
cache_hits,
nodes_extracted,
duration_ms: duration.as_millis() as u64,
errors,
})
}
/// Parses a single file and returns its nodes.
#[allow(dead_code)]
pub fn parse_single_file(path: &Path) -> Result<Vec<CodeNode>, arbor_core::ParseError> {
parse_file(path)
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
use tempfile::tempdir;
#[test]
fn test_index_empty_directory() {
let dir = tempdir().unwrap();
let result = index_directory(dir.path(), IndexOptions::default()).unwrap();
assert_eq!(result.files_indexed, 0);
assert_eq!(result.nodes_extracted, 0);
}
#[test]
fn test_index_with_rust_file() {
let dir = tempdir().unwrap();
let file_path = dir.path().join("test.rs");
fs::write(
&file_path,
r#"
pub fn hello() {
println!("Hello!");
}
"#,
)
.unwrap();
let result = index_directory(dir.path(), IndexOptions::default()).unwrap();
assert_eq!(result.files_indexed, 1);
assert!(result.nodes_extracted > 0);
}
/// Helper to create a directory symlink cross-platform.
/// Returns None if symlink creation fails (e.g., no privileges on Windows).
fn create_dir_symlink(original: &std::path::Path, link: &std::path::Path) -> Option<()> {
#[cfg(unix)]
{
std::os::unix::fs::symlink(original, link).ok()
}
#[cfg(windows)]
{
std::os::windows::fs::symlink_dir(original, link).ok()
}
#[cfg(not(any(unix, windows)))]
{
None
}
}
#[test]
fn test_index_does_not_follow_symlinks_by_default() {
let dir = tempdir().unwrap();
let linked_dir = tempdir().unwrap();
// Create a file in the linked directory
let linked_file = linked_dir.path().join("linked.rs");
fs::write(&linked_file, "pub fn linked_func() {}").unwrap();
// Create a symlink to the linked directory
let symlink_path = dir.path().join("linked");
if create_dir_symlink(linked_dir.path(), &symlink_path).is_none() {
// Skip test if symlinks not supported (e.g., Windows without privileges)
return;
}
// Index without following symlinks (default)
let result = index_directory(dir.path(), IndexOptions::default()).unwrap();
assert_eq!(result.files_indexed, 0);
}
#[test]
fn test_index_follows_symlinks_when_enabled() {
let dir = tempdir().unwrap();
let linked_dir = tempdir().unwrap();
// Create a file in the linked directory
let linked_file = linked_dir.path().join("linked.rs");
fs::write(&linked_file, "pub fn linked_func() {}").unwrap();
// Create a symlink to the linked directory
let symlink_path = dir.path().join("linked");
if create_dir_symlink(linked_dir.path(), &symlink_path).is_none() {
// Skip test if symlinks not supported (e.g., Windows without privileges)
return;
}
// Index with follow_symlinks enabled
let options = IndexOptions {
follow_symlinks: true,
cache_path: None,
};
let result = index_directory(dir.path(), options).unwrap();
assert_eq!(result.files_indexed, 1);
assert!(result.nodes_extracted > 0);
}
}
