utils.py•1.69 kB
"""
Utility functions for RepoMap.
"""
import os
import sys
from pathlib import Path
from typing import Optional, List
from collections import namedtuple
try:
import tiktoken
except ImportError:
print("Error: tiktoken is required. Install with: pip install tiktoken")
sys.exit(1)
# Tag namedtuple for storing parsed code definitions and references
Tag = namedtuple("Tag", "rel_fname fname line name kind".split())
def count_tokens(text: str, model_name: str = "gpt-4") -> int:
"""Count tokens in text using tiktoken."""
if not text:
return 0
try:
encoding = tiktoken.encoding_for_model(model_name)
except KeyError:
# Fallback for unknown models
encoding = tiktoken.get_encoding("cl100k_base")
return len(encoding.encode(text))
def read_text(filename: str, encoding: str = "utf-8", silent: bool = False) -> Optional[str]:
"""Read text from file with error handling."""
try:
return Path(filename).read_text(encoding=encoding, errors='ignore')
except FileNotFoundError:
if not silent:
print(f"Error: {filename} not found.")
return None
except IsADirectoryError:
if not silent:
print(f"Error: {filename} is a directory.")
return None
except OSError as e:
if not silent:
print(f"Error reading {filename}: {e}")
return None
except UnicodeError as e:
if not silent:
print(f"Error decoding {filename}: {e}")
return None
except Exception as e:
if not silent:
print(f"An unexpected error occurred while reading {filename}: {e}")
return None