"""Utility functions for RAG loaders."""
def load_from_url(
url: str, kwargs: dict, accept_header: str = "*/*", loader_name: str = "Loader"
) -> str:
"""Load content from a URL.
Args:
url: The URL to fetch content from
kwargs: Additional keyword arguments (can include 'headers' override)
accept_header: The Accept header value for the request
loader_name: The name of the loader for the User-Agent header
Returns:
The text content from the URL
Raises:
ValueError: If there's an error fetching the URL
"""
import requests
headers = kwargs.get(
"headers",
{
"Accept": accept_header,
"User-Agent": f"Mozilla/5.0 (compatible; crewai-tools {loader_name})",
},
)
try:
response = requests.get(url, headers=headers, timeout=30)
response.raise_for_status()
return response.text
except Exception as e:
raise ValueError(f"Error fetching content from URL {url}: {e!s}") from e
