[project]
name = "crawl4ai-mcp"
version = "0.1.0"
description = "MCP server for integrating web crawling and RAG into AI agents and AI coding assistants"
readme = "README.md"
requires-python = ">=3.12"
dependencies = [
"torch>=2.0.0", # Explicitly specify torch first to control version
"crawl4ai==0.7.6",
"fastmcp>=2.14.1",
"supabase>=2.15.1",
"openai>=1.107.2",
"pydantic>=2.0.0",
"pydantic-settings>=2.0.0",
"pydantic-ai>=1.18.0", # Pydantic AI for agent framework
"dotenv==0.9.9",
"sentence-transformers>=4.1.0",
"neo4j>=5.28.1",
"requests>=2.25.0",
"httpx>=0.27.0",
"qdrant-client>=1.15.1",
"pytest>=7.0.0",
"pytest-asyncio>=0.21.0",
"pytest-cov>=4.0.0",
"pytest-xdist>=3.5.0",
"pytest-json-report>=1.5.0",
"docker>=6.0.0",
"testcontainers>=3.7.0",
"psutil>=5.9.0",
"ruff>=0.8.0",
"python-jose[cryptography]>=3.3.0",
"passlib[bcrypt]>=1.7.4",
"python-multipart>=0.0.6",
"authlib>=1.3.0",
"jinja2>=3.1.0",
"itsdangerous>=2.1.0",
"mypy>=1.0.0",
"types-requests>=2.31.0",
"types-psutil>=5.9.0",
"types-docker>=6.0.0",
"types-passlib>=1.7.7.20250602",
"types-jinja2>=2.11.9",
"types-python-jose>=3.5.0.20250531",
]
[project.scripts]
crawl4ai-mcp = "src.main:main"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src"]
# ========================================
# RUFF CONFIGURATION
# ========================================
[tool.ruff]
target-version = "py312"
line-length = 88
indent-width = 4
exclude = [
".git",
".pytest_cache",
"__pycache__",
".venv",
"htmlcov",
"archived_configs",
"searxng-backup",
"searxng-test",
"qdrant-config",
"logs",
]
[tool.ruff.lint]
# Enable comprehensive rule sets
select = [
"E", # pycodestyle errors
"W", # pycodestyle warnings
"F", # Pyflakes
"I", # isort
"N", # pep8-naming
"D", # pydocstyle
"UP", # pyupgrade
"YTT", # flake8-2020
"ANN", # flake8-annotations
"S", # flake8-bandit
"BLE", # flake8-blind-except
"FBT", # flake8-boolean-trap
"B", # flake8-bugbear
"A", # flake8-builtins
"COM", # flake8-commas
"C4", # flake8-comprehensions
"DTZ", # flake8-datetimez
"T10", # flake8-debugger
"EM", # flake8-errmsg
"FA", # flake8-future-annotations
"ISC", # flake8-implicit-str-concat
"ICN", # flake8-import-conventions
"G", # flake8-logging-format
"INP", # flake8-no-pep420
"PIE", # flake8-pie
"T20", # flake8-print
"PYI", # flake8-pyi
"PT", # flake8-pytest-style
"Q", # flake8-quotes
"RSE", # flake8-raise
"RET", # flake8-return
"SLF", # flake8-self
"SLOT", # flake8-slots
"SIM", # flake8-simplify
"TID", # flake8-tidy-imports
"TCH", # flake8-type-checking
"INT", # flake8-gettext
"ARG", # flake8-unused-arguments
"PTH", # flake8-use-pathlib
"ERA", # eradicate
"PD", # pandas-vet
"PGH", # pygrep-hooks
"PL", # Pylint
"TRY", # tryceratops
"FLY", # flynt
"NPY", # NumPy-specific rules
"PERF", # Perflint
"RUF", # Ruff-specific rules
]
ignore = [
# Allow non-abstract empty methods in abstract base classes
"B027",
# Allow boolean positional values in function calls
"FBT003",
# Ignore complexity (covered by other tools)
"C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
# Allow print statements (useful for debugging MCP servers)
"T201",
# Allow broad exception handling (common in async code)
"BLE001",
# Allow string formatting in logging (common pattern)
"G002",
# Allow TODO comments
"FIX002",
# Allow missing type annotations in tests
"ANN",
# Allow assert statements in tests
"S101",
# Docstring rules - be more lenient
"D100", "D101", "D102", "D103", "D104", "D105", "D106", "D107",
"D200", "D212", "D415",
]
# Allow unused variables when underscore-prefixed
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
[tool.ruff.lint.per-file-ignores]
# Tests can use more relaxed rules
"tests/**" = [
"S101", # Allow assert
"ANN", # No type annotations required
"D", # No docstrings required
"PLR2004", # Allow magic values
"S106", # Allow hardcoded passwords in tests
"S108", # Allow temp file creation
"ARG", # Allow unused function arguments (fixtures)
"PT011", # Allow broad pytest.raises
]
# Scripts and utilities can be more relaxed
"scripts/**" = ["T201", "S602", "S603", "S607"]
"knowledge_graphs/**" = ["T201", "S602", "S603", "S607"]
[tool.ruff.lint.isort]
known-first-party = ["src", "tests"]
split-on-trailing-comma = true
force-single-line = false
[tool.ruff.lint.pydocstyle]
convention = "google"
[tool.ruff.format]
quote-style = "double"
indent-style = "space"
line-ending = "auto"
# ========================================
# PYTEST CONFIGURATION
# ========================================
[tool.pytest.ini_options]
minversion = "7.0"
addopts = [
"-ra",
"--strict-markers",
"--strict-config",
"--cov=src",
"--cov-report=term-missing",
"--cov-report=html",
"--cov-report=xml",
"--cov-fail-under=80",
"--tb=short",
]
testpaths = ["tests"]
python_files = ["test_*.py", "*_test.py"]
python_classes = ["Test*"]
python_functions = ["test_*"]
markers = [
"integration: marks tests as integration tests (deselect with '-m \"not integration\"')",
"unit: marks tests as unit tests",
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
"network: marks tests that require network access",
"docker: marks tests that require Docker",
"qdrant: marks tests that require Qdrant",
"neo4j: marks tests that require Neo4j",
]
filterwarnings = [
"error",
"ignore::UserWarning",
"ignore::DeprecationWarning",
"ignore::PendingDeprecationWarning",
"ignore::ImportWarning",
]
# ========================================
# COVERAGE CONFIGURATION
# ========================================
[tool.coverage.run]
source = ["src"]
omit = [
"*/tests/*",
"*/test_*",
"*/__pycache__/*",
".venv/*",
"*/conftest.py",
]
branch = true
[tool.coverage.report]
exclude_lines = [
"pragma: no cover",
"def __repr__",
"if self.debug:",
"if settings.DEBUG",
"raise AssertionError",
"raise NotImplementedError",
"if 0:",
"if __name__ == .__main__.:",
"class .*\\bProtocol\\):",
"@(abc\\.)?abstractmethod",
]
show_missing = true
precision = 2
[tool.coverage.html]
directory = "htmlcov"
# ========================================
# MYPY CONFIGURATION (STRICT MODE)
# ========================================
[tool.mypy]
python_version = "3.12"
mypy_path = "stubs"
# Warning flags - ALL enabled
warn_return_any = true
warn_unused_configs = true
warn_redundant_casts = true
warn_unused_ignores = true
warn_no_return = true
warn_unreachable = true
strict_equality = true
# Strict type checking - NO exceptions
disallow_untyped_defs = true
disallow_incomplete_defs = true
disallow_untyped_decorators = true
disallow_untyped_calls = true
disallow_any_unimported = false # Would break pydantic-ai
check_untyped_defs = true
disallow_any_generics = true
disallow_subclassing_any = true
# Import handling - STRICT (no ignore_missing_imports)
no_implicit_optional = true
warn_incomplete_stub = true
# Strict mode for core modules
[[tool.mypy.overrides]]
module = [
"src.config.*",
"src.core.*",
"src.database.base",
"src.database.factory",
]
strict = true
# Tests can be slightly more relaxed (but still typed)
[[tool.mypy.overrides]]
module = "tests.*"
disallow_untyped_defs = false
disallow_incomplete_defs = false
disallow_untyped_decorators = false
# Exclude test/experimental scripts
[[tool.mypy.overrides]]
module = "src.knowledge_graph.test_script"
ignore_errors = true
# Files with external library API compatibility issues
[[tool.mypy.overrides]]
module = [
"src.services.crawling",
"src.services.agentic_search",
"src.services.validated_search",
"src.utils.embeddings",
"src.database.qdrant_adapter",
"src.core.context",
]
# Still check types but allow some flexibility for external APIs
disallow_untyped_calls = false
warn_return_any = false
# ========================================
# PYRIGHT CONFIGURATION
# ========================================
[tool.pyright]
pythonVersion = "3.12"
stubPath = "stubs"
# ========================================
# BANDIT CONFIGURATION
# ========================================
[tool.bandit]
exclude_dirs = ["tests", "scripts"]
skips = ["B101", "B601"] # Skip assert and shell injection in specific contexts
# ========================================
# UV CONFIGURATION FOR CPU-ONLY PYTORCH
# ========================================
[[tool.uv.index]]
name = "pytorch-cpu"
url = "https://download.pytorch.org/whl/cpu"
explicit = true
[tool.uv.sources]
torch = [
{ index = "pytorch-cpu" },
]
torchvision = [
{ index = "pytorch-cpu" },
]
torchaudio = [
{ index = "pytorch-cpu" },
]
[dependency-groups]
dev = [
"mypy>=1.18.2",
"pyright>=1.1.407",
"pytype>=2024.10.11",
"stubdefaulter>=0.1.0",
]
