name: PR Bots
on:
pull_request_target:
types: [opened, synchronize, labeled]
concurrency:
group: pr-bots-${{ github.event.pull_request.number }}
cancel-in-progress: ${{ github.event.action == 'synchronize' }}
jobs:
size-label:
name: Size Label
if: github.event.action != 'labeled'
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
pull-requests: write
steps:
- name: Calculate PR size and set label
run: |
# Fetch file changes for this PR
FILES=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files --paginate | jq -s 'add')
# Calculate lines by category (excluding uv.lock and cassettes)
CODE=$(echo "$FILES" | jq '[.[] | select(
(.filename | (startswith("tests/") or startswith("docs/") or endswith(".md")) | not) and
(.filename != "uv.lock") and
(.filename | contains("/cassettes/") | not)
) | .additions + .deletions] | add // 0')
DOCS=$(echo "$FILES" | jq '[.[] | select(
(.filename | (startswith("docs/") or endswith(".md"))) and
(.filename != "uv.lock") and
(.filename | contains("/cassettes/") | not)
) | .additions + .deletions] | add // 0')
TESTS=$(echo "$FILES" | jq '[.[] | select(
(.filename | startswith("tests/")) and
(.filename | contains("/cassettes/") | not) and
(.filename | endswith(".md") | not)
) | .additions + .deletions] | add // 0')
# Calculate weighted score: code + 50% docs + 50% tests
SCORE=$((CODE + DOCS / 2 + TESTS / 2))
echo "Code: $CODE, Docs: $DOCS, Tests: $TESTS"
echo "Weighted score: $SCORE"
# Determine size label based on cutoffs
if [ $SCORE -le 100 ]; then
SIZE="size: S"
elif [ $SCORE -le 500 ]; then
SIZE="size: M"
elif [ $SCORE -le 1500 ]; then
SIZE="size: L"
else
SIZE="size: XL"
fi
echo "Size: $SIZE"
# Remove any existing size labels (except the one we're setting) via API
for label in "size: S" "size: M" "size: L" "size: XL"; do
if [ "$label" != "$SIZE" ]; then
gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels/${label}" --method DELETE 2>/dev/null || true
fi
done
# Add the new size label via API
gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels" --method POST -f "labels[]=$SIZE"
echo "Set label: $SIZE (score: $SCORE)"
env:
GH_TOKEN: ${{ github.token }}
category-label:
name: Category Label
if: github.event.action != 'labeled'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
pull-requests: write
steps:
- name: Check for modified config files
if: github.event.pull_request.head.repo.fork
run: |
CHANGED=$(gh pr diff ${{ github.event.pull_request.number }} --name-only --repo ${{ github.repository }})
if echo "$CHANGED" | grep -qiE '(^|/)AGENTS\.md$|(^|/)CLAUDE\.md$|(^|/)\.claude/'; then
echo "::error::PR modifies agent config files (AGENTS.md, CLAUDE.md, or .claude/). Skipping auto-labeling for security."
exit 1
fi
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Check if category label already exists
id: check-label
run: |
LABELS=$(gh pr view ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --json labels --jq '.labels[].name')
CATEGORY_LABELS=("bug" "feature" "docs" "chore" "dependency")
for label in "${CATEGORY_LABELS[@]}"; do
if echo "$LABELS" | grep -q "^${label}$"; then
echo "has_label=true" >> $GITHUB_OUTPUT
echo "PR already has category label: $label"
exit 0
fi
done
echo "has_label=false" >> $GITHUB_OUTPUT
env:
GH_TOKEN: ${{ github.token }}
- name: Determine category label with Claude Code
if: steps.check-label.outputs.has_label == 'false'
uses: anthropics/claude-code-action@v1
env:
ANTHROPIC_BASE_URL: ${{ secrets.CLAUDE_CODE_BASE_URL }}
with:
anthropic_api_key: ${{ secrets.CLAUDE_CODE_API_KEY || secrets.ANTHROPIC_API_KEY }}
github_token: ${{ secrets.GITHUB_TOKEN }}
allowed_non_write_users: '*'
claude_args: |
--allowedTools "Bash(gh pr view:*),Bash(gh pr diff:*),Bash(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels:*)"
prompt: |
Classify PR #${{ github.event.pull_request.number }} in ${{ github.repository }} and add exactly one label.
Run `gh pr view ${{ github.event.pull_request.number }} --repo ${{ github.repository }}` for the title and description.
Run `gh pr diff ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --name-only` for the changed files.
Categories:
- `bug`: Fixes broken behavior
- `feature`: New functionality
- `docs`: Documentation-only (no code changes)
- `chore`: CI, refactoring, dev dependencies, tests-only
- `dependency`: Production dependency updates
When both code and docs change, prefer `feature` or `bug` over `docs`.
If pyproject.toml files changed, check their diff to distinguish production deps (`dependency`) from dev-only deps in `[dependency-groups]` (`chore`).
Add the label:
gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels --method POST -f "labels[]=<category>"
review:
name: Review
needs: [size-label, category-label]
if: >-
!failure() && !cancelled() &&
github.event.action == 'labeled' && github.event.label.name == 'auto-review'
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
issues: write
pull-requests: write
actions: read
steps:
- uses: actions/checkout@v6
with:
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}
fetch-depth: 0
- name: Check for modified config files
if: github.event.pull_request.head.repo.fork
run: |
CHANGED=$(gh pr diff ${{ github.event.pull_request.number }} --name-only --repo ${{ github.repository }})
if echo "$CHANGED" | grep -qiE '(^|/)AGENTS\.md$|(^|/)CLAUDE\.md$|(^|/)\.claude/'; then
echo "::error::PR modifies agent config files (AGENTS.md, CLAUDE.md, or .claude/). Skipping auto-review for security."
exit 1
fi
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Select review model
id: model
run: |
LABELS=$(gh pr view ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --json labels --jq '.labels[].name')
echo "Labels: $LABELS"
CATEGORY=$(echo "$LABELS" | grep -E '^(bug|feature|docs|chore|dependency)$' | head -1)
SIZE=$(echo "$LABELS" | grep -E '^size: ' | head -1)
# Default to Opus for large/complex PRs
MODEL="claude-opus-4-6"
# Disabled for now because Sonnet reviews have been disappointing
# # Use Sonnet for docs, dependency, chore, or small/medium PRs
# if [ "$CATEGORY" = "docs" ] || [ "$CATEGORY" = "dependency" ] || [ "$CATEGORY" = "chore" ] || [ "$SIZE" = "size: S" ] || [ "$SIZE" = "size: M" ]; then
# MODEL="claude-sonnet-4-5"
# fi
echo "model=$MODEL" >> $GITHUB_OUTPUT
echo "Selected model: $MODEL (category: $CATEGORY, size: $SIZE)"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Gather PR context
run: |
# Use the script from the base repo (main branch), not the fork
gh api repos/${{ github.repository }}/contents/scripts/gather-review-context.sh?ref=${{ github.event.pull_request.base.ref }} --jq .content | base64 -d > /tmp/gather-review-context.sh
bash /tmp/gather-review-context.sh ${{ github.event.pull_request.number }} ${{ github.repository }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: anthropics/claude-code-action@v1
env:
ANTHROPIC_BASE_URL: ${{ secrets.CLAUDE_CODE_BASE_URL }}
with:
anthropic_api_key: ${{ secrets.CLAUDE_CODE_API_KEY || secrets.ANTHROPIC_API_KEY }}
github_token: ${{ secrets.GITHUB_TOKEN }}
allowed_non_write_users: '*'
additional_permissions: |
actions: read
claude_args: |
--model ${{ steps.model.outputs.model }}
--allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr checks:*),Bash(gh pr list:*),Bash(gh issue view:*),Bash(gh issue list:*),Bash(gh run view:*),Bash(gh run list:*),Bash(gh api:*),Bash(git log:*),Bash(git diff:*),Bash(git grep:*),Bash(git show:*),Bash(git status:*),Bash(jq:*),Bash(cat:*),Bash(rg:*),Bash(ls:*),Bash(tree:*),Bash(grep:*),WebSearch,WebFetch"
prompt: |
REPO: ${{ github.repository }}
PR NUMBER: ${{ github.event.pull_request.number }}
PR AUTHOR: ${{ github.event.pull_request.user.login }} (${{ github.event.pull_request.author_association }})
Review this pull request. The PR branch is already checked out in the current working directory and the `CLAUDE.md` at the root (symlinked to `AGENTS.md`) is already loaded in your system prompt.
# What to look for
- If the PR should not have been created yet (e.g. no issue, insufficiently defined scope, not ready for implementation, duplicate of existing open PR), just leave a comment informing the user and maintainer of this and don't bother doing a thorough review.
- Any change that does not align with the project's standards, philosophy, or requirements for every contribution as stated in `AGENTS.md` (symlinked from `CLAUDE.md`).
- Any change that does not match maintainer guidance in the issue or earlier PR comments on what an acceptable solution would look like.
- Any change or design decision or tradeoff (in both behavior and API) that needs explicit consideration, discussion, or maintainer awareness and approval.
- Any line of code that violates the concrete guidelines/rules laid out in the relevant `AGENTS.md` file(s): the top-level guidelines apply to all changes, while directory-specific guidelines affect only the changes in that directory.
- Anything else that the responsibilities you are assigned in `AGENTS.md` suggest that you should be calling out: use your best judgment.
Generally, the priority in terms of "crucial to get right" and "what to focus on first in a new PR" is public API > concepts and behavior > documentation > tests > code style.
If the PR has high level problems that will likely require significant changes at lower levels, hold off on looking for or commenting on lower level problems until the higher level problems are addressed, so that the PR author (and your context window) don't get overwhelmed.
Note that while another agent (Devin) is responsible for thoroughly reviewing the implementation for bugs, security issues, and edge cases,
_you_ are responsible for catching every violation of the repository's standards and guidelines listed in the `AGENTS.md` files.
Do not focus exclusively on high-level concerns: by the time the author has addressed every comment you've left over multiple rounds of review, the PR should be ready to merge.
# Gathering context
Before doing anything else, read ALL of the following pre-gathered context files in a single parallel tool call:
- `.github/.review-context/pr-details.json` — PR title, body, author, branch info, labels, state, review decision, timestamps
- `.github/.review-context/pr-comments.txt` — existing top-level PR comments
- `.github/.review-context/review-comments.txt` — existing inline review comments (resolved+outdated threads and threads predating the last auto-review are collapsed to one-liners with comment IDs so you can fetch full details if needed)
- `.github/.review-context/related-issues.txt` — linked issues and their comments
- `.github/.review-context/changed-files.txt` — changed files with per-file addition/deletion counts (tab-separated; non-generated files include a third column with the path to their per-file diff)
- `.github/.review-context/agents-md.txt` — directory-specific `AGENTS.md` files for changed directories
The diff is split into per-file diffs under `.github/.review-context/diff/` (excluding `uv.lock` and cassettes), that you can read on demand and in parallel.
The diffs include function-level context (`git diff -W`), so you can see the full function/method being modified without needing to read the source file separately.
Each commentable line in the diff is prefixed with its source line number: `NL:<number>` for new or context lines, `OL:<number>` for deleted lines.
For newly added files, the diff contains the complete file contents — do not re-read these from disk.
The diff file paths are listed in the third column of `changed-files.txt`.
The pre-gathered diffs are the source of truth for what this PR changes. Do not re-fetch diffs or file lists using `gh pr diff` or `gh api`.
When you need code context beyond what the diffs provide, use the `Read` tool on the checked-out source files.
Use the `gh` CLI only when you need additional information not already in these files (e.g. to read other referenced PRs or issues, check CI status, or read files excluded from the gathered diff).
When using `gh api`, prefer using the built-in `--jq` and `--template` flags for filtering/formatting since `python3` is not allowed.
Be careful about loading large diffs if you're unlikely to need them yet, like massive test files when there's plenty of more interesting code to comment on first, as you don't want to blow your context window too early.
You will usually want to read all the "core implementation" and docs diffs in one go, though, so you have the full context of the PR as you identify problems, instead of going file by file.
# Posting comments
While gathering context and learning about the PR, keep track of problems/points of discussion as you find them, and wait to post comments until the end,
as the comments you write will be better, less duplicative and more focused on the changes that really matter if you have the full set of problems as context.
For each identified issue that is determined to be worth a new comment, use `mcp__github_inline_comment__create_inline_comment` to attach the feedback to a specific line of code.
- Only lines with an `NL:` or `OL:` prefix in the diff are commentable. For OL lines, use `side: LEFT`.
- Include the reasoning, but don't quote specific rules from the `AGENTS.md` files.
- Include a concrete suggestion if appropriate (but to not use ` ```suggestion ` blocks as they can render incorrectly when the line numbers are off)
- Include a ping to the maintainer (`@DouweM`) on any change that requires maintainer input before the PR author can move forward.
- If the same issue shows up in multiple places, post a comment on each instance but have later comments refer to the first comment using a link.
- Use `gh pr comment` only for important feedback that doesn't relate to a specific line or file, not for a summary of feedback you've already posted inline.
Your comments should be:
- actionable: they should request a change, flag a concern that needs discussion, and/or suggest an improvement; don't comment on positive aspects of the PR like "excellent design choices".
- concise and to the point: don't use unnecessary emojis, lists, or subheadings, but do link to code if appropriate; 1 to 3 paragraphs are pretty much always enough.
- friendly without being sycophantic: use the tone and language of a helpful and encouraging project maintainer, but no need to compliment the author on positive aspects of the PR or point out changes that are good.
- non-repetitive: don't repeat things pointed out in earlier review comments, unless it looks like they'll be forgotten if you don't point them out; e.g. when they're marked as resolved/outdated but the problem persists without a satisfactory resolution (like a maintainer comment saying the comment does not need to be addressed).
You are meant to be helpful to the contributor and the maintainer, so your comments should never add noise to the conversation:
- Do not post a final summary comment; inline comments are sufficient.
- Do not comment on lines that do not need improvement, maintainer awareness, or discussion; comments pointing out a good choice are just noise.
- Do not post multiple comments for the same exact issue unless it shows up in different places.
It bears repeating that you are the first line of defense against low-quality contributions and maintainer headaches, and you have a big role in ensuring that every contribution to this project meets or exceeds the high standards that the Pydantic brand is known and loved for.
- name: Remove auto-review label
if: always()
run: gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels/auto-review" --method DELETE 2>/dev/null || true
env:
GH_TOKEN: ${{ github.token }}
