benchmark.yml•7.29 kB
name: Performance Benchmarks
on:
push:
branches: [main, feat/comprehensive-testing-suite]
paths-ignore:
- '**.md'
- '**.txt'
- 'docs/**'
- 'examples/**'
- '.github/FUNDING.yml'
- '.github/ISSUE_TEMPLATE/**'
- '.github/pull_request_template.md'
- '.gitignore'
- 'LICENSE*'
- 'ATTRIBUTION.md'
- 'SECURITY.md'
- 'CODE_OF_CONDUCT.md'
pull_request:
branches: [main]
paths-ignore:
- '**.md'
- '**.txt'
- 'docs/**'
- 'examples/**'
- '.github/FUNDING.yml'
- '.github/ISSUE_TEMPLATE/**'
- '.github/pull_request_template.md'
- '.gitignore'
- 'LICENSE*'
- 'ATTRIBUTION.md'
- 'SECURITY.md'
- 'CODE_OF_CONDUCT.md'
workflow_dispatch:
permissions:
# For PR comments
pull-requests: write
# For pushing to gh-pages branch
contents: write
# For deployment to GitHub Pages
pages: write
id-token: write
jobs:
benchmark:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
# Fetch all history for proper benchmark comparison
fetch-depth: 0
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- name: Install dependencies
run: npm ci
- name: Build project
run: npm run build
- name: Run benchmarks
run: npm run benchmark:ci
- name: Format benchmark results
run: node scripts/format-benchmark-results.js
- name: Upload benchmark artifacts
uses: actions/upload-artifact@v4
with:
name: benchmark-results
path: |
benchmark-results.json
benchmark-results-formatted.json
benchmark-summary.json
# Ensure gh-pages branch exists
- name: Check and create gh-pages branch
run: |
git fetch origin gh-pages:gh-pages 2>/dev/null || {
echo "gh-pages branch doesn't exist. Creating it..."
git checkout --orphan gh-pages
git rm -rf .
echo "# Benchmark Results" > README.md
git add README.md
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git commit -m "Initial gh-pages commit"
git push origin gh-pages
git checkout ${{ github.ref_name }}
}
# Clean up workspace before benchmark action
- name: Clean workspace
run: |
git add -A
git stash || true
# Store benchmark results and compare
- name: Store benchmark result
uses: benchmark-action/github-action-benchmark@v1
continue-on-error: true
id: benchmark
with:
name: n8n-mcp Benchmarks
tool: 'customSmallerIsBetter'
output-file-path: benchmark-results-formatted.json
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
# Where to store benchmark data
benchmark-data-dir-path: 'benchmarks'
# Alert when performance regresses by 10%
alert-threshold: '110%'
# Comment on PR when regression is detected
comment-on-alert: true
alert-comment-cc-users: '@czlonkowski'
# Summary always
summary-always: true
# Max number of data points to retain
max-items-in-chart: 50
fail-on-alert: false
# Comment on PR with benchmark results
- name: Comment PR with results
uses: actions/github-script@v7
if: github.event_name == 'pull_request'
continue-on-error: true
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
try {
const fs = require('fs');
const summary = JSON.parse(fs.readFileSync('benchmark-summary.json', 'utf8'));
// Format results for PR comment
let comment = '## 📊 Performance Benchmark Results\n\n';
comment += `🕐 Run at: ${new Date(summary.timestamp).toLocaleString()}\n\n`;
comment += '| Benchmark | Time | Ops/sec | Range |\n';
comment += '|-----------|------|---------|-------|\n';
// Group benchmarks by category
const categories = {};
for (const benchmark of summary.benchmarks) {
const [category, ...nameParts] = benchmark.name.split(' - ');
if (!categories[category]) categories[category] = [];
categories[category].push({
...benchmark,
shortName: nameParts.join(' - ')
});
}
// Display by category
for (const [category, benchmarks] of Object.entries(categories)) {
comment += `\n### ${category}\n`;
for (const benchmark of benchmarks) {
comment += `| ${benchmark.shortName} | ${benchmark.time} | ${benchmark.opsPerSec} | ${benchmark.range} |\n`;
}
}
// Add comparison link
comment += '\n\n📈 [View historical benchmark trends](https://czlonkowski.github.io/n8n-mcp/benchmarks/)\n';
comment += '\n⚡ Performance regressions >10% will be flagged automatically.\n';
await github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: comment
});
} catch (error) {
console.error('Failed to create PR comment:', error.message);
console.log('This is likely due to insufficient permissions for external PRs.');
console.log('Benchmark results have been saved to artifacts instead.');
}
# Deploy benchmark results to GitHub Pages
deploy:
needs: benchmark
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: gh-pages
continue-on-error: true
# If gh-pages checkout failed, create a minimal structure
- name: Ensure gh-pages content exists
run: |
if [ ! -f "index.html" ]; then
echo "Creating minimal gh-pages structure..."
mkdir -p benchmarks
echo '<!DOCTYPE html><html><head><title>n8n-mcp Benchmarks</title></head><body><h1>n8n-mcp Benchmarks</h1><p>Benchmark data will appear here after the first run.</p></body></html>' > index.html
fi
- name: Setup Pages
uses: actions/configure-pages@v4
- name: Upload Pages artifact
uses: actions/upload-pages-artifact@v3
with:
path: '.'
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4