name: Manual E2E Tests
on:
workflow_dispatch:
inputs:
models:
description: "Models to test"
required: false
default: "gpt-4.1-mini,gpt-4o-mini"
type: string
permissions:
contents: read
actions: write
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1
with:
egress-policy: audit
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
- name: Validate inputs
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
const models = '${{ inputs.models }}';
console.log('Validating models input:', models);
if (!/^[a-zA-Z0-9,.\-_]+$/.test(models)) {
core.setFailed('Models input contains invalid characters. Only alphanumeric, hyphens, commas, periods, and underscores are allowed.');
return;
}
console.log('Input validation passed');
- name: Set up Python 3.12
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
with:
python-version: "3.12"
- name: Install the latest version of uv
uses: astral-sh/setup-uv@39eb6c9dde236bbc368681611e63120a6eb4afac
with:
version: "0.7.13"
activate-environment: true
- name: Install dependencies
run: |
uv sync --extra dev
- name: Test with pytest
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
MODELS_TO_TEST: ${{ inputs.models }}
MCP_USE_ANONYMIZED_TELEMETRY: false
run: |
pytest --run-e2e
- name: Generate HTML report
if: always()
run: |
python scripts/generate_e2e_report.py
- name: Upload HTML report artifact
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
if: always()
with:
name: e2e-test-report
path: ./static_test_report.html
