model:
provider: openai
name: gpt-4o-mini # Faster and more cost-effective for CI
evals:
# Minimal CI-focused evaluations
# These test tool availability and basic functionality without requiring live WordPress sites
- name: tool_availability_check
description: Check that WordPress tools are available
prompt: "What WordPress tools are available for managing posts?"
expected_tools:
- wp_list_posts
- wp_get_post
- wp_create_post
success_criteria:
- Tools are listed and available
- No connection errors occur
- Appropriate error messages for missing credentials
- name: multi_site_error_handling
description: Multi-site configuration error handling
prompt: "List posts without specifying a site"
expected_tools:
- wp_list_posts
success_criteria:
- Appropriate error message about missing site parameter
- Available sites are listed if multiple configured
- No crashes or unexpected errors
- name: configuration_validation
description: Configuration validation in CI environment
prompt: "Get current user information"
expected_tools:
- wp_get_current_user
success_criteria:
- Graceful handling of missing credentials
- Clear error messages about configuration
- No server crashes or unexpected failures