#!/usr/bin/env bash
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
# Model conformance test runner for Python plugins.
#
# Runs `genkit dev:test-model` against the specified plugin's conformance
# spec and entry point. Each plugin has its own spec file and lightweight
# Python entry point under py/tests/conformance/<plugin>/.
#
# Usage:
# py/bin/test-model-conformance google-genai # test one plugin
# py/bin/test-model-conformance anthropic # test another
# py/bin/test-model-conformance --all # test all plugins
#
# Required environment variables per plugin:
# google-genai: GEMINI_API_KEY
# anthropic: ANTHROPIC_API_KEY
# compat-oai: OPENAI_API_KEY
# mistral: MISTRAL_API_KEY
# deepseek: DEEPSEEK_API_KEY
# xai: XAI_API_KEY
# amazon-bedrock: AWS_REGION (+ AWS credentials)
# huggingface: HF_TOKEN
# microsoft-foundry: AZURE_OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT
# cloudflare-workers-ai: CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_API_TOKEN
# ollama: (none, local server)
set -euo pipefail
shopt -s nullglob
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
CONFORMANCE_DIR="${REPO_ROOT}/py/tests/conformance"
# Map plugin names to required environment variables.
# When a plugin requires multiple env vars, they are space-separated.
declare -A PLUGIN_ENV_VARS=(
[google-genai]="GEMINI_API_KEY"
[anthropic]="ANTHROPIC_API_KEY"
[compat-oai]="OPENAI_API_KEY"
[mistral]="MISTRAL_API_KEY"
[deepseek]="DEEPSEEK_API_KEY"
[xai]="XAI_API_KEY"
[amazon-bedrock]="AWS_REGION"
[huggingface]="HF_TOKEN"
[microsoft-foundry]="AZURE_OPENAI_API_KEY AZURE_OPENAI_ENDPOINT"
[cloudflare-workers-ai]="CLOUDFLARE_ACCOUNT_ID CLOUDFLARE_API_TOKEN"
)
usage() {
echo "Usage: $0 <plugin-name> | --all" >&2
echo "" >&2
echo "Available plugins:" >&2
for plugin_dir in "${CONFORMANCE_DIR}"/*/; do
plugin=$(basename "${plugin_dir}")
echo " ${plugin}" >&2
done
exit 1
}
check_env() {
local plugin="$1"
local env_vars="${PLUGIN_ENV_VARS[${plugin}]:-}"
local missing=0
for env_var in ${env_vars}; do
if [[ -z "${!env_var:-}" ]]; then
echo "ERROR: ${env_var} is not set (required for ${plugin})" >&2
missing=1
fi
done
return ${missing}
}
run_plugin() {
local plugin="$1"
local spec_file="${CONFORMANCE_DIR}/${plugin}/model-conformance.yaml"
local entry_point="${CONFORMANCE_DIR}/${plugin}/conformance_entry.py"
if [[ ! -f "${spec_file}" ]]; then
echo "ERROR: Spec file not found: ${spec_file}" >&2
return 1
fi
if [[ ! -f "${entry_point}" ]]; then
echo "ERROR: Entry point not found: ${entry_point}" >&2
return 1
fi
if ! check_env "${plugin}"; then
return 1
fi
echo "=== Testing ${plugin} ==="
echo " Spec: ${spec_file}"
echo " Entry: ${entry_point}"
echo ""
genkit dev:test-model \
--from-file "${spec_file}" \
-- uv run --project "${REPO_ROOT}/py" --active "${entry_point}"
}
if [[ $# -eq 0 ]]; then
usage
fi
failed=0
if [[ "$1" == "--all" ]]; then
for plugin_dir in "${CONFORMANCE_DIR}"/*/; do
plugin=$(basename "${plugin_dir}")
if ! run_plugin "${plugin}"; then
failed=1
echo "FAIL: ${plugin}"
else
echo "PASS: ${plugin}"
fi
echo ""
done
else
plugin="$1"
if [[ ! -d "${CONFORMANCE_DIR}/${plugin}" ]]; then
echo "ERROR: Unknown plugin '${plugin}'" >&2
usage
fi
if ! run_plugin "${plugin}"; then
failed=1
fi
fi
if [[ ${failed} -ne 0 ]]; then
echo "Some conformance tests failed."
exit 1
fi
echo "All conformance tests passed."
