name: API Test
on:
workflow_call:
inputs:
environment:
type: string
required: true
description: "where to test"
workflow_dispatch:
inputs:
environment:
type: choice
required: true
description: "where to test"
default: "tools"
options:
- tools
- production
- perf
repository_dispatch:
types:
- webhook
jobs:
define-test-matrix:
runs-on: ubuntu-latest
environment: ${{ inputs.environment }}
outputs:
tests: ${{ steps.tests.outputs.tests }}
steps:
- uses: actions/checkout@v4
- id: tests
working-directory: bin/si-sdf-api-test/tests
run: |
# Find .ts files, remove ./ prefix, and format as JSON array
files=$(find "." -mindepth 1 -maxdepth 1 -type f -name "*.ts" | sed -r "s/\.\/(.*)\.ts/\1/" | sort)
# Get the number of tests
test_count=$(echo "$files" | wc -l)
# Get the list of workspace IDs from the environment variable
workspace_ids="${{ vars.API_TEST_WORKSPACE_IDS }}"
echo "workspace_ids found to be $workspace_ids"
workspace_count=$(echo "$workspace_ids" | tr ',' '\n' | wc -l)
# Validate that the number of workspace IDs is greater than or equal to the number of tests
if [ "$workspace_count" -lt "$test_count" ]; then
echo "Error: The number of workspace IDs ($workspace_count) is less than the number of tests ($test_count)."
exit 1
fi
# Format files as JSON array with correct numbering and sorted order
indexed_files=$(echo "$files" | awk '{print "{\"name\": \"" $0 "\", \"index\": " NR-1 "}"}' | jq -s .)
# Ensure indexed_files are formatted correctly
test_output=$(echo "$indexed_files" | jq -c '.')
echo "tests=$test_output" >> "$GITHUB_OUTPUT"
echo "$test_output"
api-test:
name: API Test SDF
environment: ${{ inputs.environment }}
runs-on: ubuntu-latest
needs: define-test-matrix
strategy:
fail-fast: true
matrix:
tests: ${{ fromJSON(needs.define-test-matrix.outputs.tests) }}
env:
SDF_API_URL: ${{ vars.SDF_API_URL }}
AUTH_API_URL: ${{ vars.AUTH_API_URL }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install Deno
uses: denoland/setup-deno@v2
- name: Run the deno exec with retry
run: |
cd bin/si-sdf-api-test
echo "Running test ${{ matrix.tests.name }} with index: ${{ matrix.tests.index }}"
# Split the workspace IDs into an array
workspace_ids="${{ vars.API_TEST_WORKSPACE_IDS }}"
IFS=',' read -r -a workspace_array <<< "$workspace_ids"
# Pick the correct workspace ID based on the index
workspace_id="${workspace_array[${{ matrix.tests.index }}]}"
echo "Using workspace ID: $workspace_id"
# Retry loop with 5 attempts
n=0
max_retries=5
exit_code=0
until [ $n -ge $max_retries ]
do
unset exit_code || echo "exit_code not set"
# Run the deno task and store exit code in a variable
deno run \
--allow-all \
--no-config \
main.ts \
--workspaceId "$workspace_id" \
--userId ${{ secrets.API_TEST_EMAIL }} \
--password ${{ secrets.API_TEST_PASSWORD }} \
--tests ${{ matrix.tests.name }} || exit_code=$?
# Check the exit code
if [ -z "$exit_code" ]; then
echo "Deno task succeeded [ or the orchestration failed for a totally non-valid reason ]!"
exit 0
fi
n=$((n+1))
echo "Attempt $n/$max_retries failed with exit code $exit_code! Retrying..."
sleep 60
done
if [ $n -ge $max_retries ]; then
echo "All $max_retries attempts failed."
fi
echo "last_exit_code=$exit_code" >> "$GITHUB_ENV"
exit "$exit_code"
- name: Upload artifact if exit code 53
if: failure()
run: |
if [ "${{ env.last_exit_code }}" == "53" ]; then
echo "Uploading marker for test ${{ matrix.tests.name }}"
mkdir -p artifacts/${{ matrix.tests.name }}
echo "failure-marker" > artifacts/${{ matrix.tests.name }}/failure-marker
fi
- name: Upload artifacts
if: failure()
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.tests.name }}
path: artifacts/${{ matrix.tests.name }}
luminork-api-test:
name: API Test Luminork
environment: ${{ inputs.environment }}
runs-on: ubuntu-latest
env:
LUMINORK_API_URL: ${{ vars.LUMINORK_API_URL }}
LUMINORK_WORKSPACE_ID: ${{ vars.LUMINORK_WORKSPACE_ID }}
LUMINORK_AUTH_TOKEN: ${{ secrets.LUMINORK_AUTH_TOKEN }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install Deno
uses: denoland/setup-deno@v2
- name: Run the deno exec with retry
run: |
cd bin/si-luminork-api-tests
n=0
max_retries=5
exit_code=0
until [ $n -ge $max_retries ]
do
unset exit_code || echo "exit_code not set"
# Run the luminork tests and store exit code in a variable
./scripts/run-tests.sh || exit_code=$?
# Check the exit code
if [ -z "$exit_code" ]; then
echo "Deno task succeeded [ or the orchestration failed for a totally non-valid reason ]!"
exit 0
fi
n=$((n+1))
echo "Attempt $n/$max_retries failed with exit code $exit_code! Retrying..."
# If we failed, back off and retry!
sleep_time=$((6 * n))
echo "Waiting ${sleep_time} seconds before retry..."
sleep $sleep_time
done
if [ $n -ge $max_retries ]; then
echo "All $max_retries attempts failed."
echo "Luminork API tests failed"
cd ../..
mkdir -p artifacts/luminork_tests
echo "failure-marker" > artifacts/luminork_tests/failure-marker
fi
echo "last_exit_code=$exit_code" >> "$GITHUB_ENV"
exit "$exit_code"
- name: Upload artifact if exit code 53
if: failure()
run: |
if [ "${{ env.last_exit_code }}" == "53" ]; then
echo "Uploading marker for test luminork_tests"
mkdir -p artifacts/luminork_tests
echo "failure-marker" > artifacts/luminork_tests/failure-marker
fi
- name: Upload artifacts
if: failure()
uses: actions/upload-artifact@v4
with:
name: luminork_tests
path: artifacts/luminork_tests
on-failure:
runs-on: ubuntu-latest
needs:
- api-test
- luminork-api-test
environment: ${{ inputs.environment }}
if: failure()
steps:
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
path: artifacts
- run: |
has_artifacts=false
# Check for marker files
for marker in artifacts/*/failure-marker; do
if [ -f "$marker" ]; then
echo "Artifact detected for failed test: $marker"
echo "Setting failure to true and breaking"
has_artifacts=true
break
fi
done
# If at least one valid failure marker is present, then page
if [ "$has_artifacts" = true ] && [ "${{ github.ref_name }}" = "main" ]; then
curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \
--header "Content-Type: application/json" \
--data "{
\"summary\": \"API ${{ inputs.environment }} Tests Fail\",
\"body\": \"API Tests have failed for ${{ inputs.environment }}.\",
\"links\": [
{
\"href\": \"https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID\",
\"text\": \"E2E Test Run ${{ inputs.environment }}\"
}
],
\"tags\": [
\"service:github\"
]
}"
fi
- run: |
# Always send the Internal Slack Notification if failure detected, regardless of error source
curl --location "${{ secrets.SLACK_WEBHOOK_URL }}" -X POST \
--header 'Content-type: application/json' \
--data "{\"text\": \":si: Failed API Tests for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\"}"