DollhouseMCP

---
name: QA Tests

on:
  pull_request:
    branches: [develop]
  workflow_dispatch:

permissions:
  contents: read

env:
  NODE_OPTIONS: '--max-old-space-size=4096 --experimental-vm-modules'
  CI: true
  TEST_PERSONAS_DIR: ${{ github.workspace }}/test-personas
  TEST_GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }}

jobs:
  qa-tests:
    name: QA Automated Tests
    runs-on: ubuntu-latest
    timeout-minutes: 10
    continue-on-error: true

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - name: Setup Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: '20.x'
          cache: 'npm'
          cache-dependency-path: package-lock.json

      - name: Validate QA environment
        shell: bash
        run: |
          echo "🔍 Validating QA test environment..."
          echo "TEST_PERSONAS_DIR: $TEST_PERSONAS_DIR"
          echo "NODE_OPTIONS: $NODE_OPTIONS"
          echo "CI: $CI"
          
          # Verify TEST_PERSONAS_DIR is set
          if [ -z "$TEST_PERSONAS_DIR" ]; then
            echo "❌ TEST_PERSONAS_DIR is not set!"
            exit 1
          fi
          
          # Create test personas directory if it doesn't exist
          mkdir -p "$TEST_PERSONAS_DIR"
          echo "✅ Test personas directory ready: $TEST_PERSONAS_DIR"
          
          # Display workspace info for debugging
          echo "GitHub workspace: ${{ github.workspace }}"
          echo "Current directory: $(pwd)"
          echo "✅ QA environment validation passed"

      - name: Install dependencies
        run: npm ci

      - name: Build project
        run: npm run build

      - name: Verify QA scripts exist
        shell: bash
        run: |
          echo "🔍 Verifying QA test scripts..."
          if [ -f "scripts/qa-test-runner.js" ]; then
            echo "✅ QA test runner found"
          else
            echo "❌ QA test runner not found!"
            exit 1
          fi
          
          if [ -f "scripts/qa-utils.js" ]; then
            echo "✅ QA utils found"
          else
            echo "❌ QA utils not found!"
            exit 1
          fi
          
          # Ensure docs/QA directory exists for results
          mkdir -p docs/QA
          echo "✅ QA results directory ready"

      - name: Run QA Test Suite
        id: qa_tests
        shell: bash
        run: |
          echo "🚀 Starting QA test suite..."
          
          # Run BOTH test approaches for comprehensive validation:
          
          # 1. Direct SDK tests (internal validation - fast and reliable)
          echo "📌 Running Direct SDK Tests (internal validation)..."
          timeout 270 node scripts/qa-direct-test.js || {
            exit_code=$?
            if [ $exit_code -eq 124 ]; then
              echo "⏰ Direct SDK tests timed out after 4.5 minutes"
              echo "direct_status=timeout" >> $GITHUB_OUTPUT
            else
              echo "❌ Direct SDK tests failed with exit code: $exit_code"
              echo "direct_status=failed" >> $GITHUB_OUTPUT
            fi
            # Don't exit yet - run Inspector tests too
          }
          
          # 2. Inspector CLI tests (CRITICAL: external client validation for protocol compliance)
          echo ""
          echo "📌 Running Inspector CLI Tests (EXTERNAL validation - protocol compliance)..."
          timeout 270 npx @modelcontextprotocol/inspector --version && \
          node scripts/qa-inspector-cli-test.js || {
            exit_code=$?
            if [ $exit_code -eq 124 ]; then
              echo "⏰ Inspector CLI tests timed out after 4.5 minutes"
              echo "inspector_status=timeout" >> $GITHUB_OUTPUT
            else
              echo "❌ Inspector CLI tests failed with exit code: $exit_code"
              echo "inspector_status=failed" >> $GITHUB_OUTPUT
            fi
            # Don't fail the job yet - we want to see both results
          }
          
          echo "✅ QA test suite completed"
          echo "qa_status=success" >> $GITHUB_OUTPUT

      - name: Verify test results
        shell: bash
        run: |
          echo "📊 Checking QA test results..."
          
          # Check if any result files were created (qa-direct-test creates different filename pattern)
          if ls docs/QA/qa-*-test-results-*.json 1> /dev/null 2>&1; then
            echo "✅ QA test results found:"
            ls -la docs/QA/qa-*-test-results-*.json
            
            # Show summary of latest results
            latest_result=$(ls -t docs/QA/qa-*-test-results-*.json | head -1)
            echo "📋 Latest result summary from: $latest_result"
            
            # Extract key metrics using jq if available, otherwise use grep
            if command -v jq >/dev/null 2>&1; then
              echo "Total Tests: $(jq -r '.summary.total_tests' "$latest_result")"
              echo "Success Rate: $(jq -r '.summary.success_rate' "$latest_result")"
              echo "Duration: $(jq -r '.duration' "$latest_result")"
            else
              echo "Result file created (jq not available for detailed parsing)"
              grep -E '"total_tests"|"success_rate"|"duration"' "$latest_result" || true
            fi
          else
            echo "⚠️ No QA test result files found"
            echo "This may indicate the tests didn't complete or encountered early failure"
          fi

      - name: Upload QA test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: qa-test-results-${{ github.run_number }}
          path: docs/QA/qa-*-test-results-*.json
          retention-days: 30
          if-no-files-found: warn

      - name: Upload QA test logs
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: qa-test-logs-${{ github.run_number }}
          path: |
            docs/QA/*.md
            docs/QA/agent-reports/*.json
          retention-days: 7
          if-no-files-found: ignore

      - name: Comment QA results summary
        if: always() && github.event_name == 'pull_request'
        shell: bash
        run: |
          echo "📝 Preparing QA results summary for PR comment..."
          
          # Create a summary comment for the PR
          if ls docs/QA/qa-test-results-*.json 1> /dev/null 2>&1; then
            latest_result=$(ls -t docs/QA/qa-test-results-*.json | head -1)
            
            echo "## 🧪 QA Test Results" > qa_summary.md
            echo "" >> qa_summary.md
            echo "**Status:** ${{ steps.qa_tests.outputs.qa_status || 'unknown' }}" >> qa_summary.md
            echo "" >> qa_summary.md
            
            if command -v jq >/dev/null 2>&1; then
              echo "**📊 Test Metrics:**" >> qa_summary.md
              echo "- Total Tests: $(jq -r '.summary.total_tests' "$latest_result")" >> qa_summary.md
              echo "- Executed Tests: $(jq -r '.summary.executed_tests' "$latest_result")" >> qa_summary.md
              echo "- Skipped Tests: $(jq -r '.summary.skipped_tests' "$latest_result")" >> qa_summary.md
              echo "- Success Rate: $(jq -r '.summary.success_rate' "$latest_result")" >> qa_summary.md
              echo "- Duration: $(jq -r '.duration' "$latest_result")" >> qa_summary.md
              echo "- Available Tools: $(jq -r '.tool_discovery.available_tools_count' "$latest_result")" >> qa_summary.md
            else
              echo "- Result file created: ✅" >> qa_summary.md
              echo "- Detailed metrics available in artifacts" >> qa_summary.md
            fi
            
            echo "" >> qa_summary.md
            echo "*📎 Detailed results available in workflow artifacts*" >> qa_summary.md
            echo "" >> qa_summary.md
            echo "*🔧 This is a non-blocking test suite in development phase*" >> qa_summary.md
            
            cat qa_summary.md
          else
            echo "⚠️ No QA results to summarize"
          fi

      - name: QA Tests Complete
        shell: bash
        run: |
          echo "🎉 QA Tests workflow completed!"
          echo "Status: ${{ steps.qa_tests.outputs.qa_status || 'unknown' }}"
          echo ""
          echo "ℹ️  This workflow is currently non-blocking (continue-on-error: true)"
          echo "   Future releases may make these tests blocking for merge"
          echo ""
          echo "📋 Next steps:"
          echo "   - Review QA test results in artifacts"
          echo "   - Address any failures found"
          echo "   - Monitor for consistency across PRs"