hello-theme/.github/workflows/daily-test-matrix.yml

# 🧪 Hello Theme Daily Test Matrix
# 
# This workflow runs daily automated testing across different version combinations:
# - Core Matrix: Hello Theme × Elementor versions  
# - Plus Matrix: Hello Theme × Hello Plus versions
# - By default tests main branches (cutting-edge compatibility)
# - Can be manually triggered with specific versions for release testing
#
# Each matrix workflow handles its own version calculation and testing logic.
#
# 📊 Targeted Matrix Generation:
#
# Hello Theme × Elementor:
# - Hello Theme (main) × Elementor (main, previous GA minor.x, previous-previous GA minor.x)
# - Hello Theme (ga, if exists) × Elementor (main, previous GA minor.x, previous-previous GA minor.x)
#
# Hello Theme × Hello Plus:
# - Hello Theme (main) × Hello Plus (latest, previous patch - WordPress.org releases only)
# - Hello Theme (ga, if exists) × Hello Plus (latest, previous patch - WordPress.org releases only)
#
# Example Output (HT GA=3.4.4, EL latest=3.30.4, HP latest=1.7.2):
# [
#   {"combination":"ht-main-el-main","name":"Hello Theme main + Elementor main","hello_theme_version":"main","elementor_version":"main","delay":0},
#   {"combination":"ht-main-el-prev","name":"Hello Theme main + Elementor 3.31.1 (GA)","hello_theme_version":"main","elementor_version":"3.31.1","delay":15},
#   {"combination":"ht-main-el-prev","name":"Hello Theme main + Elementor 3.30.4 (GA)","hello_theme_version":"main","elementor_version":"3.30.4","delay":30},
#   {"combination":"ht-ga-el-main","name":"Hello Theme 3.4.4 (GA) + Elementor main","hello_theme_version":"3.4.4","elementor_version":"main","delay":60},
#   {"combination":"ht-ga-el-prev","name":"Hello Theme 3.4.4 (GA) + Elementor 3.31.1 (GA)","hello_theme_version":"3.4.4","elementor_version":"3.31.1","delay":75},
#   {"combination":"ht-ga-el-prev","name":"Hello Theme 3.4.4 (GA) + Elementor 3.30.4 (GA)","hello_theme_version":"3.4.4","elementor_version":"3.30.4","delay":90},
#   {"combination":"ht-main-hp-latest","name":"Hello Theme main + Hello Plus 1.7.2","hello_theme_version":"main","hello_plus_version":"1.7.2","delay":105},
#   {"combination":"ht-main-hp-prev","name":"Hello Theme main + Hello Plus 1.7.1","hello_theme_version":"main","hello_plus_version":"1.7.1","delay":120}
# ]

name: Daily Test Matrix

on:
  schedule:
    # Run daily at 8:00 AM UTC (spread load from other themes)
    - cron: '0 8 * * *'
  workflow_dispatch:
    inputs:
      hello_theme_version:
        description: 'Hello Theme version to test (e.g., 3.4.4) - defaults to main for scheduled runs'
        required: false
        default: 'main'
        type: string
      hello_plus_version:
        description: 'Hello Plus version to test (e.g., 1.7.2) - defaults to latest-stable for scheduled runs'
        required: false
        default: 'latest-stable'
        type: string
      elementor_version:
        description: 'Elementor version to test (e.g., 3.29.7) - defaults to main for scheduled runs'
        required: false
        default: 'main'
        type: string
      compatibility_depth:
        description: 'Number of previous versions to test'
        required: false
        default: '2'
        type: choice
        options: ['1', '2', '3']
      run_targeted_tests:
        description: 'Run targeted compatibility tests'
        required: false
        default: true
        type: boolean
      tag:
        description: 'Provide @tag or a keyword for Playwright tests'
        required: false
        type: string

permissions:
  contents: read
  actions: write

env:
  CORE_WORKFLOW: 'playwright-with-specific-elementor-version.yml'
  PLUS_WORKFLOW: 'playwright-with-specific-hello-plus-version.yml'

jobs:
  calculate-test-matrix:
    name: Calculate Dynamic Test Matrix
    runs-on: ubuntu-22.04
    outputs:
      hello-theme-version: ${{ steps.matrix.outputs.hello-theme-version }}
      hello-plus-version: ${{ steps.matrix.outputs.hello-plus-version }}
      elementor-version: ${{ steps.matrix.outputs.elementor-version }}
      hello-theme-previous: ${{ steps.matrix.outputs.hello-theme-previous }}
      targeted-tests: ${{ steps.matrix.outputs.targeted-tests }}
      hello-theme-version-for-display: ${{ steps.matrix.outputs.hello-theme-version-for-display }}
    steps:
      - name: Checkout for scripts
        uses: actions/checkout@v4

      - name: Calculate version matrix
        id: matrix
        run: |
          HT_VERSION="${{ inputs.hello_theme_version || 'main' }}"
          HP_VERSION="${{ inputs.hello_plus_version || 'latest-stable' }}"
          EL_VERSION="${{ inputs.elementor_version || 'main' }}"
          DEPTH="${{ inputs.compatibility_depth || '2' }}"

          if [[ "${{ inputs.run_targeted_tests || 'true' }}" == "true" ]]; then
            RUN_TARGETED_TESTS="true"
          else
            RUN_TARGETED_TESTS="false"
          fi

          ./.github/scripts/generate-targeted-matrix.sh "$HT_VERSION" "$HP_VERSION" "$EL_VERSION" "$DEPTH" "$RUN_TARGETED_TESTS"

  notify-on-early-failure:
    name: Notify on Early Failure
    needs: calculate-test-matrix
    runs-on: ubuntu-22.04
    if: always() && needs.calculate-test-matrix.result != 'success'
    steps:
      - name: Send Slack notification on early failure
        continue-on-error: true
        uses: ./.github/actions/theme-slack-notification-daily-tests
        with:
          CLOUD_SLACK_BOT_TOKEN: ${{ secrets.CLOUD_SLACK_BOT_TOKEN }}
          WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          SLACK_CHANNEL: '#tmz-alerts'

  trigger-targeted-tests:
    name: Trigger Targeted Tests
    needs: [calculate-test-matrix]
    if: ${{ needs.calculate-test-matrix.outputs.targeted-tests != '[]' }}
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.calculate-test-matrix.outputs.targeted-tests) }}
    steps:
      - name: Checkout Hello Theme
        uses: actions/checkout@v4

      - name: Trigger targeted compatibility test
        id: trigger
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const { triggerWorkflowAndWait } = require('./.github/scripts/workflow-trigger.js');

            // Apply delay to prevent race conditions
            const delay = ${{ matrix.delay }} * 1000;
            if (delay > 0) {
              console.log(`Applying ${delay/1000}s delay to prevent race condition...`);
              await new Promise(resolve => setTimeout(resolve, delay));
            }

            // Determine which workflow to trigger based on combination type
            const workflowId = '${{ matrix.combination }}'.includes('hp') ? '${{ env.PLUS_WORKFLOW }}' : '${{ env.CORE_WORKFLOW }}';
            
            // Build inputs based on target workflow
            let inputs;
            if (workflowId === '${{ env.CORE_WORKFLOW }}') {
              // Core workflow: needs core_branch and hello_theme_version
              inputs = {
                core_branch: '${{ matrix.elementor_version || 'main' }}',
                hello_theme_version: '${{ matrix.hello_theme_version || 'main' }}'
              };
            } else {
              inputs = {
                hello_plus_version: '${{ matrix.hello_plus_version || 'latest-stable' }}',
                hello_theme_version: '${{ matrix.hello_theme_version || 'main' }}'
              };
            }
            
            // Add tag if provided
            if ('${{ inputs.tag || '' }}') {
              inputs.tag = '${{ inputs.tag }}';
            }
            
            const config = {
              combination: '${{ matrix.combination }}',
              name: '${{ matrix.name }}',
              workflowId: workflowId,
              ref: 'main',
              inputs: inputs
            };

            const result = await triggerWorkflowAndWait(github, context, core, config);
            console.log(`✅ Triggered: ${result.runUrl} for ${config.combination}`);
            
            // Store run metadata in a file for collection
            const fs = require('fs');
            const timestamp = Date.now();
            const runMetadata = {
              id: result.runId,
              url: result.runUrl,
              name: '${{ matrix.name }}',
              combination: '${{ matrix.combination }}',
              hello_theme_version: '${{ matrix.hello_theme_version }}',
              elementor_version: '${{ matrix.elementor_version || '' }}',
              hello_plus_version: '${{ matrix.hello_plus_version || '' }}',
              type: '${{ matrix.combination }}'.includes('hp') ? 'plus-matrix' : 'core-matrix',
              source: 'current-execution',
              timestamp: timestamp
            };

            const metadataFileName = `run-metadata-${{ matrix.combination }}-${timestamp}.json`;
            fs.writeFileSync(metadataFileName, JSON.stringify(runMetadata, null, 2));
            
            // Set output for the upload step
            core.setOutput('metadata-filename', metadataFileName);

      - name: Upload run metadata
        uses: actions/upload-artifact@v4
        with:
          name: run-metadata-${{ matrix.combination }}-${{ github.run_id }}-${{ strategy.job-index }}
          path: ${{ steps.trigger.outputs.metadata-filename }}
          retention-days: 1

  collect-matrix-results:
    name: Collect Matrix Results
    needs: [calculate-test-matrix, trigger-targeted-tests]
    runs-on: ubuntu-22.04
    if: always() && needs.calculate-test-matrix.result == 'success'
    outputs:
      all-workflow-data: ${{ steps.collect-data.outputs.all-workflow-data }}
    steps:
      - name: Checkout for scripts
        uses: actions/checkout@v4
      
      - name: Download all metadata artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: run-metadata-*
          merge-multiple: true
      
      - name: Collect workflow data
        id: collect-data
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const fs = require('fs');
            const path = require('path');
            
            let allWorkflowData = [];
            
            // Find all metadata files
            const files = fs.readdirSync('.').filter(file => file.startsWith('run-metadata-') && file.endsWith('.json'));
            
            console.log(`Found ${files.length} metadata files`);
            
            // Read and combine all metadata
            for (const file of files) {
              try {
                const content = fs.readFileSync(file, 'utf8');
                const metadata = JSON.parse(content);
                allWorkflowData.push(metadata);
                console.log(`Loaded metadata for ${metadata.combination}: ${metadata.id}`);
              } catch (error) {
                console.error(`Error reading ${file}: ${error.message}`);
              }
            }
            
            console.log(`Collected ${allWorkflowData.length} workflow runs`);
            
            // Set output for the next job
            core.setOutput('all-workflow-data', JSON.stringify(allWorkflowData));
            
            return allWorkflowData;

  daily-trigger-report:
    name: Daily Trigger Report
    needs: [calculate-test-matrix, trigger-targeted-tests, collect-matrix-results]
    runs-on: ubuntu-22.04
    if: always() && needs.calculate-test-matrix.result == 'success'
    outputs:
      overall-status: ${{ steps.aggregate-status.outputs.overall-status }}
      test-summary: ${{ steps.aggregate-status.outputs.test-summary }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Generate Hello Theme daily test report
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const { generateDailyTriggerReport } = require('./.github/scripts/daily-report.js');
            const { getDailyTriggerTimingConfig } = require('./.github/scripts/workflow-reporting.js');

            // Use the collected workflow data
            const collectedWorkflowData = '${{ needs.collect-matrix-results.outputs.all-workflow-data }}';
            let workflowData = [];

            if (collectedWorkflowData && collectedWorkflowData !== '') {
              try {
                workflowData = JSON.parse(collectedWorkflowData);
                console.log(`Using collected workflow data: ${workflowData.length} runs`);
              } catch (error) {
                console.error('Failed to parse collected workflow data:', error.message);
                console.log('Using empty workflow data array');
                workflowData = [];
              }
            } else {
              console.warn('No collected workflow data found');
              workflowData = [];
            }

            const timing = getDailyTriggerTimingConfig();

            console.log(`Generating report for ${workflowData.length} compatibility tests`);
            await generateDailyTriggerReport(github, context, core, workflowData, timing);

      - name: Aggregate test status
        id: aggregate-status
        uses: actions/github-script@v7
        with:
          script: |
            // Use the collected workflow data
            const collectedWorkflowData = '${{ needs.collect-matrix-results.outputs.all-workflow-data }}';
            let workflowData = [];

            if (collectedWorkflowData && collectedWorkflowData !== '') {
              try {
                workflowData = JSON.parse(collectedWorkflowData);
                console.log(`Using collected workflow data for aggregation: ${workflowData.length} runs`);
              } catch (error) {
                console.error('Failed to parse collected workflow data:', error.message);
                workflowData = [];
              }
            }
            
            const matrixResult = '${{ needs.trigger-targeted-tests.result }}';
            
            let overallStatus = 'success';
            let summary = {
              total: workflowData.length,
              success: 0,
              failed: 0,
              cancelled: 0,
              in_progress: 0,
              skipped: 0
            };
            
            console.log(`Aggregating status for ${workflowData.length} workflow runs`);
            console.log(`Matrix trigger result: ${matrixResult}`);
            
            // If matrix triggering failed, overall status is failure
            if (matrixResult !== 'success') {
              overallStatus = 'failure';
              core.setOutput('overall-status', overallStatus);
              core.setOutput('test-summary', JSON.stringify(summary));
              console.log('Matrix triggering failed, marking overall status as failure');
              return;
            }
            
            // For now, since we're just triggering workflows, we'll base status on successful triggering
            // In a future enhancement, we would poll the actual workflow results
            if (workflowData.length > 0) {
              summary.success = workflowData.length; // All workflows were successfully triggered
              console.log(`${workflowData.length} workflows triggered successfully`);
            } else {
              overallStatus = 'failure';
              console.log('No workflows were triggered');
            }
            
            console.log('Final summary:', JSON.stringify(summary, null, 2));
            console.log(`Overall status: ${overallStatus}`);
            
            core.setOutput('overall-status', overallStatus);
            core.setOutput('test-summary', JSON.stringify(summary));

      - name: Matrix configuration summary
        run: |
          echo "## ⚙️ Matrix Configuration Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "*This table shows the input parameters used to generate the test matrix, not the individual test results.*" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Component | Input Version | Resolution Source |" >> $GITHUB_STEP_SUMMARY
          echo "|-----------|---------------|-------------------|" >> $GITHUB_STEP_SUMMARY
          echo "| Hello Theme | \`${{ needs.calculate-test-matrix.outputs.hello-theme-version-for-display }}\` | GitHub/WordPress.org |" >> $GITHUB_STEP_SUMMARY
          echo "| Hello Plus | \`${{ needs.calculate-test-matrix.outputs.hello-plus-version }}\` | WordPress.org |" >> $GITHUB_STEP_SUMMARY
          echo "| Elementor | \`${{ needs.calculate-test-matrix.outputs.elementor-version }}\` | GitHub Releases |" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "**Previous versions included in matrix:** ${{ needs.calculate-test-matrix.outputs.hello-theme-previous || 'none' }}" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "**Matrix generated at:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "---" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

      - name: Final status check
        uses: actions/github-script@v7
        with:
          script: |
            const overallStatus = '${{ steps.aggregate-status.outputs.overall-status }}';
            const summary = JSON.parse('${{ steps.aggregate-status.outputs.test-summary }}');
            
            console.log(`Final workflow status check: ${overallStatus}`);
            console.log('Test summary:', summary);
            
            if (overallStatus === 'failure') {
              core.setFailed(`❌ Daily test matrix completed with failures: ${summary.failed} failed out of ${summary.total} tests`);
            } else if (overallStatus === 'cancelled') {
              core.setFailed(`🚫 Daily test matrix completed with cancellations: ${summary.cancelled} cancelled out of ${summary.total} tests`);
            } else if (overallStatus === 'in_progress') {
              console.log(`⏳ Some tests are still in progress: ${summary.in_progress} out of ${summary.total} tests`);
            } else {
              console.log(`✅ Daily test matrix completed successfully: ${summary.success} passed out of ${summary.total} tests`);
            }

      - name: Send Slack notification on failure
        if: failure()
        continue-on-error: true
        uses: ./.github/actions/theme-slack-notification-daily-tests
        with:
          CLOUD_SLACK_BOT_TOKEN: ${{ secrets.CLOUD_SLACK_BOT_TOKEN }}
          WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          SLACK_CHANNEL: '#tmz-alerts'