Discourse/discourse
Discourse/discourse
0
0
Fork 0
mirror of https://gh.wpcy.net/https://github.com/discourse/discourse.git synced 2026-05-08 05:58:27 +08:00
Code Packages Activity Actions
discourse/.github/workflows/tests.yml
David Taylor aa2fb29fa6
DEV: Use rollup for theme JS compilation (#33103) 
This commit is a complete reimplementation of our theme JS compilation
system.

Previously, we compiled theme JS into AMD `define` statements on a
per-source-file basis, and then concatenated them together for the
client. These AMD modules would integrate with those in Discourse core,
allowing two way access between core/theme modules. Going forward, we'll
be moving away from AMD, and towards native ES modules in core. Before
we can do that, we need to stop relying on AMD as the 'glue' between
core and themes/plugins.

This change introduces Rollup (running in mini-racer) as a compiler for
theme JS. This is configured to generate a single ES Module which
exports a list of 'compat modules'. Core `import()`s the modules for
each active theme, and adds them all to AMD. In future, this consumption
can be updated to avoid AMD entirely.

All module resolution within a theme is handled by Rollup, and does not
use AMD.

Import of core/plugin modules from themes are automatically transformed
into calls to a new `window.moduleBroker` interface. For now, this is a
direct interface to AMD. In future, this can be updated to point to real
ES Modules in core.

Despite the complete overhaul of the internals, this is not a breaking
change, and should have no impact on existing themes. If any
incompatibilities are found, please report them on
https://meta.discourse.org.

---------

Co-authored-by: Jarek Radosz <jarek@cvx.dev>
Co-authored-by: Chris Manson <chris@manson.ie>
2025-07-25 12:02:29 +01:00

413 lines
16 KiB
YAML
Raw Permalink Blame History

name: Tests
on:
pull_request:
paths-ignore:
- ".github/workflows/migration-tests.yml"
- ".github/dependabot.yml"
- ".github/labeler.yml"
- "migrations/**"
push:
branches:
- main
- beta
- stable
paths-ignore:
- ".github/workflows/migration-tests.yml"
- ".github/dependabot.yml"
- ".github/labeler.yml"
- "migrations/**"
concurrency:
group: tests-${{ format('{0}-{1}', github.head_ref || github.run_number, github.job) }}
cancel-in-progress: true
permissions:
contents: read
jobs:
build:
if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'
name: ${{ matrix.target }} ${{ matrix.build_type }}${{ (matrix.target == 'core' && matrix.build_type == 'frontend' && format(' ({0})', matrix.browser)) || '' }} # Update fetch-job-id step if changing this
runs-on: ${{ (github.repository_owner == 'discourse' && (matrix.target == 'plugins' && matrix.build_type == 'system' && 'debian-12-16core' || 'debian-12-8core')) || 'ubuntu-latest' }}
container: discourse/discourse_test:release
timeout-minutes: ${{ ((matrix.build_type == 'system' && matrix.target == 'plugins') && 30) || 20 }}
env:
RAILS_ENV: test
PGUSER: discourse
PGPASSWORD: discourse
USES_PARALLEL_DATABASES: ${{ matrix.build_type == 'backend' || matrix.build_type == 'system' }}
CAPYBARA_DEFAULT_MAX_WAIT_TIME: 20
MINIO_RUNNER_LOG_LEVEL: DEBUG
DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS: ${{ (matrix.build_type == 'system' || matrix.build_type == 'backend') && '1' }}
CHEAP_SOURCE_MAPS: "1"
MINIO_RUNNER_INSTALL_DIR: /home/discourse/.minio_runner
TESTEM_BROWSER: ${{ (startsWith(matrix.browser, 'Firefox') && 'Firefox') || matrix.browser }}
TESTEM_FIREFOX_PATH: ${{ (matrix.browser == 'Firefox Evergreen') && '/opt/firefox-evergreen/firefox' }}
EMBER_ENV: development
LOAD_PLUGINS: ${{ (matrix.target == 'plugins' || matrix.target == 'chat') && '1' || '0' }}
strategy:
fail-fast: false
matrix:
build_type: [backend, frontend, system]
target: [core, plugins, themes]
browser: [Chrome]
exclude:
- build_type: backend
target: themes
include:
- build_type: system
target: chat
- build_type: frontend
target: core
browser: Firefox Evergreen
- build_type: frontend
target: core
browser: Firefox ESR
steps:
- name: Set working directory owner
run: chown root:root .
- name: Set PARALLEL_TEST_PROCESSORS for system tests
if: matrix.build_type == 'system'
run: |
echo "PARALLEL_TEST_PROCESSORS=$(($(nproc) / 2))" >> $GITHUB_ENV
- name: Set QUNIT_PARALLEL for QUnit tests
if: matrix.build_type == 'frontend'
run: |
if [ "${{ matrix.target }}" = "themes" ]; then
echo "QUNIT_PARALLEL=2" >> $GITHUB_ENV
else
echo "QUNIT_PARALLEL=$(($(nproc) / 2))" >> $GITHUB_ENV
fi
- uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Setup Git
run: |
git config --global user.email "ci@ci.invalid"
git config --global user.name "Discourse CI"
- name: Start redis
run: |
redis-server /etc/redis/redis.conf &
- name: Start Postgres
run: |
chown -R postgres /var/run/postgresql
sudo -E -u postgres script/start_test_db.rb
sudo -u postgres psql -c "CREATE ROLE $PGUSER LOGIN SUPERUSER PASSWORD '$PGPASSWORD';"
- name: Symlink vendor/bundle from image
run: |
ln -s /var/www/discourse/vendor/bundle vendor/bundle
- name: Setup gems
run: |
gem install bundler --conservative -v $(awk '/BUNDLED WITH/ { getline; gsub(/ /,""); print $0 }' Gemfile.lock)
bundle config --local path vendor/bundle
bundle config --local deployment true
bundle config --local without development
bundle install --jobs $(($(nproc) - 1))
bundle clean
- name: pnpm install
run: pnpm install --frozen-lockfile
- name: Checkout official plugins
if: matrix.target == 'plugins'
run: bin/rake plugin:install_all_official
- name: Symlinking plugin gems from image
if: matrix.target == 'plugins'
run: |
for dir in /var/www/discourse/plugins/*/gems; do
plugin_name=$(basename "$(dirname "$dir")")
plugin_dir="plugins/$plugin_name"
gem_dir="$plugin_dir/gems"
if [ ! -d "$plugin_dir" ]; then
echo "Skipping $plugin_name: Plugin directory does not exist"
elif [ -d "$gem_dir" ]; then
echo "Skipping $plugin_name: Source gems directory exists"
else
echo "Symlinking $dir to $gem_dir"
ln -s "$dir" "$gem_dir"
fi
done
- name: Pull compatible versions of plugins
if: matrix.target == 'plugins' && (github.ref_name != 'main' && github.base_ref != 'main')
run: bin/rake plugin:pull_compatible_all
- name: Checkout official themes
if: matrix.target == 'themes'
run: bin/rake themes:clone_all_official themes:pull_compatible_all
- name: Add hosts to /etc/hosts, otherwise Chrome cannot reach minio
if: matrix.build_type == 'system' && matrix.target == 'core'
run: |
echo "127.0.0.1 minio.local" | sudo tee -a /etc/hosts
echo "127.0.0.1 discoursetest.minio.local" | sudo tee -a /etc/hosts
- name: Get CPU cores
id: cpu-info
run: echo "cpu-cores=$(nproc)" >> $GITHUB_OUTPUT
- name: Fetch app state cache
uses: actions/cache@v4
id: app-cache
with:
path: tmp/app-cache
key: >-
${{ runner.os }}-
${{ steps.cpu-info.outputs.cpu-cores }}-
${{ hashFiles('.github/workflows/tests.yml') }}-
${{ hashFiles('db/**/*', 'plugins/**/db/**/*') }}-
${{ hashFiles('config/environments/test.rb') }}-
${{ env.LOAD_PLUGINS }}-
${{ env.USES_PARALLEL_DATABASES }}-
${{ env.PARALLEL_TEST_PROCESSORS }}-
${{ matrix.target }}-
- name: Restore database from cache
if: steps.app-cache.outputs.cache-hit == 'true'
run: script/silence_successful_output psql --quiet -o /dev/null -f tmp/app-cache/cache.sql postgres
- name: Restore uploads from cache
if: steps.app-cache.outputs.cache-hit == 'true'
run: rm -rf public/uploads && cp -r tmp/app-cache/uploads public/uploads
- name: Create and migrate database
if: steps.app-cache.outputs.cache-hit != 'true'
env:
LOAD_PLUGINS: ${{ (matrix.target == 'plugins' || matrix.target == 'chat') && '1' || '0' }}
run: |
bin/rake db:create
script/silence_successful_output bin/rake db:migrate
- name: Create and migrate parallel databases
if: >-
env.USES_PARALLEL_DATABASES == 'true' &&
steps.app-cache.outputs.cache-hit != 'true'
env:
LOAD_PLUGINS: ${{ (matrix.target == 'plugins' || matrix.target == 'chat') && '1' || '0' }}
run: |
bin/rake parallel:create
script/silence_successful_output bin/rake parallel:migrate
- name: Dump database for cache
if: steps.app-cache.outputs.cache-hit != 'true'
run: mkdir -p tmp/app-cache && pg_dumpall > tmp/app-cache/cache.sql
- name: Dump uploads for cache
if: steps.app-cache.outputs.cache-hit != 'true'
run: rm -rf tmp/app-cache/uploads && cp -r public/uploads tmp/app-cache/uploads
- name: Fetch turbo_rspec_runtime.log cache
uses: actions/cache@v4
id: test-runtime-cache
if: matrix.build_type == 'backend' || matrix.build_type == 'system'
with:
path: tmp/turbo_rspec_runtime.log
key: rspec-runtime-${{ matrix.build_type }}-${{ matrix.target }}-${{ github.run_id }}
restore-keys: rspec-runtime-${{ matrix.build_type }}-${{ matrix.target }}-
- name: Check Annotations
if: matrix.build_type == 'backend' && (matrix.target == 'core' || matrix.target == 'plugins' || matrix.target == 'chat')
run: |
bin/rake annotate:ensure_all_indexes
directories=""
if [ "${{ matrix.target }}" = "core" ]; then
directories="app/models"
else
directories="$(script/list_bundled_plugins | xargs -I{} find {} -type d -path '*/app/models' -maxdepth 3)"
fi
bin/annotaterb models --model-dir $(echo "${directories}" | tr '\n' ',')
if [ ! -z "$(git status --porcelain ${directories})" ]; then
echo "Core annotations are not up to date. To resolve, run:"
echo " bin/rake annotate:clean"
echo
echo "Or manually apply the diff printed below:"
echo "---------------------------------------------"
git -c color.ui=always diff ${directories}
exit 1
fi
- name: Check Zeitwerk reloading
if: matrix.build_type == 'backend'
run: |
if ! bin/rails runner 'Rails.application.reloader.reload!'; then
echo
echo "---------------------------------------------"
echo
echo "::error::Zeitwerk reload failed - the app will not be able to reload properly in development."
echo "To reproduce locally, run \`bin/rails runner 'Rails.application.reloader.reload!'\`."
echo
exit 1
fi
- name: Check SKIP_DB_AND_REDIS bootability
if: matrix.build_type == 'backend'
run: |
if ! (SKIP_DB_AND_REDIS=1 DISCOURSE_DEV_DB="nonexist" bin/rails runner "puts 'booted successfully'"); then
echo
echo "---------------------------------------------"
echo
echo "::error::SKIP_DB_AND_REDIS boot failed. Make sure the database is not being accessed during the Rails boot process."
echo "To reproduce locally, run \`SKIP_DB_AND_REDIS=1 DISCOURSE_DEV_DB='nonexist' bin/rails runner \"puts 'booted successfully'\"\`."
echo
exit 1
fi
- name: Core RSpec
if: matrix.build_type == 'backend' && matrix.target == 'core'
run: bin/turbo_rspec --use-runtime-info --verbose --format=documentation --profile=50
- name: Plugin RSpec
if: matrix.build_type == 'backend' && matrix.target == 'plugins'
run: bin/rake plugin:turbo_spec['*','--verbose --format=documentation --use-runtime-info --profile=50']
- name: Core QUnit
if: matrix.build_type == 'frontend' && matrix.target == 'core'
run: QUNIT_WRITE_EXECUTION_FILE=1 bin/rake qunit:test
- name: Theme Transpiler Tests
if: matrix.build_type == 'frontend' && matrix.target == 'core'
working-directory: app/assets/javascripts/theme-transpiler
run: pnpm test
- name: Plugin QUnit
if: matrix.build_type == 'frontend' && matrix.target == 'plugins'
run: bin/rake plugin:qunit['*']
- name: Theme QUnit
if: matrix.build_type == 'frontend' && matrix.target == 'themes'
run: DISCOURSE_DEV_DB=discourse_test bin/rake themes:qunit_all_official
- uses: actions/upload-artifact@v4
if: always() && matrix.build_type == 'frontend'
with:
name: ember-exam-execution-${{ matrix.target }}-${{ matrix.browser }}-frontend-${{ hashFiles('./app/assets/javascripts/discourse/test-execution-*.json') }}
path: ./app/assets/javascripts/discourse/test-execution-*.json
- name: Ember Build for System Tests
if: matrix.build_type == 'system'
run: script/assemble_ember_build.rb
- name: Install playwright
if: matrix.build_type == 'system'
run: pnpm playwright install --with-deps --no-shell chromium
- name: Core System Tests
if: matrix.build_type == 'system' && matrix.target == 'core'
env:
CHECKOUT_TIMEOUT: 10
run: RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation spec/system
- name: Plugin System Tests
if: matrix.build_type == 'system' && matrix.target == 'plugins'
env:
CHECKOUT_TIMEOUT: 10
run: |
RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error bin/turbo_rspec --exclude-pattern="plugins/chat/*" --use-runtime-info --profile=50 --verbose --format documentation plugins/*/spec/system
shell: bash
- name: Chat System Tests
if: matrix.build_type == 'system' && matrix.target == 'chat'
env:
CHECKOUT_TIMEOUT: 10
run: RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation plugins/chat/spec/system
- name: Theme System Tests
if: matrix.build_type == 'system' && matrix.target == 'themes'
env:
CHECKOUT_TIMEOUT: 10
run: |
RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error bin/turbo_rspec --profile=50 --verbose --format documentation tmp/themes/*/spec/system themes/*/spec/system
shell: bash
- name: Check for failed system test artifacts
id: check-failed-system-test-artifacts
if: always() && matrix.build_type == 'system'
run: |
if [ -d tmp/capybara ] && [ "$(ls -A tmp/capybara/)" ]; then
echo "exists=true" >> $GITHUB_OUTPUT
else
echo "exists=false" >> $GITHUB_OUTPUT
fi
shell: bash
- name: Upload failed system test artifacts
uses: actions/upload-artifact@v4
if: always() && steps.check-failed-system-test-artifacts.outputs.exists == 'true'
with:
name: failed-system-test-artifacts-${{ matrix.build_type }}-${{ matrix.target }}
path: |
tmp/capybara/*.png
tmp/capybara/*.zip
tmp/capybara/*.webm
- name: Check for flaky tests report
id: check-flaky-spec-report
if: github.repository == 'discourse/discourse' && github.ref_name == 'main' && env.DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS == '1'
run: |
if [ -f tmp/turbo_rspec_flaky_tests.json ]; then
echo "exists=true" >> $GITHUB_OUTPUT
else
echo "exists=false" >> $GITHUB_OUTPUT
fi
shell: bash
- name: Fetch Job ID
id: fetch-job-id
if: steps.check-flaky-spec-report.outputs.exists == 'true'
run: |
job_id=$(ruby script/get_github_workflow_run_job_id.rb ${{ github.run_id }} ${{ github.run_attempt }} '${{ matrix.target }} ${{ matrix.build_type }}${{ (matrix.target == 'core' && matrix.build_type == 'frontend' && format(' ({0})', matrix.browser)) || '' }}')
echo "job_id=$job_id" >> $GITHUB_OUTPUT
- name: Create flaky tests report artifact
if: steps.check-flaky-spec-report.outputs.exists == 'true'
run: cp tmp/turbo_rspec_flaky_tests.json tmp/turbo_rspec_flaky_tests-${{ matrix.build_type }}-${{ matrix.target }}-${{ steps.fetch-job-id.outputs.job_id }}.json
- name: Upload flaky tests report
uses: actions/upload-artifact@v4
if: steps.check-flaky-spec-report.outputs.exists == 'true'
with:
name: flaky-test-reports-${{ matrix.build_type }}-${{ matrix.target }}
path: tmp/turbo_rspec_flaky_tests-${{ matrix.build_type }}-${{ matrix.target }}-${{ steps.fetch-job-id.outputs.job_id }}.json
merge:
if: github.repository == 'discourse/discourse' && github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
needs: build
steps:
- name: Merge Artifacts
uses: actions/upload-artifact/merge@v4
with:
name: failed-system-test-artifacts
pattern: failed-system-test-artifacts-*
delete-merged: true
separate-directories: false
# Don't fail the job if there aren't any artifacts to merge.
continue-on-error: true
- name: Merge Artifacts
uses: actions/upload-artifact/merge@v4
with:
name: flaky-test-reports
pattern: flaky-test-reports-*
delete-merged: true
separate-directories: false
# Don't fail the job if there aren't any artifacts to merge.
continue-on-error: true
View git blame Copy permalink