2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-10-03 17:21:20 +08:00

DEV: Publish and enable use of pre-built JS assets (#33973)

Building the Discourse ember app is resource-intensive process. This
commit introduces a framework for us to build these assets centrally,
and make them available for people to download.

On every commit to `main`, a new GitHub actions workflow will build
development & production versions of the core assets, and publish them
as a github release under the `discourse/discourse-assets` repository. A
separate repository is being used to avoid polluting the main
`discourse/discourse` repository with one-git-tag-per-release.

The `assemble_ember_build.rb` script is updated to fetch the relevant
asset bundle. Requests are made to `get.discourse.org`, which then
redirects to GitHub releases. This redirection service is being used so
that we have the option to switch away from GitHub releases in future
without breaking existing Discourse installations.

For now, this behavior can be enabled by setting
`DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS=1`. In the near future, we hope to
make this the default, with opt-out via
`DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS=0`.
This commit is contained in:
David Taylor 2025-07-31 14:22:51 +01:00 committed by GitHub
parent 459a58f3e1
commit fcaa068b87
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 179 additions and 16 deletions

52
.github/workflows/publish-assets.yml vendored Normal file
View file

@ -0,0 +1,52 @@
name: Publish Assets

on:
push:
branches:
- main
- stable

concurrency:
group: publish-assets-${{ format('{0}-{1}', github.ref, github.job) }}
cancel-in-progress: true

jobs:
publish-assets:
if: github.repository == 'discourse/discourse'
runs-on: 'debian-12-8core'
container: discourse/discourse_test:release
timeout-minutes: 30
steps:
- name: Install gh cli
run: |
# https://github.com/cli/cli/blob/trunk/docs/install_linux.md#debian-ubuntu-linux-raspberry-pi-os-apt
(type -p wget >/dev/null || (sudo apt update && sudo apt install wget -y)) \
&& sudo mkdir -p -m 755 /etc/apt/keyrings \
&& out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
&& cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
&& sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
&& sudo mkdir -p -m 755 /etc/apt/sources.list.d \
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
&& sudo apt update \
&& sudo apt install gh -y
- name: Set working directory owner
run: chown root:root .

- uses: actions/checkout@v4
with:
fetch-depth: 1

- name: Setup Git
run: |
git config --global user.email "ci@ci.invalid"
git config --global user.name "Discourse CI"

- name: pnpm install
run: pnpm install --frozen-lockfile

- name: build and release assets
env:
GITHUB_TOKEN: ${{ secrets.DISCOURSE_ASSETS_RELEASE_TOKEN }}
run: |
script/publish_built_assets.rb

View file

@ -6,7 +6,7 @@ task "assets:precompile:build" do

raise "Unknown ember version '#{ember_version}'" if !%w[5].include?(ember_version)

compile_command = "#{Rails.root}/script/assemble_ember_build.rb"
compile_command = "EMBER_ENV=production #{Rails.root}/script/assemble_ember_build.rb"

only_ember_precompile_build_remaining = (ARGV.last == "assets:precompile:build")
only_assets_precompile_remaining = (ARGV.last == "assets:precompile")

View file

@ -103,17 +103,20 @@ task "qunit:test", %i[qunit_path filter] do |_, args|
cmd = []

parallel = ENV["QUNIT_PARALLEL"]
reuse_build = ENV["QUNIT_REUSE_BUILD"] == "1"

if qunit_path
# Bypass `ember test` - it only works properly for the `/tests` path.
# We have to trigger a `build` manually so that JS is available for rails to serve.
system(
"pnpm",
"ember",
"build",
chdir: "#{Rails.root}/app/assets/javascripts/discourse",
exception: true,
)
if !reuse_build
system(
"pnpm",
"ember",
"build",
chdir: "#{Rails.root}/app/assets/javascripts/discourse",
exception: true,
)
end

env["THEME_TEST_PAGES"] = if ENV["THEME_IDS"]
ENV["THEME_IDS"]
@ -131,6 +134,7 @@ task "qunit:test", %i[qunit_path filter] do |_, args|
cmd += ["pnpm", "ember", "exam", "--query", query]
cmd += ["--load-balance", "--parallel", parallel] if parallel && !ENV["PLUGIN_TARGETS"]
cmd += ["--filter", filter] if filter
cmd += %w[--path dist] if reuse_build
cmd << "--write-execution-file" if ENV["QUNIT_WRITE_EXECUTION_FILE"]
end


View file

@ -6,6 +6,14 @@ require "fileutils"
require "tempfile"
require "open3"
require "json"
require "time"

require_relative "../lib/version"

DOWNLOAD_PRE_BUILT_ASSETS = ENV["DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS"] == "1"
DOWNLOAD_TEMP_FILE = "#{__dir__}/../tmp/assets.tar.gz"

PRE_BUILD_ROOT = "https://get.discourse.org/discourse-assets"

BUILD_INFO_FILE = "dist/BUILD_INFO.json"

@ -17,6 +25,10 @@ def capture(*args)
output
end

def log(message)
STDERR.puts "[assemble_ember_build] #{message}"
end

# Returns a git tree-hash representing the current state of Discourse core.
# If the working directory is clean, it will match the tree hash (note: different to the commit hash) of the HEAD commit.
def core_tree_hash
@ -41,7 +53,7 @@ def low_memory_environment?
end

def resolved_ember_env
ENV["EMBER_ENV"] || "production"
ENV["EMBER_ENV"] || "development"
end

def build_info
@ -50,7 +62,7 @@ end

def existing_core_build_usable?
if !File.exist?(BUILD_INFO_FILE)
STDERR.puts "No existing build info file found."
log "No existing build info file found."
return false
end

@ -60,7 +72,7 @@ def existing_core_build_usable?
if existing == expected
true
else
STDERR.puts <<~MSG
log <<~MSG
Existing build is not reusable.
- Existing: #{existing.inspect}
- Current: #{expected.inspect}
@ -69,6 +81,43 @@ def existing_core_build_usable?
end
end

def download_prebuild_assets!
return false if !DOWNLOAD_PRE_BUILT_ASSETS

git_is_clean = capture("git", "status", "--porcelain").strip.empty?
if !git_is_clean
log "Git working directory is not clean. Cannot download prebuilt assets."
return false
end

core_commit_hash = capture("git", "rev-parse", "HEAD").strip
version_string = "#{Discourse::VERSION::STRING}-#{core_commit_hash.slice(0, 8)}"

url = "#{PRE_BUILD_ROOT}/#{version_string}/#{resolved_ember_env}.tar.gz"
puts "Fetching and extracting #{url}..."

begin
system("curl", "--fail-with-body", "-L", url, "-o", DOWNLOAD_TEMP_FILE, exception: true)
rescue RuntimeError => e
log "Failed to download prebuilt assets: #{e.message}"
return false
end

FileUtils.rm_rf("dist")
FileUtils.mkdir_p("dist")
begin
system("tar", "--strip-components=1", "-xzf", DOWNLOAD_TEMP_FILE, "-C", "dist", exception: true)
rescue RuntimeError => e
log "Failed to extract prebuilt assets: #{e.message}"
return false
end

puts "Prebuilt assets downloaded and extracted successfully."
true
ensure
FileUtils.rm_f(DOWNLOAD_TEMP_FILE) if File.exist?(DOWNLOAD_TEMP_FILE)
end

build_cmd = %w[pnpm ember build]
build_env = { "CI" => "1" }

@ -78,7 +127,7 @@ if Etc.nprocessors > 2
end

if low_memory_environment?
STDERR.puts "Node.js heap_size_limit is less than 2048MB. Setting --max-old-space-size=2048 and CHEAP_SOURCE_MAPS=1"
log "Node.js heap_size_limit is less than 2048MB. Setting --max-old-space-size=2048 and CHEAP_SOURCE_MAPS=1"
build_env["NODE_OPTIONS"] = "--max_old_space_size=2048"
build_env["CHEAP_SOURCE_MAPS"] = "1"
build_env["JOBS"] = "1"
@ -86,8 +135,8 @@ end

build_cmd << "-prod" if resolved_ember_env == "production"

if existing_core_build_usable?
STDERR.puts "Reusing existing core ember build. Only building plugins..."
if existing_core_build_usable? || (download_prebuild_assets! && existing_core_build_usable?)
log "Reusing existing core ember build. Only building plugins..."
build_env["SKIP_CORE_BUILD"] = "1"
build_cmd << "-o" << "dist/_plugin_only_build"
begin
@ -97,9 +146,9 @@ if existing_core_build_usable?
ensure
FileUtils.rm_rf("dist/_plugin_only_build")
end
STDERR.puts "Plugin build successfully integrated into dist"
log "Plugin build successfully integrated into dist"
else
STDERR.puts "Running full core build..."
log "Running full core build..."
system(build_env, *build_cmd, exception: true)
File.write(BUILD_INFO_FILE, JSON.pretty_generate(build_info))
end

58
script/publish_built_assets.rb Executable file
View file

@ -0,0 +1,58 @@
#!/usr/bin/env ruby
# frozen_string_literal: true
# rubocop:disable Discourse/NoChdir

require "fileutils"
require_relative "../lib/version"
require "open3"

tmp_dir = "#{__dir__}/../tmp/prebuilt_asset_bundles"
FileUtils.mkdir_p(tmp_dir)

core_commit_hash = `git rev-parse HEAD`.strip
version_string = "v#{Discourse::VERSION::STRING}-#{core_commit_hash.slice(0, 8)}"
release_repo = "discourse/discourse-assets"

puts "Checking if release #{version_string} already exists in #{release_repo}..."

out, status =
Open3.capture2e("gh", "--repo", release_repo, "release", "view", version_string, "--json", "name")
puts out

if status.success?
puts "Release #{version_string} already exists in #{release_repo}. Exiting."
exit 0
end

common_env = { "DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS" => "0", "LOAD_PLUGINS" => "0" }

Dir.chdir("#{__dir__}/../app/assets/javascripts/discourse")
FileUtils.rm_rf("dist")

system({ **common_env, "EMBER_ENV" => "production" }, "#{__dir__}/assemble_ember_build.rb")
FileUtils.rm_rf("dist/assets/plugins")
system("tar", "-czf", "#{tmp_dir}/production.tar.gz", "dist", exception: true)

FileUtils.rm_rf("dist")
system({ **common_env, "EMBER_ENV" => "development" }, "#{__dir__}/assemble_ember_build.rb")
FileUtils.rm_rf("dist/assets/plugins")
system("tar", "-czf", "#{tmp_dir}/development.tar.gz", "dist", exception: true)

puts "Creating release #{version_string} in #{release_repo}..."
system(
"gh",
"--repo",
release_repo,
"release",
"create",
version_string,
"#{tmp_dir}/production.tar.gz",
"#{tmp_dir}/development.tar.gz",
"--title",
version_string,
"--notes",
version_string,
exception: true,
)

FileUtils.rm_rf(tmp_dir)