mirror of
https://github.com/discourse/discourse.git
synced 2025-10-03 17:21:20 +08:00
DEV: Publish and enable use of pre-built JS assets (#33973)
Building the Discourse ember app is resource-intensive process. This commit introduces a framework for us to build these assets centrally, and make them available for people to download. On every commit to `main`, a new GitHub actions workflow will build development & production versions of the core assets, and publish them as a github release under the `discourse/discourse-assets` repository. A separate repository is being used to avoid polluting the main `discourse/discourse` repository with one-git-tag-per-release. The `assemble_ember_build.rb` script is updated to fetch the relevant asset bundle. Requests are made to `get.discourse.org`, which then redirects to GitHub releases. This redirection service is being used so that we have the option to switch away from GitHub releases in future without breaking existing Discourse installations. For now, this behavior can be enabled by setting `DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS=1`. In the near future, we hope to make this the default, with opt-out via `DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS=0`.
This commit is contained in:
parent
459a58f3e1
commit
fcaa068b87
5 changed files with 179 additions and 16 deletions
52
.github/workflows/publish-assets.yml
vendored
Normal file
52
.github/workflows/publish-assets.yml
vendored
Normal file
|
@ -0,0 +1,52 @@
|
|||
name: Publish Assets
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- stable
|
||||
|
||||
concurrency:
|
||||
group: publish-assets-${{ format('{0}-{1}', github.ref, github.job) }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
publish-assets:
|
||||
if: github.repository == 'discourse/discourse'
|
||||
runs-on: 'debian-12-8core'
|
||||
container: discourse/discourse_test:release
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Install gh cli
|
||||
run: |
|
||||
# https://github.com/cli/cli/blob/trunk/docs/install_linux.md#debian-ubuntu-linux-raspberry-pi-os-apt
|
||||
(type -p wget >/dev/null || (sudo apt update && sudo apt install wget -y)) \
|
||||
&& sudo mkdir -p -m 755 /etc/apt/keyrings \
|
||||
&& out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
|
||||
&& cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
|
||||
&& sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
|
||||
&& sudo mkdir -p -m 755 /etc/apt/sources.list.d \
|
||||
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
|
||||
&& sudo apt update \
|
||||
&& sudo apt install gh -y
|
||||
- name: Set working directory owner
|
||||
run: chown root:root .
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup Git
|
||||
run: |
|
||||
git config --global user.email "ci@ci.invalid"
|
||||
git config --global user.name "Discourse CI"
|
||||
|
||||
- name: pnpm install
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: build and release assets
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.DISCOURSE_ASSETS_RELEASE_TOKEN }}
|
||||
run: |
|
||||
script/publish_built_assets.rb
|
|
@ -6,7 +6,7 @@ task "assets:precompile:build" do
|
|||
|
||||
raise "Unknown ember version '#{ember_version}'" if !%w[5].include?(ember_version)
|
||||
|
||||
compile_command = "#{Rails.root}/script/assemble_ember_build.rb"
|
||||
compile_command = "EMBER_ENV=production #{Rails.root}/script/assemble_ember_build.rb"
|
||||
|
||||
only_ember_precompile_build_remaining = (ARGV.last == "assets:precompile:build")
|
||||
only_assets_precompile_remaining = (ARGV.last == "assets:precompile")
|
||||
|
|
|
@ -103,17 +103,20 @@ task "qunit:test", %i[qunit_path filter] do |_, args|
|
|||
cmd = []
|
||||
|
||||
parallel = ENV["QUNIT_PARALLEL"]
|
||||
reuse_build = ENV["QUNIT_REUSE_BUILD"] == "1"
|
||||
|
||||
if qunit_path
|
||||
# Bypass `ember test` - it only works properly for the `/tests` path.
|
||||
# We have to trigger a `build` manually so that JS is available for rails to serve.
|
||||
system(
|
||||
"pnpm",
|
||||
"ember",
|
||||
"build",
|
||||
chdir: "#{Rails.root}/app/assets/javascripts/discourse",
|
||||
exception: true,
|
||||
)
|
||||
if !reuse_build
|
||||
system(
|
||||
"pnpm",
|
||||
"ember",
|
||||
"build",
|
||||
chdir: "#{Rails.root}/app/assets/javascripts/discourse",
|
||||
exception: true,
|
||||
)
|
||||
end
|
||||
|
||||
env["THEME_TEST_PAGES"] = if ENV["THEME_IDS"]
|
||||
ENV["THEME_IDS"]
|
||||
|
@ -131,6 +134,7 @@ task "qunit:test", %i[qunit_path filter] do |_, args|
|
|||
cmd += ["pnpm", "ember", "exam", "--query", query]
|
||||
cmd += ["--load-balance", "--parallel", parallel] if parallel && !ENV["PLUGIN_TARGETS"]
|
||||
cmd += ["--filter", filter] if filter
|
||||
cmd += %w[--path dist] if reuse_build
|
||||
cmd << "--write-execution-file" if ENV["QUNIT_WRITE_EXECUTION_FILE"]
|
||||
end
|
||||
|
||||
|
|
|
@ -6,6 +6,14 @@ require "fileutils"
|
|||
require "tempfile"
|
||||
require "open3"
|
||||
require "json"
|
||||
require "time"
|
||||
|
||||
require_relative "../lib/version"
|
||||
|
||||
DOWNLOAD_PRE_BUILT_ASSETS = ENV["DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS"] == "1"
|
||||
DOWNLOAD_TEMP_FILE = "#{__dir__}/../tmp/assets.tar.gz"
|
||||
|
||||
PRE_BUILD_ROOT = "https://get.discourse.org/discourse-assets"
|
||||
|
||||
BUILD_INFO_FILE = "dist/BUILD_INFO.json"
|
||||
|
||||
|
@ -17,6 +25,10 @@ def capture(*args)
|
|||
output
|
||||
end
|
||||
|
||||
def log(message)
|
||||
STDERR.puts "[assemble_ember_build] #{message}"
|
||||
end
|
||||
|
||||
# Returns a git tree-hash representing the current state of Discourse core.
|
||||
# If the working directory is clean, it will match the tree hash (note: different to the commit hash) of the HEAD commit.
|
||||
def core_tree_hash
|
||||
|
@ -41,7 +53,7 @@ def low_memory_environment?
|
|||
end
|
||||
|
||||
def resolved_ember_env
|
||||
ENV["EMBER_ENV"] || "production"
|
||||
ENV["EMBER_ENV"] || "development"
|
||||
end
|
||||
|
||||
def build_info
|
||||
|
@ -50,7 +62,7 @@ end
|
|||
|
||||
def existing_core_build_usable?
|
||||
if !File.exist?(BUILD_INFO_FILE)
|
||||
STDERR.puts "No existing build info file found."
|
||||
log "No existing build info file found."
|
||||
return false
|
||||
end
|
||||
|
||||
|
@ -60,7 +72,7 @@ def existing_core_build_usable?
|
|||
if existing == expected
|
||||
true
|
||||
else
|
||||
STDERR.puts <<~MSG
|
||||
log <<~MSG
|
||||
Existing build is not reusable.
|
||||
- Existing: #{existing.inspect}
|
||||
- Current: #{expected.inspect}
|
||||
|
@ -69,6 +81,43 @@ def existing_core_build_usable?
|
|||
end
|
||||
end
|
||||
|
||||
def download_prebuild_assets!
|
||||
return false if !DOWNLOAD_PRE_BUILT_ASSETS
|
||||
|
||||
git_is_clean = capture("git", "status", "--porcelain").strip.empty?
|
||||
if !git_is_clean
|
||||
log "Git working directory is not clean. Cannot download prebuilt assets."
|
||||
return false
|
||||
end
|
||||
|
||||
core_commit_hash = capture("git", "rev-parse", "HEAD").strip
|
||||
version_string = "#{Discourse::VERSION::STRING}-#{core_commit_hash.slice(0, 8)}"
|
||||
|
||||
url = "#{PRE_BUILD_ROOT}/#{version_string}/#{resolved_ember_env}.tar.gz"
|
||||
puts "Fetching and extracting #{url}..."
|
||||
|
||||
begin
|
||||
system("curl", "--fail-with-body", "-L", url, "-o", DOWNLOAD_TEMP_FILE, exception: true)
|
||||
rescue RuntimeError => e
|
||||
log "Failed to download prebuilt assets: #{e.message}"
|
||||
return false
|
||||
end
|
||||
|
||||
FileUtils.rm_rf("dist")
|
||||
FileUtils.mkdir_p("dist")
|
||||
begin
|
||||
system("tar", "--strip-components=1", "-xzf", DOWNLOAD_TEMP_FILE, "-C", "dist", exception: true)
|
||||
rescue RuntimeError => e
|
||||
log "Failed to extract prebuilt assets: #{e.message}"
|
||||
return false
|
||||
end
|
||||
|
||||
puts "Prebuilt assets downloaded and extracted successfully."
|
||||
true
|
||||
ensure
|
||||
FileUtils.rm_f(DOWNLOAD_TEMP_FILE) if File.exist?(DOWNLOAD_TEMP_FILE)
|
||||
end
|
||||
|
||||
build_cmd = %w[pnpm ember build]
|
||||
build_env = { "CI" => "1" }
|
||||
|
||||
|
@ -78,7 +127,7 @@ if Etc.nprocessors > 2
|
|||
end
|
||||
|
||||
if low_memory_environment?
|
||||
STDERR.puts "Node.js heap_size_limit is less than 2048MB. Setting --max-old-space-size=2048 and CHEAP_SOURCE_MAPS=1"
|
||||
log "Node.js heap_size_limit is less than 2048MB. Setting --max-old-space-size=2048 and CHEAP_SOURCE_MAPS=1"
|
||||
build_env["NODE_OPTIONS"] = "--max_old_space_size=2048"
|
||||
build_env["CHEAP_SOURCE_MAPS"] = "1"
|
||||
build_env["JOBS"] = "1"
|
||||
|
@ -86,8 +135,8 @@ end
|
|||
|
||||
build_cmd << "-prod" if resolved_ember_env == "production"
|
||||
|
||||
if existing_core_build_usable?
|
||||
STDERR.puts "Reusing existing core ember build. Only building plugins..."
|
||||
if existing_core_build_usable? || (download_prebuild_assets! && existing_core_build_usable?)
|
||||
log "Reusing existing core ember build. Only building plugins..."
|
||||
build_env["SKIP_CORE_BUILD"] = "1"
|
||||
build_cmd << "-o" << "dist/_plugin_only_build"
|
||||
begin
|
||||
|
@ -97,9 +146,9 @@ if existing_core_build_usable?
|
|||
ensure
|
||||
FileUtils.rm_rf("dist/_plugin_only_build")
|
||||
end
|
||||
STDERR.puts "Plugin build successfully integrated into dist"
|
||||
log "Plugin build successfully integrated into dist"
|
||||
else
|
||||
STDERR.puts "Running full core build..."
|
||||
log "Running full core build..."
|
||||
system(build_env, *build_cmd, exception: true)
|
||||
File.write(BUILD_INFO_FILE, JSON.pretty_generate(build_info))
|
||||
end
|
||||
|
|
58
script/publish_built_assets.rb
Executable file
58
script/publish_built_assets.rb
Executable file
|
@ -0,0 +1,58 @@
|
|||
#!/usr/bin/env ruby
|
||||
# frozen_string_literal: true
|
||||
# rubocop:disable Discourse/NoChdir
|
||||
|
||||
require "fileutils"
|
||||
require_relative "../lib/version"
|
||||
require "open3"
|
||||
|
||||
tmp_dir = "#{__dir__}/../tmp/prebuilt_asset_bundles"
|
||||
FileUtils.mkdir_p(tmp_dir)
|
||||
|
||||
core_commit_hash = `git rev-parse HEAD`.strip
|
||||
version_string = "v#{Discourse::VERSION::STRING}-#{core_commit_hash.slice(0, 8)}"
|
||||
release_repo = "discourse/discourse-assets"
|
||||
|
||||
puts "Checking if release #{version_string} already exists in #{release_repo}..."
|
||||
|
||||
out, status =
|
||||
Open3.capture2e("gh", "--repo", release_repo, "release", "view", version_string, "--json", "name")
|
||||
puts out
|
||||
|
||||
if status.success?
|
||||
puts "Release #{version_string} already exists in #{release_repo}. Exiting."
|
||||
exit 0
|
||||
end
|
||||
|
||||
common_env = { "DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS" => "0", "LOAD_PLUGINS" => "0" }
|
||||
|
||||
Dir.chdir("#{__dir__}/../app/assets/javascripts/discourse")
|
||||
FileUtils.rm_rf("dist")
|
||||
|
||||
system({ **common_env, "EMBER_ENV" => "production" }, "#{__dir__}/assemble_ember_build.rb")
|
||||
FileUtils.rm_rf("dist/assets/plugins")
|
||||
system("tar", "-czf", "#{tmp_dir}/production.tar.gz", "dist", exception: true)
|
||||
|
||||
FileUtils.rm_rf("dist")
|
||||
system({ **common_env, "EMBER_ENV" => "development" }, "#{__dir__}/assemble_ember_build.rb")
|
||||
FileUtils.rm_rf("dist/assets/plugins")
|
||||
system("tar", "-czf", "#{tmp_dir}/development.tar.gz", "dist", exception: true)
|
||||
|
||||
puts "Creating release #{version_string} in #{release_repo}..."
|
||||
system(
|
||||
"gh",
|
||||
"--repo",
|
||||
release_repo,
|
||||
"release",
|
||||
"create",
|
||||
version_string,
|
||||
"#{tmp_dir}/production.tar.gz",
|
||||
"#{tmp_dir}/development.tar.gz",
|
||||
"--title",
|
||||
version_string,
|
||||
"--notes",
|
||||
version_string,
|
||||
exception: true,
|
||||
)
|
||||
|
||||
FileUtils.rm_rf(tmp_dir)
|
Loading…
Add table
Add a link
Reference in a new issue