2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-08-17 18:04:11 +08:00

DEV: Use rollup for theme JS compilation (#33103)

This commit is a complete reimplementation of our theme JS compilation
system.

Previously, we compiled theme JS into AMD `define` statements on a
per-source-file basis, and then concatenated them together for the
client. These AMD modules would integrate with those in Discourse core,
allowing two way access between core/theme modules. Going forward, we'll
be moving away from AMD, and towards native ES modules in core. Before
we can do that, we need to stop relying on AMD as the 'glue' between
core and themes/plugins.

This change introduces Rollup (running in mini-racer) as a compiler for
theme JS. This is configured to generate a single ES Module which
exports a list of 'compat modules'. Core `import()`s the modules for
each active theme, and adds them all to AMD. In future, this consumption
can be updated to avoid AMD entirely.

All module resolution within a theme is handled by Rollup, and does not
use AMD.

Import of core/plugin modules from themes are automatically transformed
into calls to a new `window.moduleBroker` interface. For now, this is a
direct interface to AMD. In future, this can be updated to point to real
ES Modules in core.

Despite the complete overhaul of the internals, this is not a breaking
change, and should have no impact on existing themes. If any
incompatibilities are found, please report them on
https://meta.discourse.org.

---------

Co-authored-by: Jarek Radosz <jarek@cvx.dev>
Co-authored-by: Chris Manson <chris@manson.ie>
This commit is contained in:
David Taylor 2025-07-25 12:02:29 +01:00 committed by GitHub
parent dc37c326a6
commit aa2fb29fa6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
40 changed files with 2310 additions and 718 deletions

View file

@ -282,6 +282,11 @@ jobs:
if: matrix.build_type == 'frontend' && matrix.target == 'core'
run: QUNIT_WRITE_EXECUTION_FILE=1 bin/rake qunit:test
- name: Theme Transpiler Tests
if: matrix.build_type == 'frontend' && matrix.target == 'core'
working-directory: app/assets/javascripts/theme-transpiler
run: pnpm test
- name: Plugin QUnit
if: matrix.build_type == 'frontend' && matrix.target == 'plugins'
run: bin/rake plugin:qunit['*']

View file

@ -100,7 +100,6 @@ gem "rinku"
gem "sidekiq"
gem "mini_scheduler"
gem "execjs", require: false
gem "mini_racer"
gem "highline", require: false

View file

@ -157,7 +157,6 @@ GEM
erubi (1.13.1)
excon (1.2.5)
logger
execjs (2.10.0)
exifr (1.4.1)
extralite-bundle (2.12)
fabrication (3.0.0)
@ -758,7 +757,6 @@ DEPENDENCIES
ed25519
email_reply_trimmer
excon
execjs
extralite-bundle
fabrication
faker
@ -945,7 +943,6 @@ CHECKSUMS
erb (5.0.2) sha256=d30f258143d4300fb4ecf430042ac12970c9bb4b33c974a545b8f58c1ec26c0f
erubi (1.13.1) sha256=a082103b0885dbc5ecf1172fede897f9ebdb745a4b97a5e8dc63953db1ee4ad9
excon (1.2.5) sha256=ca040bb61bc0059968f34a17115a00d2db8562e3c0c5c5c7432072b551c85a9d
execjs (2.10.0) sha256=6bcb8be8f0052ff9d370b65d1c080f2406656e150452a0abdb185a133048450d
exifr (1.4.1) sha256=768374cc6b6ff3743acba57c1c35229bdd8c6b9fbc1285952047fc1215c4b894
extralite-bundle (2.12) sha256=9c9912b7ab592e7064089ee608cf86ab9edd81d20065acb87d1f4fed06052987
fabrication (3.0.0) sha256=a6a0bfad9071348ad3cb1701df788524b888c0cdd044b988893e7509f7463be3

View file

@ -19,6 +19,37 @@ import { buildResolver } from "discourse/resolver";
const _pluginCallbacks = [];
let _unhandledThemeErrors = [];
window.moduleBroker = {
async lookup(moduleName) {
return require(moduleName);
},
};
async function loadThemeFromModulePreload(link) {
const themeId = link.dataset.themeId;
try {
const compatModules = (await import(/* webpackIgnore: true */ link.href))
.default;
for (const [key, mod] of Object.entries(compatModules)) {
define(`discourse/theme-${themeId}/${key}`, () => mod);
}
} catch (error) {
// eslint-disable-next-line no-console
console.error(
`Failed to load theme ${link.dataset.themeId} from ${link.href}`,
error
);
fireThemeErrorEvent({ themeId: link.dataset.themeId, error });
}
}
export async function loadThemes() {
const promises = [
...document.querySelectorAll("link[rel=modulepreload][data-theme-id]"),
].map(loadThemeFromModulePreload);
await Promise.all(promises);
}
class Discourse extends Application {
modulePrefix = "discourse";
rootElement = "#main";

View file

@ -1,6 +1,10 @@
document.addEventListener("discourse-init", (e) => {
document.addEventListener("discourse-init", async (e) => {
performance.mark("discourse-init");
const config = e.detail;
const app = require(`${config.modulePrefix}/app`)["default"].create(config);
const { default: klass, loadThemes } = require(`${config.modulePrefix}/app`);
await loadThemes();
const app = klass.create(config);
app.start();
});

View file

@ -2,10 +2,13 @@ import loadEmberExam from "ember-exam/test-support/load";
import { setupEmberOnerrorValidation, start } from "ember-qunit";
import * as QUnit from "qunit";
import { setup } from "qunit-dom";
import { loadThemes } from "discourse/app";
import setupTests from "discourse/tests/setup-tests";
import config from "../config/environment";
document.addEventListener("discourse-init", () => {
document.addEventListener("discourse-init", async () => {
await loadThemes();
if (!window.EmberENV.TESTS_FILE_LOADED) {
throw new Error(
'The tests file was not loaded. Make sure your tests index.html includes "assets/tests.js".'
@ -21,9 +24,10 @@ document.addEventListener("discourse-init", () => {
const params = new URLSearchParams(window.location.search);
const target = params.get("target") || "core";
const testingTheme = !!document.querySelector("script[data-theme-id]");
const disableAutoStart = params.get("qunit_disable_auto_start") === "1";
const hasThemeJs = !!document.querySelector("script[data-theme-id]");
const hasThemeJs = !!document.querySelector(
"link[rel=modulepreload][data-theme-id]"
);
document.body.insertAdjacentHTML(
"afterbegin",
@ -36,7 +40,7 @@ document.addEventListener("discourse-init", () => {
`
);
const testingCore = !testingTheme && target === "core";
const testingCore = !hasThemeJs && target === "core";
if (testingCore) {
setupEmberOnerrorValidation();
}

View file

@ -0,0 +1,63 @@
export default function (babel) {
const { types: t } = babel;
const visitor = {
Program(path) {
const importDeclarations = [];
if (path.scope.bindings.themePrefix) {
const themePrefix = path.scope.bindings.themePrefix;
if (themePrefix.kind !== "module") {
throw new Error(
"`themePrefix` is already defined. Unable to add import."
);
} else if (themePrefix.path.parent.source.value !== "virtual:theme") {
throw new Error(
"`themePrefix` is already imported. Unable to add import from `virtual:theme`."
);
}
} else {
importDeclarations.push(
t.importSpecifier(
t.identifier("themePrefix"),
t.identifier("themePrefix")
)
);
}
if (path.scope.bindings.settings) {
const settings = path.scope.bindings.settings;
if (settings.kind !== "module") {
throw new Error(
"`settings` is already defined. Unable to add import."
);
} else if (settings.path.parent.source.value !== "virtual:theme") {
throw new Error(
"`settings` is already imported. Unable to add import from `virtual:theme`."
);
}
} else {
importDeclarations.push(
t.importSpecifier(t.identifier("settings"), t.identifier("settings"))
);
}
if (importDeclarations.length > 0) {
path.node.body.unshift(
t.importDeclaration(
importDeclarations,
t.stringLiteral("virtual:theme")
)
);
}
},
};
return {
pre(file) {
babel.traverse(file.ast, visitor, file.scope);
file.scope.crawl();
},
};
}

View file

@ -0,0 +1,74 @@
/* eslint-disable qunit/require-expect */
import { transformSync } from "@babel/core";
import { expect, test } from "vitest";
import AddThemeGlobals from "./add-theme-globals.js";
function compile(input) {
return transformSync(input, {
configFile: false,
plugins: [AddThemeGlobals],
}).code;
}
test("adds imports automatically", () => {
expect(
compile(`
console.log(settings, themePrefix);
`)
).toMatchInlineSnapshot(`
"import { themePrefix, settings } from "virtual:theme";
console.log(settings, themePrefix);"
`);
});
test("throws error if settings or themePrefix are defined locally", () => {
expect(() =>
compile(`
const settings = {};
console.log(settings);
`)
).toThrowErrorMatchingInlineSnapshot(
`[Error: unknown file: \`settings\` is already defined. Unable to add import.]`
);
expect(() =>
compile(`
const themePrefix = "foo";
console.log(themePrefix);
`)
).toThrowErrorMatchingInlineSnapshot(
`[Error: unknown file: \`themePrefix\` is already defined. Unable to add import.]`
);
});
test("works if settings and themePrefix are already imported", () => {
expect(
compile(`
import { themePrefix, settings } from "virtual:theme";
console.log(settings, themePrefix);
`)
).toMatchInlineSnapshot(`
"import { themePrefix, settings } from "virtual:theme";
console.log(settings, themePrefix);"
`);
});
test("throws error if settings or themePrefix are already imported from the wrong place", () => {
expect(() =>
compile(`
import { themePrefix } from "foo";
console.log(themePrefix);
`)
).toThrowErrorMatchingInlineSnapshot(
`[Error: unknown file: \`themePrefix\` is already imported. Unable to add import from \`virtual:theme\`.]`
);
expect(() =>
compile(`
import { settings } from "foo";
console.log(settings);
`)
).toThrowErrorMatchingInlineSnapshot(
`[Error: unknown file: \`settings\` is already imported. Unable to add import from \`virtual:theme\`.]`
);
});

View file

@ -0,0 +1,77 @@
import rollupVirtualImports from "./rollup-virtual-imports";
export default function (babel) {
const { types: t } = babel;
return {
visitor: {
ImportDeclaration(path) {
const moduleName = path.node.source.value;
if (
moduleName.startsWith(".") ||
rollupVirtualImports[moduleName] ||
moduleName.startsWith("discourse/theme-")
) {
return;
}
const namespaceImports = [];
const properties = path.node.specifiers
.map((specifier) => {
if (specifier.type === "ImportDefaultSpecifier") {
return t.objectProperty(
t.identifier("default"),
t.identifier(specifier.local.name)
);
} else if (specifier.type === "ImportNamespaceSpecifier") {
namespaceImports.push(t.identifier(specifier.local.name));
} else {
return t.objectProperty(
t.identifier(specifier.imported.name),
t.identifier(specifier.local.name)
);
}
})
.filter(Boolean);
const replacements = [];
const moduleBrokerLookup = t.awaitExpression(
t.callExpression(
t.memberExpression(
t.memberExpression(
t.identifier("window"),
t.identifier("moduleBroker")
),
t.identifier("lookup")
),
[t.stringLiteral(moduleName)]
)
);
if (properties.length) {
replacements.push(
t.variableDeclaration("const", [
t.variableDeclarator(
t.objectPattern(properties),
moduleBrokerLookup
),
])
);
}
if (namespaceImports.length) {
for (const namespaceImport of namespaceImports) {
replacements.push(
t.variableDeclaration("const", [
t.variableDeclarator(namespaceImport, moduleBrokerLookup),
])
);
}
}
path.replaceWithMultiple(replacements);
},
},
};
}

View file

@ -0,0 +1,44 @@
/* eslint-disable qunit/require-expect */
import { transformSync } from "@babel/core";
import { expect, test } from "vitest";
import BabelReplaceImports from "./babel-replace-imports.js";
function compile(input) {
return transformSync(input, {
configFile: false,
plugins: [BabelReplaceImports],
}).code;
}
test("replaces imports with moduleBroker calls", () => {
expect(
compile(`
import concatClass from "discourse/helpers/concat-class";
import { default as renamedDefaultImport, namedImport, otherNamedImport as renamedImport } from "discourse/module-1";
`)
).toMatchInlineSnapshot(`
"const {
default: concatClass
} = await window.moduleBroker.lookup("discourse/helpers/concat-class");
const {
default: renamedDefaultImport,
namedImport: namedImport,
otherNamedImport: renamedImport
} = await window.moduleBroker.lookup("discourse/module-1");"
`);
});
test("handles namespace imports", () => {
expect(
compile(`
import * as MyModule from "discourse/module-1";
import defaultExport, * as MyModule2 from "discourse/module-2";
`)
).toMatchInlineSnapshot(`
"const MyModule = await window.moduleBroker.lookup("discourse/module-1");
const {
default: defaultExport
} = await window.moduleBroker.lookup("discourse/module-2");
const MyModule2 = await window.moduleBroker.lookup("discourse/module-2");"
`);
});

View file

@ -52,10 +52,17 @@ esbuild
path: "path-browserify",
url: "./url-polyfill",
"source-map-js": "source-map-js",
assert: "./noop",
fs: "./noop",
stream: "readable-stream",
"abort-controller": "abort-controller/dist/abort-controller",
},
banner: {
js: `var process = { "env": { "EMBER_ENV": "production" }, "cwd": () => "/" };`,
},
define: {
"import.meta.url": "'http://example.com'",
},
external: [],
entryPoints: ["./transpiler.js"],
plugins: [wasmPlugin],

View file

@ -0,0 +1 @@
export default {};

View file

@ -6,9 +6,17 @@
"author": "Discourse",
"license": "GPL-2.0-only",
"keywords": [],
"scripts": {
"test": "vitest"
},
"dependencies": {
"@babel/core": "^7.27.4",
"@babel/preset-env": "^7.28.0",
"@babel/standalone": "^7.28.2",
"@csstools/postcss-light-dark-function": "^2.0.9",
"@rollup/browser": "^4.45.1",
"@rollup/plugin-babel": "^6.0.4",
"abort-controller": "^3.0.0",
"autoprefixer": "^10.4.21",
"babel-plugin-ember-template-compilation": "^2.4.1",
"content-tag": "^4.0.0",
@ -20,11 +28,14 @@
"ember-source": "~5.12.0",
"ember-this-fallback": "^0.4.0",
"fastestsmallesttextencoderdecoder": "^1.0.22",
"magic-string": "^0.30.17",
"memfs": "^4.17.2",
"path-browserify": "^1.0.1",
"polyfill-crypto.getrandomvalues": "^1.0.0",
"postcss": "^8.5.6",
"postcss-js": "^4.0.1",
"postcss-media-minmax": "^5.0.0",
"readable-stream": "^4.7.0",
"source-map-js": "^1.2.1",
"terser": "^5.43.1"
},
@ -33,5 +44,8 @@
"npm": "please-use-pnpm",
"yarn": "please-use-pnpm",
"pnpm": "^9"
},
"devDependencies": {
"vitest": "^3.2.4"
}
}

View file

@ -0,0 +1,81 @@
import MagicString from "magic-string";
import { basename, dirname, join } from "path";
export default function discourseColocation({ themeBase }) {
return {
name: "discourse-colocation",
async resolveId(source, context) {
let resolvedSource = source;
if (source.startsWith(".")) {
resolvedSource = join(dirname(context), source);
}
if (
!(
resolvedSource.startsWith(`${themeBase}discourse/components/`) ||
resolvedSource.startsWith(`${themeBase}admin/components/`)
)
) {
return;
}
if (source.endsWith(".js")) {
const hbs = await this.resolve(
`./${basename(source).replace(/.js$/, ".hbs")}`,
resolvedSource
);
const js = await this.resolve(source, context);
if (!js && hbs) {
return {
id: resolvedSource,
meta: {
"rollup-hbs-plugin": {
type: "template-only-component-js",
},
},
};
}
}
},
load(id) {
if (
this.getModuleInfo(id)?.meta?.["rollup-hbs-plugin"]?.type ===
"template-only-component-js"
) {
return {
code: `import templateOnly from '@ember/component/template-only';\nexport default templateOnly();\n`,
};
}
},
transform: {
async handler(input, id) {
if (
!id.startsWith(`${themeBase}discourse/components/`) &&
!id.startsWith(`${themeBase}admin/components/`)
) {
return;
}
if (id.endsWith(".js")) {
const relativeHbs = `./${basename(id).replace(/.js$/, ".hbs")}`;
const hbs = await this.resolve(relativeHbs, id);
if (hbs) {
const s = new MagicString(input);
s.prepend(
`import template from '${relativeHbs}';\nconst __COLOCATED_TEMPLATE__ = template;\n`
);
return {
code: s.toString(),
map: s.generateMap({ hires: true }),
};
}
}
},
},
};
}

View file

@ -0,0 +1,21 @@
export default function discourseExtensionSearch() {
return {
name: "discourse-extension-search",
async resolveId(source, context) {
if (source.match(/\.\w+$/)) {
// Already has an extension
return null;
}
for (const ext of ["", ".js", ".gjs", ".hbs"]) {
const resolved = await this.resolve(`${source}${ext}`, context);
if (resolved) {
return resolved;
}
}
return null;
},
};
}

View file

@ -0,0 +1,10 @@
export default function discourseExternalLoader() {
return {
name: "discourse-external-loader",
async resolveId(source) {
if (!source.startsWith(".")) {
return { id: source, external: true };
}
},
};
}

View file

@ -0,0 +1,26 @@
import { Preprocessor } from "../content-tag";
const preprocessor = new Preprocessor();
export default function discourseGjs() {
return {
name: "discourse-gjs",
transform: {
// Enforce running the gjs transform before any others like babel that expect valid JS
order: "pre",
handler(input, id) {
if (!id.endsWith(".gjs")) {
return null;
}
let { code, map } = preprocessor.process(input, {
filename: id,
});
return {
code,
map,
};
},
},
};
}

View file

@ -0,0 +1,19 @@
export default function discourseHbs() {
return {
name: "discourse-hbs",
transform: {
order: "pre",
handler(input, id) {
if (id.endsWith(".hbs")) {
return {
code: `
import { hbs } from 'ember-cli-htmlbars';
export default hbs(${JSON.stringify(input)}, { moduleName: ${JSON.stringify(id)} });
`,
map: null,
};
}
},
},
};
}

View file

@ -0,0 +1,16 @@
export default function discourseIndexSearch() {
return {
name: "discourse-index-search",
async resolveId(source, context) {
if (source.match(/\.\w+$/) || source.match(/\/index$/)) {
// Already has an extension or is an index
return null;
}
return (
(await this.resolve(source, context)) ||
(await this.resolve(`${source}/index`, context))
);
},
};
}

View file

@ -0,0 +1,31 @@
import { minify as terserMinify } from "terser";
export default function discourseTerser({ opts }) {
return {
name: "discourse-terser",
async renderChunk(code, chunk, outputOptions) {
if (!opts.minify) {
return;
}
// Based on https://github.com/ember-cli/ember-cli-terser/blob/28df3d90a5/index.js#L12-L26
const defaultOptions = {
sourceMap:
outputOptions.sourcemap === true ||
typeof outputOptions.sourcemap === "string",
compress: {
negate_iife: false,
sequences: 30,
drop_debugger: false,
},
output: {
semicolons: false,
},
};
defaultOptions.module = true;
return await terserMinify(code, defaultOptions);
},
};
}

View file

@ -0,0 +1,23 @@
import rollupVirtualImports from "../rollup-virtual-imports";
export default function discourseVirtualLoader({ themeBase, modules, opts }) {
return {
name: "discourse-virtual-loader",
resolveId(source) {
if (rollupVirtualImports[source]) {
return `${themeBase}${source}`;
}
},
load(id) {
if (!id.startsWith(themeBase)) {
return;
}
const fromBase = id.slice(themeBase.length);
if (rollupVirtualImports[fromBase]) {
return rollupVirtualImports[fromBase](modules, opts);
}
},
};
}

View file

@ -0,0 +1,87 @@
const SUPPORTED_FILE_EXTENSIONS = [".js", ".js.es6", ".hbs", ".gjs"];
export default {
"virtual:main": (tree, { themeId }) => {
let output = cleanMultiline(`
import "virtual:init-settings";
const themeCompatModules = {};
`);
let i = 1;
for (const moduleFilename of Object.keys(tree)) {
if (
!SUPPORTED_FILE_EXTENSIONS.some((ext) => moduleFilename.endsWith(ext))
) {
// Unsupported file type. Log a warning and skip
output += `console.warn("[THEME ${themeId}] Unsupported file type: ${moduleFilename}");\n`;
continue;
}
const filenameWithoutExtension = moduleFilename.replace(
/\.[^\.]+(\.es6)?$/,
""
);
let compatModuleName = filenameWithoutExtension;
if (moduleFilename.match(/(^|\/)connectors\//)) {
const isTemplate = moduleFilename.endsWith(".hbs");
const isInTemplatesDirectory =
moduleFilename.match(/(^|\/)templates\//);
if (isTemplate && !isInTemplatesDirectory) {
compatModuleName = compatModuleName.replace(
/(^|\/)connectors\//,
"$1templates/connectors/"
);
} else if (!isTemplate && isInTemplatesDirectory) {
compatModuleName = compatModuleName.replace(/^templates\//, "");
}
}
output += `import * as Mod${i} from "./${filenameWithoutExtension}";\n`;
output += `themeCompatModules["${compatModuleName}"] = Mod${i};\n\n`;
i += 1;
}
output += "export default themeCompatModules;\n";
return output;
},
"virtual:init-settings": (_, { themeId, settings }) => {
return (
`import { registerSettings } from "discourse/lib/theme-settings-store";\n\n` +
`registerSettings(${themeId}, ${JSON.stringify(settings, null, 2)});\n`
);
},
"virtual:theme": (_, { themeId }) => {
return cleanMultiline(`
import { getObjectForTheme } from "discourse/lib/theme-settings-store";
export const settings = getObjectForTheme(${themeId});
export function themePrefix(key) {
return \`theme_translations.${themeId}.\${key}\`;
}
`);
},
};
function cleanMultiline(str) {
const lines = str.split("\n");
if (lines.at(0).trim() === "") {
lines.shift();
}
if (lines.at(-1).trim() === "") {
lines.pop();
}
const minLeadingWhitspace = Math.min(
...lines.filter(Boolean).map((line) => line.match(/^\s*/)[0].length)
);
return lines.map((line) => line.slice(minLeadingWhitspace)).join("\n") + "\n";
}

View file

@ -4,10 +4,17 @@ import "core-js/actual/url";
import { TextDecoder, TextEncoder } from "fastestsmallesttextencoderdecoder";
import path from "path";
import getRandomValues from "polyfill-crypto.getrandomvalues";
import BindingsWasm from "./node_modules/@rollup/browser/dist/bindings_wasm_bg.wasm";
const CONSOLE_PREFIX = "[DiscourseJsProcessor] ";
globalThis.window = {};
globalThis.console = {
debug(...args) {
rails.logger.info(CONSOLE_PREFIX + args.join(" "));
},
info(...args) {
rails.logger.info(CONSOLE_PREFIX + args.join(" "));
},
log(...args) {
rails.logger.info(CONSOLE_PREFIX + args.join(" "));
},
@ -27,3 +34,21 @@ path.win32 = {
};
globalThis.crypto = { getRandomValues };
const oldInstantiate = WebAssembly.instantiate;
WebAssembly.instantiate = async function (bytes, bindings) {
if (bytes === BindingsWasm) {
const mod = new WebAssembly.Module(bytes);
const instance = new WebAssembly.Instance(mod, bindings);
return instance;
} else {
return oldInstantiate(...arguments);
}
};
globalThis.fetch = function (url) {
if (url.toString() === "http://example.com/bindings_wasm_bg.wasm") {
return new Promise((resolve) => resolve(BindingsWasm));
}
throw "fetch not implemented";
};

View file

@ -0,0 +1,123 @@
import BabelPresetEnv from "@babel/preset-env";
import { rollup } from "@rollup/browser";
import { babel, getBabelOutputPlugin } from "@rollup/plugin-babel";
import HTMLBarsInlinePrecompile from "babel-plugin-ember-template-compilation";
import DecoratorTransforms from "decorator-transforms";
import colocatedBabelPlugin from "ember-cli-htmlbars/lib/colocated-babel-plugin";
import { precompile } from "ember-source/dist/ember-template-compiler";
import EmberThisFallback from "ember-this-fallback";
import { memfs } from "memfs";
import { WidgetHbsCompiler } from "discourse-widget-hbs/lib/widget-hbs-compiler";
import { browsers } from "../discourse/config/targets";
import AddThemeGlobals from "./add-theme-globals";
import BabelReplaceImports from "./babel-replace-imports";
import discourseColocation from "./rollup-plugins/discourse-colocation";
import discourseExtensionSearch from "./rollup-plugins/discourse-extension-search";
import discourseExternalLoader from "./rollup-plugins/discourse-external-loader";
import discourseGjs from "./rollup-plugins/discourse-gjs";
import discourseHbs from "./rollup-plugins/discourse-hbs";
import discourseIndexSearch from "./rollup-plugins/discourse-index-search";
import discourseTerser from "./rollup-plugins/discourse-terser";
import discourseVirtualLoader from "./rollup-plugins/discourse-virtual-loader";
import buildEmberTemplateManipulatorPlugin from "./theme-hbs-ast-transforms";
let lastRollupResult;
let lastRollupError;
globalThis.rollup = function (modules, opts) {
const themeBase = `theme-${opts.themeId}/`;
const { vol } = memfs(modules, themeBase);
const resultPromise = rollup({
input: "virtual:main",
logLevel: "info",
fs: vol.promises,
onLog(level, message) {
// eslint-disable-next-line no-console
console.info(level, message);
},
plugins: [
discourseExtensionSearch(),
discourseIndexSearch(),
discourseVirtualLoader({
themeBase,
modules,
opts,
}),
discourseExternalLoader(),
discourseColocation({ themeBase }),
getBabelOutputPlugin({
plugins: [BabelReplaceImports],
}),
babel({
extensions: [".js", ".gjs", ".hbs"],
babelHelpers: "bundled",
plugins: [
[DecoratorTransforms, { runEarly: true }],
AddThemeGlobals,
colocatedBabelPlugin,
WidgetHbsCompiler,
[
HTMLBarsInlinePrecompile,
{
compiler: { precompile },
enableLegacyModules: [
"ember-cli-htmlbars",
"ember-cli-htmlbars-inline-precompile",
"htmlbars-inline-precompile",
],
transforms: [
EmberThisFallback._buildPlugin({
enableLogging: false,
isTheme: true,
}).plugin,
buildEmberTemplateManipulatorPlugin(opts.themeId),
],
},
],
],
presets: [
[
BabelPresetEnv,
{
modules: false,
targets: { browsers },
},
],
],
}),
discourseHbs(),
discourseGjs(),
discourseTerser({ opts }),
],
});
resultPromise
.then((bundle) => {
return bundle.generate({
format: "es",
sourcemap: "hidden",
});
})
.then(({ output }) => {
lastRollupResult = {
code: output[0].code,
map: JSON.stringify(output[0].map),
};
})
.catch((error) => {
lastRollupError = error;
});
};
globalThis.getRollupResult = function () {
const error = lastRollupError;
const result = lastRollupResult;
lastRollupError = lastRollupResult = null;
if (error) {
throw error;
}
return result;
};

View file

@ -1,7 +1,6 @@
// This is executed in mini_racer to provide the JS logic for lib/discourse_js_processor.rb
import "./shims";
import "./postcss";
import "./theme-rollup";
import { transform as babelTransform } from "@babel/standalone";
import HTMLBarsInlinePrecompile from "babel-plugin-ember-template-compilation";
import DecoratorTransforms from "decorator-transforms";

View file

@ -11,6 +11,6 @@ module QunitHelper
"#{Discourse.base_path}" \
"/theme-javascripts/tests/#{theme.id}-#{digest}.js" \
"?__ws=#{Discourse.current_hostname}"
"<script defer src='#{src}' data-theme-id='#{theme.id}' nonce='#{csp_nonce_placeholder}'></script>".html_safe
"<link rel='modulepreload' href='#{src}' data-theme-id='#{theme.id}' nonce='#{ThemeField::CSP_NONCE_PLACEHOLDER}' />".html_safe
end
end

View file

@ -6,7 +6,7 @@ require "json_schemer"
class Theme < ActiveRecord::Base
include GlobalPath
BASE_COMPILER_VERSION = 94
BASE_COMPILER_VERSION = 97
CORE_THEMES = { "foundation" => -1, "horizon" => -2 }
EDITABLE_SYSTEM_ATTRIBUTES = %w[
child_theme_ids
@ -555,7 +555,7 @@ class Theme < ActiveRecord::Base
.compact
caches.map { |c| <<~HTML.html_safe }.join("\n")
<script defer src="#{c.url}" data-theme-id="#{c.theme_id}" nonce="#{ThemeField::CSP_NONCE_PLACEHOLDER}"></script>
<link rel="modulepreload" href="#{c.url}" data-theme-id="#{c.theme_id}" nonce="#{ThemeField::CSP_NONCE_PLACEHOLDER}" />
HTML
end
when :translations

View file

@ -260,8 +260,8 @@ class ThemeField < ActiveRecord::Base
javascript_cache.source_map = js_compiler.source_map
javascript_cache.save!
doc.add_child(<<~HTML.html_safe) if javascript_cache.content.present?
<script defer src='#{javascript_cache.url}' data-theme-id='#{theme_id}' nonce="#{CSP_NONCE_PLACEHOLDER}"></script>
doc.add_child(<<~HTML.html_safe)
<link rel="modulepreload" href="#{javascript_cache.url}" data-theme-id="#{theme_id}" nonce="#{CSP_NONCE_PLACEHOLDER}" />
HTML
else
javascript_cache&.destroy!

View file

@ -1,6 +1,4 @@
# frozen_string_literal: true
require "execjs"
require "mini_racer"
class DiscourseJsProcessor
class TranspileError < StandardError
@ -27,7 +25,6 @@ class DiscourseJsProcessor
end
def self.build_theme_transpiler
FileUtils.rm_rf("tmp/theme-transpiler") # cleanup old files - remove after Jan 2025
Discourse::Utils.execute_command(
"pnpm",
"-C=app/assets/javascripts/theme-transpiler",
@ -57,6 +54,8 @@ class DiscourseJsProcessor
@processor_mutex.synchronize { build_theme_transpiler }
end
# source = File.read("app/assets/javascripts/theme-transpiler/theme-transpiler.js")
ctx.eval(source, filename: "theme-transpiler.js")
ctx
@ -156,6 +155,10 @@ class DiscourseJsProcessor
self.class.v8_call("minify", tree, opts, fetch_result_call: "getMinifyResult")
end
def rollup(tree, opts)
self.class.v8_call("rollup", tree, opts, fetch_result_call: "getRollupResult")
end
def post_css(css:, map:, source_map_file:)
self.class.v8_call(
"postCss",

View file

@ -1,6 +1,5 @@
# frozen_string_literal: true
require "mini_racer"
require "nokogiri"
require "erb"

View file

@ -1,9 +1,6 @@
# frozen_string_literal: true
class ThemeJavascriptCompiler
COLOCATED_CONNECTOR_REGEX =
%r{\A(?<prefix>.*/?)connectors/(?<outlet>[^/]+)/(?<name>[^/\.]+)\.(?<extension>.+)\z}
class CompileError < StandardError
end
@ -31,16 +28,11 @@ class ThemeJavascriptCompiler
@compiled = true
@input_tree.freeze
output_tree = compile_tree!
output =
if !output_tree.present?
{ "code" => "" }
elsif @@terser_disabled || !@minify
{ "code" => output_tree.map { |filename, source| source }.join("") }
else
DiscourseJsProcessor::Transpiler.new.terser(output_tree, terser_config)
end
DiscourseJsProcessor::Transpiler.new.rollup(
@input_tree.transform_keys { |k| k.sub(/\.js\.es6$/, ".js") },
{ themeId: @theme_id, settings: @settings, minify: @minify && !@@terser_disabled },
)
@content = output["code"]
@source_map = output["map"]
@ -48,28 +40,10 @@ class ThemeJavascriptCompiler
[@content, @source_map]
rescue DiscourseJsProcessor::TranspileError => e
message = "[THEME #{@theme_id} '#{@theme_name}'] Compile error: #{e.message}"
@content = "console.error(#{message.to_json});\n"
@content = "throw new Error(#{message.to_json});\n"
[@content, @source_map]
end
def terser_config
# Based on https://github.com/ember-cli/ember-cli-terser/blob/28df3d90a5/index.js#L12-L26
{
sourceMap: {
includeSources: true,
root: "theme-#{@theme_id}/",
},
compress: {
negate_iife: false,
sequences: 30,
drop_debugger: false,
},
output: {
semicolons: false,
},
}
end
def content
compile!
@content
@ -83,164 +57,4 @@ class ThemeJavascriptCompiler
def append_tree(tree)
@input_tree.merge!(tree)
end
private
def compile_tree!
input_tree = @input_tree.dup
# Replace legacy extensions
input_tree.transform_keys! do |filename|
if filename.ends_with? ".js.es6"
filename.sub(/\.js\.es6\z/, ".js")
else
filename
end
end
# Some themes are colocating connector JS under `/connectors`. Move template to /templates to avoid module name clash
input_tree.transform_keys! do |filename|
match = COLOCATED_CONNECTOR_REGEX.match(filename)
next filename if !match
is_template = match[:extension] == "hbs"
is_in_templates_directory = match[:prefix].split("/").last == "templates"
if is_template && !is_in_templates_directory
"#{match[:prefix]}templates/connectors/#{match[:outlet]}/#{match[:name]}.#{match[:extension]}"
elsif !is_template && is_in_templates_directory
"#{match[:prefix].chomp("templates/")}connectors/#{match[:outlet]}/#{match[:name]}.#{match[:extension]}"
else
filename
end
end
# Handle colocated components
input_tree.dup.each_pair do |filename, content|
is_component_template =
filename.end_with?(".hbs") &&
filename.start_with?("discourse/components/", "admin/components/")
next if !is_component_template
template_contents = content
hbs_invocation_options = { moduleName: filename, parseOptions: { srcName: filename } }
hbs_invocation = "hbs(#{template_contents.to_json}, #{hbs_invocation_options.to_json})"
prefix = <<~JS
import { hbs } from 'ember-cli-htmlbars';
const __COLOCATED_TEMPLATE__ = #{hbs_invocation};
JS
js_filename = filename.sub(/\.hbs\z/, ".js")
js_contents = input_tree[js_filename] # May be nil for template-only component
if js_contents && !js_contents.include?("export default")
message =
"#{filename} does not contain a `default export`. Did you forget to export the component class?"
js_contents += "throw new Error(#{message.to_json});"
end
if js_contents.nil?
# No backing class, use template-only
js_contents = <<~JS
import templateOnly from '@ember/component/template-only';
export default templateOnly();
JS
end
js_contents = prefix + js_contents
input_tree[js_filename] = js_contents
input_tree.delete(filename)
end
output_tree = {}
register_settings(@settings, tree: output_tree)
# Transpile and write to output
input_tree.each_pair do |filename, content|
module_name, extension = filename.split(".", 2)
if extension == "js" || extension == "gjs"
append_module(content, module_name, extension, tree: output_tree)
elsif extension == "hbs"
append_ember_template(module_name, content, tree: output_tree)
else
append_js_error(
filename,
"unknown file extension '#{extension}' (#{filename})",
tree: output_tree,
)
end
rescue CompileError => e
append_js_error filename, "#{e.message} (#{filename})", tree: output_tree
end
output_tree
end
def append_ember_template(name, hbs_template, tree:)
module_name = name
module_name = "/#{module_name}" if !module_name.start_with?("/")
module_name = "discourse/theme-#{@theme_id}#{module_name}"
# Mimics the ember-cli implementation
# https://github.com/ember-cli/ember-cli-htmlbars/blob/d5aa14b3/lib/template-compiler-plugin.js#L18-L26
script = <<~JS
import { hbs } from 'ember-cli-htmlbars';
export default hbs(#{hbs_template.to_json}, { moduleName: #{module_name.to_json} });
JS
template_module = DiscourseJsProcessor.transpile(script, "", module_name, theme_id: @theme_id)
tree["#{name}.js"] = <<~JS
if ('define' in window) {
#{template_module}
}
JS
rescue MiniRacer::RuntimeError, DiscourseJsProcessor::TranspileError => ex
raise CompileError.new ex.message
end
def append_raw_script(filename, script, tree:)
tree[filename] = script + "\n"
end
def append_module(script, name, extension, include_variables: true, tree:)
original_filename = name
name = "discourse/theme-#{@theme_id}/#{name}"
script = "#{theme_settings}#{script}" if include_variables
transpiler = DiscourseJsProcessor::Transpiler.new
tree["#{original_filename}.#{extension}"] = <<~JS
if ('define' in window) {
#{transpiler.perform(script, "", name, theme_id: @theme_id, extension: extension).strip}
}
JS
rescue MiniRacer::RuntimeError, DiscourseJsProcessor::TranspileError => ex
raise CompileError.new ex.message
end
def append_js_error(filename, message, tree:)
message = "[THEME #{@theme_id} '#{@theme_name}'] Compile error: #{message}"
append_raw_script filename, "console.error(#{message.to_json});", tree:
end
def register_settings(settings_hash, tree:)
append_raw_script("settings.js", <<~JS, tree:)
(function() {
if ('require' in window) {
require("discourse/lib/theme-settings-store").registerSettings(#{@theme_id}, #{settings_hash.to_json});
}
})();
JS
end
def theme_settings
<<~JS
const settings = require("discourse/lib/theme-settings-store")
.getObjectForTheme(#{@theme_id});
const themePrefix = (key) => `theme_translations.#{@theme_id}.${key}`;
JS
end
end

View file

@ -1,6 +1,5 @@
# frozen_string_literal: true
require "mini_racer"
require "json"
module DiscourseAi

1432
pnpm-lock.yaml generated

File diff suppressed because it is too large Load diff

View file

@ -191,4 +191,215 @@ RSpec.describe DiscourseJsProcessor do
expect(map["sourcesContent"]).to contain_exactly(*sources.values)
end
end
describe "Transpiler#rollup" do
it "can rollup code" do
sources = { "discourse/initializers/hello.gjs" => <<~JS }
someDecorator = () => {}
export default class MyClass {
@someDecorator
myMethod() {
console.log("hello world");
}
<template>
<div>template content</div>
</template>
}
JS
result = DiscourseJsProcessor::Transpiler.new.rollup(sources, {})
code = result["code"]
expect(code).to include('"hello world"')
expect(code).to include("dt7948") # Decorator transform
expect(result["map"]).not_to be_nil
end
it "supports decorators and class properties without error" do
script = <<~JS.chomp
export default class MyClass {
classProperty = 1;
#privateProperty = 1;
#privateMethod() {
console.log("hello world");
}
@decorated
myMethod(){
}
}
JS
result =
DiscourseJsProcessor::Transpiler.new.rollup(
{ "discourse/initializers/foo.js" => script },
{},
)
expect(result["code"]).to include("() => dt7948.n")
end
it "supports object literal decorators without errors" do
script = <<~JS.chomp
export default {
@decorated foo: "bar",
@decorated
myMethod() {
console.log("hello world");
}
}
JS
result =
DiscourseJsProcessor::Transpiler.new.rollup(
{ "discourse/initializers/foo.js" => script },
{},
)
expect(result["code"]).to include("dt7948")
end
it "can use themePrefix in a template" do
script = <<~JS.chomp
themePrefix();
export default class Foo {
<template>{{themePrefix "bar"}}</template>
}
JS
result =
DiscourseJsProcessor::Transpiler.new.rollup(
{ "discourse/initializers/foo.gjs" => script },
{ themeId: 22 },
)
expect(result["code"]).to include(
'window.moduleBroker.lookup("discourse/lib/theme-settings-store")',
)
end
it "can use themePrefix not in a template" do
script = <<~JS.chomp
export default function foo() {
return themePrefix("bar");
}
JS
result =
DiscourseJsProcessor::Transpiler.new.rollup(
{ "discourse/initializers/foo.js" => script },
{ themeId: 22 },
)
expect(result["code"]).to include(
'window.moduleBroker.lookup("discourse/lib/theme-settings-store")',
)
end
end
it "can compile hbs" do
template = <<~HBS.chomp
{{log "hello world"}}
HBS
result =
DiscourseJsProcessor::Transpiler.new.rollup(
{ "discourse/connectors/outlet-name/foo.hbs" => template },
{ themeId: 22 },
)
expect(result["code"]).to include("createTemplateFactory")
end
it "handles colocation" do
js = <<~JS.chomp
import Component from "@glimmer/component";
export default class MyComponent extends Component {}
JS
template = <<~HBS.chomp
{{log "hello world"}}
HBS
onlyTemplate = <<~HBS.chomp
{{log "hello galaxy"}}
HBS
result =
DiscourseJsProcessor::Transpiler.new.rollup(
{
"discourse/components/foo.js" => js,
"discourse/components/foo.hbs" => template,
"discourse/components/bar.hbs" => onlyTemplate,
},
{ themeId: 22 },
)
expect(result["code"]).to include("setComponentTemplate")
expect(result["code"]).to include(
"bar = setComponentTemplate(__COLOCATED_TEMPLATE__, templateOnly());",
)
end
it "handles relative imports from one module to another" do
mod_1 = <<~JS.chomp
export default "test";
JS
mod_2 = <<~JS.chomp
import MyComponent from "../components/my-component";
console.log(MyComponent);
JS
result =
DiscourseJsProcessor::Transpiler.new.rollup(
{
"discourse/components/my-component.js" => mod_1,
"discourse/components/other-component.js" => mod_2,
},
{ themeId: 22 },
)
expect(result["code"]).not_to include("../components/my-component")
end
it "handles relative import of index file" do
mod_1 = <<~JS.chomp
import MyComponent from "./other-component";
console.log(MyComponent);
JS
mod_2 = <<~JS.chomp
export default "test";
JS
result =
DiscourseJsProcessor::Transpiler.new.rollup(
{
"discourse/components/my-component.js" => mod_1,
"discourse/components/other-component/index.js" => mod_2,
},
{ themeId: 22 },
)
expect(result["code"]).not_to include("../components/my-component")
end
it "handles relative import of gjs index file" do
mod_1 = <<~JS.chomp
import MyComponent from "./other-component";
console.log(MyComponent);
JS
mod_2 = <<~JS.chomp
export default "test";
JS
result =
DiscourseJsProcessor::Transpiler.new.rollup(
{
"discourse/components/my-component.gjs" => mod_1,
"discourse/components/other-component/index.gjs" => mod_2,
},
{ themeId: 22 },
)
expect(result["code"]).not_to include("../components/my-component")
end
end

View file

@ -633,14 +633,17 @@ RSpec.describe Discourse do
head_tag_script =
Nokogiri::HTML5
.fragment(Theme.lookup_field(theme.id, :desktop, "head_tag"))
.css("script")
.css("link[rel=modulepreload]")
.first
head_tag_js = JavascriptCache.find_by(digest: head_tag_script[:src][/\h{40}/]).content
head_tag_js = JavascriptCache.find_by(digest: head_tag_script[:href][/\h{40}/]).content
expect(head_tag_js).to include(old_upload_url)
js_file_script =
Nokogiri::HTML5.fragment(Theme.lookup_field(theme.id, :extra_js, nil)).css("script").first
file_js = JavascriptCache.find_by(digest: js_file_script[:src][/\h{40}/]).content
Nokogiri::HTML5
.fragment(Theme.lookup_field(theme.id, :extra_js, nil))
.css("link[rel=modulepreload]")
.first
file_js = JavascriptCache.find_by(digest: js_file_script[:href][/\h{40}/]).content
expect(file_js).to include(old_upload_url)
css_link_tag =
@ -659,14 +662,17 @@ RSpec.describe Discourse do
head_tag_script =
Nokogiri::HTML5
.fragment(Theme.lookup_field(theme.id, :desktop, "head_tag"))
.css("script")
.css("link[rel=modulepreload]")
.first
head_tag_js = JavascriptCache.find_by(digest: head_tag_script[:src][/\h{40}/]).content
head_tag_js = JavascriptCache.find_by(digest: head_tag_script[:href][/\h{40}/]).content
expect(head_tag_js).to include(old_upload_url)
js_file_script =
Nokogiri::HTML5.fragment(Theme.lookup_field(theme.id, :extra_js, nil)).css("script").first
file_js = JavascriptCache.find_by(digest: js_file_script[:src][/\h{40}/]).content
Nokogiri::HTML5
.fragment(Theme.lookup_field(theme.id, :extra_js, nil))
.css("link[rel=modulepreload]")
.first
file_js = JavascriptCache.find_by(digest: js_file_script[:href][/\h{40}/]).content
expect(file_js).to include(old_upload_url)
css_link_tag =
@ -684,14 +690,17 @@ RSpec.describe Discourse do
head_tag_script =
Nokogiri::HTML5
.fragment(Theme.lookup_field(theme.id, :desktop, "head_tag"))
.css("script")
.css("link[rel=modulepreload]")
.first
head_tag_js = JavascriptCache.find_by(digest: head_tag_script[:src][/\h{40}/]).content
head_tag_js = JavascriptCache.find_by(digest: head_tag_script[:href][/\h{40}/]).content
expect(head_tag_js).to include(new_upload_url)
js_file_script =
Nokogiri::HTML5.fragment(Theme.lookup_field(theme.id, :extra_js, nil)).css("script").first
file_js = JavascriptCache.find_by(digest: js_file_script[:src][/\h{40}/]).content
Nokogiri::HTML5
.fragment(Theme.lookup_field(theme.id, :extra_js, nil))
.css("link[rel=modulepreload]")
.first
file_js = JavascriptCache.find_by(digest: js_file_script[:href][/\h{40}/]).content
expect(file_js).to include(new_upload_url)
css_link_tag =

View file

@ -1,7 +1,5 @@
# frozen_string_literal: true
require "mini_racer"
RSpec.describe JsLocaleHelper do
let(:v8_ctx) do
discourse_node_modules = "#{Rails.root}/app/assets/javascripts/discourse/node_modules"

View file

@ -3,54 +3,92 @@
RSpec.describe ThemeJavascriptCompiler do
let(:compiler) { ThemeJavascriptCompiler.new(1, "marks", minify: false) }
describe "#append_ember_template" do
it "maintains module names so that discourse-boot.js can correct them" do
compiler.append_tree({ "connectors/blah-1.hbs" => "{{var}}" })
compiler.append_tree({ "connectors/blah-2.hbs" => "{{var}}" })
compiler.append_tree({ "javascripts/connectors/blah-3.hbs" => "{{var}}" })
expect(compiler.content.to_s).to include(
"themeCompatModules[\"templates/connectors/blah-1\"]",
)
expect(compiler.content.to_s).to include(
"themeCompatModules[\"templates/connectors/blah-2\"]",
)
expect(compiler.content.to_s).to include(
"themeCompatModules[\"javascripts/templates/connectors/blah-3\"]",
)
end
end
describe "connector module name handling" do
it "separates colocated connectors to avoid module name clash" do
# Colocated under `/connectors`
compiler = ThemeJavascriptCompiler.new(1, "marks")
compiler = ThemeJavascriptCompiler.new(1, "marks", minify: false)
compiler.append_tree(
{
"connectors/outlet/blah-1.hbs" => "{{var}}",
"connectors/outlet/blah-1.js" => "console.log('test')",
"connectors/outlet/blah-1.js" => "export default {};",
},
)
expect(compiler.content).to include("discourse/theme-1/connectors/outlet/blah-1")
expect(compiler.content).to include("discourse/theme-1/templates/connectors/outlet/blah-1")
expect(JSON.parse(compiler.source_map)["sources"]).to contain_exactly(
"connectors/outlet/blah-1.js",
"templates/connectors/outlet/blah-1.js",
"settings.js",
expect(compiler.content.to_s).to include(
'themeCompatModules["connectors/outlet/blah-1"]',
).once
expect(compiler.content.to_s).to include("templates/connectors/outlet/blah-1")
expect(compiler.content.to_s).not_to include("setComponentTemplate")
expect(JSON.parse(compiler.source_map)["sources"]).to include(
"theme-1/connectors/outlet/blah-1.js",
)
# Colocated under `/templates/connectors`
compiler = ThemeJavascriptCompiler.new(1, "marks")
compiler = ThemeJavascriptCompiler.new(1, "marks", minify: false)
compiler.append_tree(
{
"templates/connectors/outlet/blah-1.hbs" => "{{var}}",
"templates/connectors/outlet/blah-1.js" => "console.log('test')",
"templates/connectors/outlet/blah-1.js" => "export default {};",
},
)
expect(compiler.content).to include("discourse/theme-1/connectors/outlet/blah-1")
expect(compiler.content).to include("discourse/theme-1/templates/connectors/outlet/blah-1")
expect(JSON.parse(compiler.source_map)["sources"]).to contain_exactly(
"connectors/outlet/blah-1.js",
"templates/connectors/outlet/blah-1.js",
"settings.js",
expect(compiler.content.to_s).to include(
'themeCompatModules["connectors/outlet/blah-1"]',
).once
expect(compiler.content.to_s).to include("templates/connectors/outlet/blah-1")
expect(compiler.content.to_s).not_to include("setComponentTemplate")
expect(JSON.parse(compiler.source_map)["sources"]).to include(
"theme-1/templates/connectors/outlet/blah-1.js",
)
# Not colocated
compiler = ThemeJavascriptCompiler.new(1, "marks")
compiler = ThemeJavascriptCompiler.new(1, "marks", minify: false)
compiler.append_tree(
{
"templates/connectors/outlet/blah-1.hbs" => "{{var}}",
"connectors/outlet/blah-1.js" => "console.log('test')",
"connectors/outlet/blah-1.js" => "export default {};",
},
)
expect(compiler.content).to include("discourse/theme-1/connectors/outlet/blah-1")
expect(compiler.content).to include("discourse/theme-1/templates/connectors/outlet/blah-1")
expect(JSON.parse(compiler.source_map)["sources"]).to contain_exactly(
"connectors/outlet/blah-1.js",
"templates/connectors/outlet/blah-1.js",
"settings.js",
expect(compiler.content.to_s).to include(
'themeCompatModules["connectors/outlet/blah-1"]',
).once
expect(compiler.content.to_s).to include("templates/connectors/outlet/blah-1")
expect(compiler.content.to_s).not_to include("setComponentTemplate")
expect(JSON.parse(compiler.source_map)["sources"]).to include(
"theme-1/connectors/outlet/blah-1.js",
)
# colocation in discourse directory
compiler = ThemeJavascriptCompiler.new(1, "marks", minify: false)
compiler.append_tree(
{
"discourse/connectors/outlet/blah-1.hbs" => "{{var}}",
"discourse/connectors/outlet/blah-1.js" => "export default {};",
},
)
expect(compiler.content.to_s).to include(
'themeCompatModules["discourse/connectors/outlet/blah-1"]',
).once
expect(compiler.content.to_s).to include("discourse/templates/connectors/outlet/blah-1")
expect(compiler.content.to_s).not_to include("setComponentTemplate")
expect(JSON.parse(compiler.source_map)["sources"]).to include(
"theme-1/discourse/connectors/outlet/blah-1.js",
)
end
end
@ -58,7 +96,7 @@ RSpec.describe ThemeJavascriptCompiler do
describe "error handling" do
it "handles syntax errors in ember templates" do
compiler.append_tree({ "sometemplate.hbs" => "{{invalidtemplate" })
expect(compiler.content).to include('console.error("[THEME 1')
expect(compiler.content).to include("Parse error on line 1")
end
end
@ -66,6 +104,16 @@ RSpec.describe ThemeJavascriptCompiler do
it "can handle multiple modules" do
compiler.append_tree(
{
"discourse/initializers/my-initializer.js" => <<~JS,
import MyComponent from "../components/mycomponent";
export default {
name: "my-initializer",
initialize() {
console.log("my-initializer", MyComponent);
},
};
JS
"discourse/components/mycomponent.js" => <<~JS,
import Component from "@glimmer/component";
export default class MyComponent extends Component {}
@ -73,11 +121,9 @@ RSpec.describe ThemeJavascriptCompiler do
"discourse/templates/components/mycomponent.hbs" => "{{my-component-template}}",
},
)
expect(compiler.content).to include('themeCompatModules["discourse/components/mycomponent"]')
expect(compiler.content).to include(
'define("discourse/theme-1/discourse/components/mycomponent"',
)
expect(compiler.content).to include(
'define("discourse/theme-1/discourse/templates/components/mycomponent"',
'themeCompatModules["discourse/templates/components/mycomponent"]',
)
end
@ -110,28 +156,14 @@ RSpec.describe ThemeJavascriptCompiler do
end
it "applies theme AST transforms to colocated components" do
compiler = ThemeJavascriptCompiler.new(12_345_678_910, "my theme name")
compiler = ThemeJavascriptCompiler.new(12_345_678_910, "my theme name", minify: false)
compiler.append_tree(
{ "discourse/components/mycomponent.hbs" => '{{theme-i18n "my_translation_key"}}' },
)
template_compiled_line = compiler.content.lines.find { |l| l.include?("block:") }
template_compiled_line = compiler.content.lines.find { |l| l.include?('"block":') }
expect(template_compiled_line).to include("12345678910")
end
it "prints error when default export missing" do
compiler.append_tree(
{
"discourse/components/mycomponent.js" => <<~JS,
import Component from "@glimmer/component";
class MyComponent extends Component {}
JS
"discourse/components/mycomponent.hbs" => "{{my-component-template}}",
},
)
expect(compiler.content).to include("__COLOCATED_TEMPLATE__ =")
expect(compiler.content).to include("throw new Error")
end
it "handles template-only components" do
compiler.append_tree(
{ "discourse/components/mycomponent.hbs" => "{{my-component-template}}" },
@ -143,29 +175,33 @@ RSpec.describe ThemeJavascriptCompiler do
end
describe "terser compilation" do
let(:compiler) { ThemeJavascriptCompiler.new(1, "marks", minify: true) }
let(:compiler) { ThemeJavascriptCompiler.new(1, "marks", {}, minify: true) }
it "applies terser and provides sourcemaps" do
sources = {
"multiply.js" => "let multiply = (firstValue, secondValue) => firstValue * secondValue;",
"add.js" => "let add = (firstValue, secondValue) => firstValue + secondValue;",
"multiply.js" =>
"export const multiply = (firstValue, secondValue) => firstValue * secondValue;",
"add.js" => "export const add = (firstValue, secondValue) => firstValue + secondValue;",
}
compiler.append_tree(sources)
expect(compiler.content).to include("multiply")
expect(compiler.content).to include("add")
expect(compiler.content).not_to include("firstValue")
expect(compiler.content).not_to include("secondValue")
map = JSON.parse(compiler.source_map)
expect(map["sources"]).to contain_exactly(*sources.keys, "settings.js")
expect(map["sourcesContent"].to_s).to include("let multiply")
expect(map["sourcesContent"].to_s).to include("let add")
expect(map["sourceRoot"]).to eq("theme-1/")
expect(map["sources"]).to include("theme-1/multiply.js", "theme-1/add.js")
expect(map["sourcesContent"].to_s).to include("const multiply")
expect(map["sourcesContent"].to_s).to include("const add")
expect(map["sourcesContent"].to_s).to include("firstValue")
expect(map["sourcesContent"].to_s).to include("secondValue")
end
it "handles invalid JS" do
compiler.append_tree({ "filename.js" => "if(someCondition" })
expect(compiler.content).to include("console.error('[THEME 1")
expect(compiler.content).to include('throw new Error("[THEME 1')
expect(compiler.content).to include("Unexpected token")
end
end
@ -185,7 +221,7 @@ RSpec.describe ThemeJavascriptCompiler do
)
expect(compiler.content).to include("ember-this-fallback")
expect(compiler.content).to include(
"The `value` property path was used in the `discourse/components/my-component.hbs` template without using `this`. This fallback behavior has been deprecated, all properties must be looked up on `this` when used in the template: {{this.value}}",
"The `value` property path was used in the `theme-1/discourse/components/my-component.hbs` template without using `this`. This fallback behavior has been deprecated, all properties must be looked up on `this` when used in the template: {{this.value}}",
)
end
end
@ -205,7 +241,7 @@ RSpec.describe ThemeJavascriptCompiler do
JS
expect(compiler.content).to include(
"define(\"discourse/theme-1/discourse/components/my-component\", [\"exports\",",
"themeCompatModules[\"discourse/components/my-component\"]",
)
expect(compiler.content).to include('value = "foo";')
expect(compiler.content).to include("setComponentTemplate")

View file

@ -62,19 +62,19 @@ RSpec.describe ThemeField do
it "extracts inline javascript to an external file" do
html = <<~HTML
<script type="text/discourse-plugin" version="0.8">
var a = "inline discourse plugin";
console.log("inline discourse plugin");
</script>
<script type="text/template" data-template="custom-template">
<div>custom script type</div>
</script>
<script>
var b = "inline raw script";
console.log("inline raw script");
</script>
<script type="texT/jAvasCripT">
var c = "text/javascript";
console.log("text/javascript");
</script>
<script type="application/javascript">
var d = "application/javascript";
console.log("application/javascript");
</script>
<script src="/external-script.js"></script>
HTML
@ -84,22 +84,26 @@ RSpec.describe ThemeField do
baked_doc = Nokogiri::HTML5.fragment(theme_field.value_baked)
extracted_scripts = baked_doc.css("script[src^='/theme-javascripts/']")
expect(extracted_scripts.length).to eq(4)
simple_extracted_scripts = baked_doc.css("script[src^='/theme-javascripts/']")
expect(simple_extracted_scripts.length).to eq(3)
extracted_module_preloads =
baked_doc.css("link[rel=modulepreload][href^='/theme-javascripts/']")
expect(extracted_module_preloads.length).to eq(1)
expect(theme_field.javascript_cache.content).to include("inline discourse plugin")
raw_js_cache_contents = theme_field.raw_javascript_caches.map(&:content)
expect(raw_js_cache_contents).to contain_exactly(
'var b = "inline raw script";',
'var c = "text/javascript";',
'var d = "application/javascript";',
'console.log("inline raw script");',
'console.log("text/javascript");',
'console.log("application/javascript");',
)
expect(baked_doc.css("script[type='text/template']").length).to eq(1)
end
it "correctly extracts and generates errors for transpiled js" do
it "correctly logs errors for transpiled js" do
html = <<HTML
<script type="text/discourse-plugin" version="0.8">
badJavaScript(;
@ -108,15 +112,13 @@ HTML
field = ThemeField.create!(theme_id: -1, target_id: 0, name: "header", value: html)
field.ensure_baked!
expect(field.error).to eq(nil)
expect(field.value_baked).to include(
"<script defer=\"\" src=\"#{field.javascript_cache.url}\" data-theme-id=\"-1\" nonce=\"#{ThemeField::CSP_NONCE_PLACEHOLDER}\"></script>",
"<link rel=\"modulepreload\" href=\"#{field.javascript_cache.url}\" data-theme-id=\"-1\" nonce=\"#{ThemeField::CSP_NONCE_PLACEHOLDER}\">",
)
expect(field.javascript_cache.content).to include("[THEME -1 'Foundation'] Compile error")
field.update!(value: "")
field.ensure_baked!
expect(field.error).to eq(nil)
end
it "correctly extracts and generates errors for raw transpiled js" do
@ -160,13 +162,13 @@ HTML
javascript_cache = theme_field.javascript_cache
expect(theme_field.value_baked).to include(
"<script defer=\"\" src=\"#{javascript_cache.url}\" data-theme-id=\"-1\" nonce=\"#{ThemeField::CSP_NONCE_PLACEHOLDER}\"></script>",
"<link rel=\"modulepreload\" href=\"#{javascript_cache.url}\" data-theme-id=\"-1\" nonce=\"#{ThemeField::CSP_NONCE_PLACEHOLDER}\">",
)
expect(javascript_cache.content).to include("testing-div")
expect(javascript_cache.content).to include("string_setting")
expect(javascript_cache.content).to include("test text \\\" 123!")
expect(javascript_cache.content).to include(
"define(\"discourse/theme-#{theme_field.theme_id}/discourse/templates/my-template\"",
'themeCompatModules["discourse/templates/my-template"]',
)
end
@ -269,40 +271,33 @@ HTML
# All together
expect(theme.javascript_cache.content).to include(
"define(\"discourse/theme-#{theme.id}/discourse/templates/discovery\", [\"exports\", ",
'themeCompatModules["discourse/templates/discovery"]',
)
expect(theme.javascript_cache.content).to include(
"define(\"discourse/theme-#{theme.id}/discourse/controllers/discovery\"",
'themeCompatModules["discourse/controllers/discovery"]',
)
expect(theme.javascript_cache.content).to include(
"define(\"discourse/theme-#{theme.id}/discourse/controllers/discovery-2\"",
'themeCompatModules["discourse/controllers/discovery-2"]',
)
expect(theme.javascript_cache.content).to include("const settings =")
expect(theme.javascript_cache.content).to include("registerSettings(")
expect(theme.javascript_cache.content).to include(
"[THEME #{theme.id} '#{theme.name}'] Compile error: unknown file extension 'blah' (discourse/controllers/discovery.blah)",
"[THEME #{theme.id}] Unsupported file type: discourse/controllers/discovery.blah",
)
expect(theme.javascript_cache.content).to include(
"[THEME #{theme.id} '#{theme.name}'] Compile error: unknown file extension 'hbr' (discourse/templates/other_discovery.hbr)",
"[THEME #{theme.id}] Unsupported file type: discourse/templates/other_discovery.hbr",
)
# Check sourcemap
expect(theme.javascript_cache.source_map).to eq(nil)
ThemeJavascriptCompiler.enable_terser!
js_field.update(compiler_version: "0")
theme.save!
expect(theme.javascript_cache.source_map).not_to eq(nil)
map = JSON.parse(theme.javascript_cache.source_map)
expect(map["sources"]).to contain_exactly(
"discourse/controllers/discovery-2.js",
"discourse/controllers/discovery.blah",
"discourse/controllers/discovery.js",
"discourse/templates/discovery.js",
"discourse/templates/other_discovery.hbr",
"settings.js",
"theme-#{theme.id}/discourse/controllers/discovery-2.js",
"theme-#{theme.id}/discourse/controllers/discovery.js",
"theme-#{theme.id}/discourse/templates/discovery.hbs",
"theme-#{theme.id}/virtual:main",
"theme-#{theme.id}/virtual:theme",
"theme-#{theme.id}/virtual:init-settings",
)
expect(map["sourceRoot"]).to eq("theme-#{theme.id}/")
expect(map["sourcesContent"].length).to eq(6)
end
@ -690,7 +685,7 @@ HTML
it "injects into JS" do
html = <<~HTML
<script type="text/discourse-plugin" version="0.8">
var a = "inline discourse plugin";
console.log("inline discourse plugin", themePrefix("foo"));
</script>
HTML
@ -795,6 +790,7 @@ HTML
after { upload_file.unlink }
it "correctly handles local JS asset caching" do
# todo - make this a system spec
upload =
UploadCreator.new(upload_file, "test.js", for_theme: true).create_for(
Discourse::SYSTEM_USER_ID,
@ -831,30 +827,19 @@ HTML
theme.reload.javascript_cache.content,
common_field.reload.javascript_cache.content,
].each do |js|
js_to_eval = <<~JS
var settings;
var window = {};
var require = function(name) {
if(name == "discourse/lib/theme-settings-store") {
return({
registerSettings: function(id, s) {
settings = s;
}
});
expected_local_js_cache_url = js_field.javascript_cache.local_url
expect(expected_local_js_cache_url).to start_with("/theme-javascripts/")
expect(js).to include(<<~JS)
registerSettings(#{theme.id}, {
"hello": "world",
"theme_uploads": {
"test_js": "#{js_field.upload.url}"
},
"theme_uploads_local": {
"test_js": "#{js_field.javascript_cache.local_url}"
}
}
window.require = require;
#{js}
settings
});
JS
ctx = MiniRacer::Context.new
val = ctx.eval(js_to_eval)
ctx.dispose
expect(val["theme_uploads"]["test_js"]).to eq(js_field.upload.url)
expect(val["theme_uploads_local"]["test_js"]).to eq(js_field.javascript_cache.local_url)
expect(val["theme_uploads_local"]["test_js"]).to start_with("/theme-javascripts/")
end
# this is important, we do not want local_js_urls to leak into scss

View file

@ -246,24 +246,22 @@ HTML
html = <<HTML
<script type='text/discourse-plugin' version='0.1'>
const x = 1;
console.log(x, settings.foo);
</script>
HTML
baked, javascript_cache, field = transpile(html)
expect(baked).to include(javascript_cache.url)
expect(javascript_cache.content).to include("if ('define' in window) {")
expect(javascript_cache.content).to include(
"define(\"discourse/theme-#{field.theme_id}/discourse/initializers/theme-field-#{field.id}-mobile-html-script-1\"",
)
expect(javascript_cache.content).to include(
"settings = require(\"discourse/lib/theme-settings-store\").getObjectForTheme(#{field.theme_id});",
"themeCompatModules[\"discourse/initializers/theme-field-#{field.id}-mobile-html-script-1\"]",
)
expect(javascript_cache.content).to include("getObjectForTheme(#{field.theme_id});")
expect(javascript_cache.content).to include(
"name: \"theme-field-#{field.id}-mobile-html-script-1\",",
)
expect(javascript_cache.content).to include("after: \"inject-objects\",")
expect(javascript_cache.content).to include("(0, _pluginApi.withPluginApi)(\"0.1\", api =>")
expect(javascript_cache.content).to include("withPluginApi(\"0.1\", api =>")
expect(javascript_cache.content).to include("const x = 1;")
end
end
@ -392,7 +390,7 @@ HTML
target: :common,
name: :after_header,
value:
'<script type="text/discourse-plugin" version="1.0">alert(settings.name); let a = ()=>{};</script>',
'<script type="text/discourse-plugin" version="1.0">alert(settings.name); let a = ()=>{}; console.log(a);</script>',
)
theme.save!
@ -400,21 +398,19 @@ HTML
expect(Theme.lookup_field(theme.id, :desktop, :after_header)).to include(
theme_field.javascript_cache.url,
)
expect(theme_field.javascript_cache.content).to include("if ('require' in window) {")
expect(theme_field.javascript_cache.content).to include <<~JS
registerSettings(#{theme_field.theme.id}, {
"name": "bob"
});
JS
expect(theme_field.javascript_cache.content).to include(
"require(\"discourse/lib/theme-settings-store\").registerSettings(#{theme_field.theme.id}, {\"name\":\"bob\"});",
)
expect(theme_field.javascript_cache.content).to include("if ('define' in window) {")
expect(theme_field.javascript_cache.content).to include(
"define(\"discourse/theme-#{theme_field.theme.id}/discourse/initializers/theme-field-#{theme_field.id}-common-html-script-1\",",
"themeCompatModules[\"discourse/initializers/theme-field-#{theme_field.id}-common-html-script-1\"]",
)
expect(theme_field.javascript_cache.content).to include(
"name: \"theme-field-#{theme_field.id}-common-html-script-1\",",
)
expect(theme_field.javascript_cache.content).to include("after: \"inject-objects\",")
expect(theme_field.javascript_cache.content).to include(
"(0, _pluginApi.withPluginApi)(\"1.0\", api =>",
)
expect(theme_field.javascript_cache.content).to include("withPluginApi(\"1.0\", api =>")
expect(theme_field.javascript_cache.content).to include("alert(settings.name)")
expect(theme_field.javascript_cache.content).to include("let a = () => {}")
@ -423,9 +419,11 @@ HTML
theme.save!
theme_field.reload
expect(theme_field.javascript_cache.content).to include(
"require(\"discourse/lib/theme-settings-store\").registerSettings(#{theme_field.theme.id}, {\"name\":\"bill\"});",
)
expect(theme_field.javascript_cache.content).to include <<~JS
registerSettings(#{theme_field.theme.id}, {
"name": "bill"
});
JS
expect(Theme.lookup_field(theme.id, :desktop, :after_header)).to include(
theme_field.javascript_cache.url,
)
@ -844,7 +842,8 @@ HTML
theme.set_field(
target: :common,
name: :after_header,
value: '<script>console.log("hello world");</script>',
value:
'<script type="text/discourse-plugin" version="0.1">console.log("hello world");</script>',
)
theme.save!
@ -990,21 +989,6 @@ HTML
expect(content).to include("function migrate(settings)")
end
it "digest does not change when settings are changed" do
content, digest = theme.baked_js_tests_with_digest
expect(content).to be_present
expect(digest).to be_present
expect(content).to include("assert.ok(true);")
theme.update_setting(:some_number, 55)
theme.save!
expect(theme.build_settings_hash[:some_number]).to eq(55)
new_content, new_digest = theme.baked_js_tests_with_digest
expect(new_content).to eq(content)
expect(new_digest).to eq(digest)
end
end
describe "get_setting" do
@ -1143,7 +1127,7 @@ HTML
theme.set_field(target: :extra_js, name: "test.js.es6", value: "const hello = 'world';")
theme.save!
expect(theme.javascript_cache.content).to include('"list_setting":"aa,bb"')
expect(theme.javascript_cache.content).to include('"list_setting": "aa,bb"')
settings_field.update!(value: <<~YAML)
integer_setting: 1
@ -1166,7 +1150,7 @@ HTML
expect(setting_record.data_type).to eq(ThemeSetting.types[:list])
expect(setting_record.value).to eq("zz|aa")
expect(theme.javascript_cache.content).to include('"list_setting":"zz|aa"')
expect(theme.javascript_cache.content).to include('"list_setting": "zz|aa"')
end
it "allows changing a setting's type" do

View file

@ -142,9 +142,11 @@ RSpec.describe ThemeJavascriptsController do
_, digest = component.baked_js_tests_with_digest
get "/theme-javascripts/tests/#{component.id}-#{digest}.js"
expect(response.body).to include(
"require(\"discourse/lib/theme-settings-store\").registerSettings(#{component.id}, {\"num_setting\":5});",
)
expect(response.body).to include <<~JS
registerSettings(#{component.id}, {
"num_setting": 5
});
JS
expect(response.body).to include("assert.ok(true);")
end
@ -167,12 +169,17 @@ RSpec.describe ThemeJavascriptsController do
component.theme_fields.find_by(upload_id: js_upload.id).javascript_cache.digest
get "/theme-javascripts/tests/#{component.id}-#{digest}.js"
expect(response.body).to include(
"require(\"discourse/lib/theme-settings-store\").registerSettings(" +
"#{component.id}, {\"num_setting\":5,\"theme_uploads\":{\"vendorlib\":" +
"\"/uploads/default/test_#{ENV["TEST_ENV_NUMBER"].presence || "0"}/original/1X/#{js_upload.sha1}.js\"},\"theme_uploads_local\":{\"vendorlib\":" +
"\"/theme-javascripts/#{theme_javascript_hash}.js?__ws=test.localhost\"}});",
)
expect(response.body).to include <<~JS
registerSettings(#{component.id}, {
"num_setting": 5,
"theme_uploads": {
"vendorlib": "/uploads/default/test_#{ENV["TEST_ENV_NUMBER"].presence || "0"}/original/1X/#{js_upload.sha1}.js"
},
"theme_uploads_local": {
"vendorlib": "/theme-javascripts/#{theme_javascript_hash}.js?__ws=test.localhost"
}
});
JS
expect(response.body).to include("assert.ok(true);")
ensure
js_file&.close