From 201e5fcd53c981575f6c462f4aa021438ac1a752 Mon Sep 17 00:00:00 2001 From: matc-pub <161147791+matc-pub@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:31:07 +0100 Subject: [PATCH] Lazy loading less common languages for syntax highlighting (#2388) * Use fewer syntax highlighter languages. Reduces client.js size by about 250kB (800kB uncompressed) Common languages: bash, c, cpp, csharp, css, diff, go, graphql, ini, java, javascript, json, kotlin, less, lua, makefile, markdown, objectivec, perl, php-template, php, plaintext, python-repl, python, r, ruby, rust, scss, shell, sql, swift, typescript, vbnet, wasm, xml, yaml Additionally enabled languages: dockerfile, pgsql * Configurable syntax highlighter languages Allows to individually enable languages. * Lazy load syntax highlighter languages Allows to enable additional languages that will not be autodetected. * Include highlight.js in dynamic import check --- .eslintignore | 1 + package.json | 1 + pnpm-lock.yaml | 3 + src/client/index.tsx | 3 + src/shared/build-config.d.ts | 2 + src/shared/build-config.js | 26 ++++ .../components/comment/comment-node.tsx | 8 +- .../components/common/markdown-textarea.tsx | 4 +- .../common/registration-application.tsx | 11 +- src/shared/components/community/sidebar.tsx | 5 +- src/shared/components/home/home.tsx | 4 +- src/shared/components/home/legal.tsx | 5 +- src/shared/components/home/signup.tsx | 1 + src/shared/components/home/site-sidebar.tsx | 5 +- src/shared/components/person/profile.tsx | 4 +- src/shared/components/post/post-listing.tsx | 5 +- .../private-message-report.tsx | 4 +- .../private_message/private-message.tsx | 5 +- src/shared/dynamic-imports.ts | 13 +- src/shared/lazy-highlightjs.ts | 145 ++++++++++++++++++ src/shared/markdown.ts | 23 ++- tsconfig.json | 1 + webpack.config.js | 40 ++++- 23 files changed, 291 insertions(+), 28 deletions(-) create mode 100644 src/shared/build-config.d.ts create mode 100644 src/shared/build-config.js create mode 100644 src/shared/lazy-highlightjs.ts diff --git a/.eslintignore b/.eslintignore index e0d76256..0e9178e8 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,5 +1,6 @@ generate_translations.js webpack.config.js +src/shared/build-config.js src/api_tests **/*.png **/*.css diff --git a/package.json b/package.json index 27527963..82497943 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "emoji-mart": "^5.5.2", "emoji-short-name": "^2.0.0", "express": "~4.18.2", + "highlight.js": "^11.9.0", "history": "^5.3.0", "html-to-text": "^9.0.5", "i18next": "^23.10.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 75fad9d0..538d9e10 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -74,6 +74,9 @@ dependencies: express: specifier: ~4.18.2 version: 4.18.2 + highlight.js: + specifier: ^11.9.0 + version: 11.9.0 history: specifier: ^5.3.0 version: 5.3.0 diff --git a/src/client/index.tsx b/src/client/index.tsx index 0c8a4edf..f2518e8f 100644 --- a/src/client/index.tsx +++ b/src/client/index.tsx @@ -2,6 +2,7 @@ import { initializeSite } from "@utils/app"; import { hydrate } from "inferno-hydrate"; import { BrowserRouter } from "inferno-router"; import { App } from "../shared/components/app/app"; +import { lazyHighlightjs } from "../shared/lazy-highlightjs"; import { loadUserLanguage } from "../shared/services/I18NextService"; import { verifyDynamicImports } from "../shared/dynamic-imports"; @@ -17,6 +18,8 @@ async function startClient() { initializeSite(window.isoData.site_res); + lazyHighlightjs.enableLazyLoading(); + await loadUserLanguage(); const wrapper = ( diff --git a/src/shared/build-config.d.ts b/src/shared/build-config.d.ts new file mode 100644 index 00000000..4b94b6ba --- /dev/null +++ b/src/shared/build-config.d.ts @@ -0,0 +1,2 @@ +export const bundledSyntaxHighlighters: ["plaintext", ...string[]]; +export const lazySyntaxHighlighters: string[] | "*"; diff --git a/src/shared/build-config.js b/src/shared/build-config.js new file mode 100644 index 00000000..417fadf5 --- /dev/null +++ b/src/shared/build-config.js @@ -0,0 +1,26 @@ +// Don't import/require things here. This file is also imported in +// webpack.config.js. Needs dev server restart to apply changes. + +/** Bundled highlighters can be autodetected in markdown. + * @type ["plaintext", ...string[]] **/ +// prettier-ignore +const bundledSyntaxHighlighters = [ + "plaintext", + // The 'Common' set of highlight.js languages. + "bash", "c", "cpp", "csharp", "css", "diff", "go", "graphql", "ini", "java", + "javascript", "json", "kotlin", "less", "lua", "makefile", "markdown", + "objectivec", "perl", "php-template", "php", "python-repl", "python", "r", + "ruby", "rust", "scss", "shell", "sql", "swift", "typescript", "vbnet", + "wasm", "xml", "yaml", +]; + +/** Lazy highlighters can't be autodetected, they have to be explicitly specified + * as the language. (e.g. ```dockerfile ...) + * "*" enables all non-bundled languages + * @type string[] | "*" **/ +const lazySyntaxHighlighters = "*"; + +module.exports = { + bundledSyntaxHighlighters, + lazySyntaxHighlighters, +}; diff --git a/src/shared/components/comment/comment-node.tsx b/src/shared/components/comment/comment-node.tsx index 6de8cbe5..55969a6f 100644 --- a/src/shared/components/comment/comment-node.tsx +++ b/src/shared/components/comment/comment-node.tsx @@ -305,8 +305,12 @@ export class CommentNode extends Component { className="md-div" dangerouslySetInnerHTML={ this.props.hideImages - ? mdToHtmlNoImages(this.commentUnlessRemoved) - : mdToHtml(this.commentUnlessRemoved) + ? mdToHtmlNoImages(this.commentUnlessRemoved, () => + this.forceUpdate(), + ) + : mdToHtml(this.commentUnlessRemoved, () => + this.forceUpdate(), + ) } /> )} diff --git a/src/shared/components/common/markdown-textarea.tsx b/src/shared/components/common/markdown-textarea.tsx index 25f7734f..a35da213 100644 --- a/src/shared/components/common/markdown-textarea.tsx +++ b/src/shared/components/common/markdown-textarea.tsx @@ -250,7 +250,9 @@ export class MarkdownTextArea extends Component< {this.state.previewMode && this.state.content && (
+ this.forceUpdate(), + )} /> )} {this.state.imageUploadStatus && diff --git a/src/shared/components/common/registration-application.tsx b/src/shared/components/common/registration-application.tsx index cc3a123e..d868ed4a 100644 --- a/src/shared/components/common/registration-application.tsx +++ b/src/shared/components/common/registration-application.tsx @@ -68,7 +68,12 @@ export class RegistrationApplication extends Component<
{I18NextService.i18n.t("answer")}:
-
+
+ this.forceUpdate(), + )} + /> {a.admin && (
@@ -88,7 +93,9 @@ export class RegistrationApplication extends Component< {I18NextService.i18n.t("deny_reason")}:{" "}
+ this.forceUpdate(), + )} />
)} diff --git a/src/shared/components/community/sidebar.tsx b/src/shared/components/community/sidebar.tsx index b79b1cb7..5413998b 100644 --- a/src/shared/components/community/sidebar.tsx +++ b/src/shared/components/community/sidebar.tsx @@ -271,7 +271,10 @@ export class Sidebar extends Component { const desc = this.props.community_view.community.description; return ( desc && ( -
+
this.forceUpdate())} + /> ) ); } diff --git a/src/shared/components/home/home.tsx b/src/shared/components/home/home.tsx index 6f0788b9..cc01c0f5 100644 --- a/src/shared/components/home/home.tsx +++ b/src/shared/components/home/home.tsx @@ -401,7 +401,9 @@ export class Home extends Component { {tagline && (
+ this.forceUpdate(), + )} >
)}
{this.mobileView}
diff --git a/src/shared/components/home/legal.tsx b/src/shared/components/home/legal.tsx index 85a413eb..51786cb0 100644 --- a/src/shared/components/home/legal.tsx +++ b/src/shared/components/home/legal.tsx @@ -32,7 +32,10 @@ export class Legal extends Component { path={this.context.router.route.match.url} /> {legal && ( -
+
this.forceUpdate())} + /> )}
); diff --git a/src/shared/components/home/signup.tsx b/src/shared/components/home/signup.tsx index 94c09cd5..1714872d 100644 --- a/src/shared/components/home/signup.tsx +++ b/src/shared/components/home/signup.tsx @@ -221,6 +221,7 @@ export class Signup extends Component { className="md-div" dangerouslySetInnerHTML={mdToHtml( siteView.local_site.application_question, + () => this.forceUpdate(), )} /> )} diff --git a/src/shared/components/home/site-sidebar.tsx b/src/shared/components/home/site-sidebar.tsx index 52ecbbc7..7cb4bff2 100644 --- a/src/shared/components/home/site-sidebar.tsx +++ b/src/shared/components/home/site-sidebar.tsx @@ -99,7 +99,10 @@ export class SiteSidebar extends Component { siteSidebar(sidebar: string) { return ( -
+
this.forceUpdate())} + /> ); } diff --git a/src/shared/components/person/profile.tsx b/src/shared/components/person/profile.tsx index 77325e53..2367f40b 100644 --- a/src/shared/components/person/profile.tsx +++ b/src/shared/components/person/profile.tsx @@ -586,7 +586,9 @@ export class Profile extends Component<
+ this.forceUpdate(), + )} />
)} diff --git a/src/shared/components/post/post-listing.tsx b/src/shared/components/post/post-listing.tsx index 2ce0df57..9f1c96aa 100644 --- a/src/shared/components/post/post-listing.tsx +++ b/src/shared/components/post/post-listing.tsx @@ -177,7 +177,10 @@ export class PostListing extends Component { {this.state.viewSource ? ( 
{body}
) : ( -
+
this.forceUpdate())} + /> )} ) : ( diff --git a/src/shared/components/private_message/private-message-report.tsx b/src/shared/components/private_message/private-message-report.tsx index fee8c9c7..39e98c97 100644 --- a/src/shared/components/private_message/private-message-report.tsx +++ b/src/shared/components/private_message/private-message-report.tsx @@ -52,7 +52,9 @@ export class PrivateMessageReport extends Component { {I18NextService.i18n.t("message")}:
+ this.forceUpdate(), + )} />
diff --git a/src/shared/components/private_message/private-message.tsx b/src/shared/components/private_message/private-message.tsx index f2c9ad5f..dd59d061 100644 --- a/src/shared/components/private_message/private-message.tsx +++ b/src/shared/components/private_message/private-message.tsx @@ -135,7 +135,10 @@ export class PrivateMessage extends Component< ) : (
this.forceUpdate(), + )} /> )}
    diff --git a/src/shared/dynamic-imports.ts b/src/shared/dynamic-imports.ts index 546063fe..56955409 100644 --- a/src/shared/dynamic-imports.ts +++ b/src/shared/dynamic-imports.ts @@ -1,14 +1,17 @@ import { verifyTranslationImports } from "./services/I18NextService"; import { verifyDateFnsImports } from "@utils/app/setup-date-fns"; +import { verifyHighlighjsImports } from "./lazy-highlightjs"; export class ImportReport { error: Array<{ id: string; error: Error | string | undefined }> = []; success: string[] = []; + message?: string; } export type ImportReportCollection = { translation?: ImportReport; "date-fns"?: ImportReport; + "highlight.js"?: ImportReport; }; function collect( @@ -22,12 +25,15 @@ function collect( for (const { id, error } of report.error) { console.warn(`${kind} "${id}" failed: ${error}`); } + const message = report.message ? ` (${report.message})` : ""; const good = report.success.length; const bad = report.error.length; if (bad) { - console.error(`${bad} out of ${bad + good} ${kind} imports failed.`); + console.error( + `${bad} out of ${bad + good} ${kind} imports failed.` + message, + ); } else { - console.log(`${good} ${kind} imports verified.`); + console.log(`${good} ${kind} imports verified.` + message); } } } @@ -45,5 +51,8 @@ export async function verifyDynamicImports( await verifyDateFnsImports().then(report => collect(verbose, "date-fns", collection, report), ); + await verifyHighlighjsImports().then(report => + collect(verbose, "highlight.js", collection, report), + ); return collection; } diff --git a/src/shared/lazy-highlightjs.ts b/src/shared/lazy-highlightjs.ts new file mode 100644 index 00000000..394fde51 --- /dev/null +++ b/src/shared/lazy-highlightjs.ts @@ -0,0 +1,145 @@ +import { HLJSApi, HLJSPlugin, LanguageFn } from "highlight.js"; +import hljs from "highlight.js/lib/core"; +import { + bundledSyntaxHighlighters, + lazySyntaxHighlighters, +} from "./build-config"; +import { isBrowser } from "@utils/browser"; +import { default as MarkdownIt } from "markdown-it"; +import { ImportReport } from "./dynamic-imports"; + +async function lazyLoad(lang: string): Promise { + return import( + /* webpackChunkName: "hljs-[request]" */ + `highlight.js/lib/languages/${lang}.js` + ).then(x => x.default); +} + +class LazyHighlightjs implements HLJSPlugin { + public hljs: HLJSApi; + + private loadedLanguages: Set = new Set(); + private failedLanguages: Set = new Set(); + + constructor(hljs: HLJSApi) { + this.hljs = hljs; + this.hljs.addPlugin(this); + + // For consistent autodetection behaviour. + this.hljs.configure({ languages: bundledSyntaxHighlighters }); + + for (const lang of bundledSyntaxHighlighters) { + //"eager" means bundled, imports are resolved pseudo synchronously + import( + /* webpackMode: "eager" */ + `highlight.js/lib/languages/${lang}.js` + ) + .then(x => { + hljs.registerLanguage(lang, x.default); + this.loadedLanguages.add(lang); + }) + .catch(err => { + console.error(`Syntax highlighter "${lang}" failed:`, err); + this.failedLanguages.add(lang); + }); + } + } + + private loadLanguage(lang: string): Promise { + const promise = lazyLoad(lang); + promise + .then(x => { + this.hljs.registerLanguage(lang, x); + this.loadedLanguages.add(lang); + }) + .catch(() => { + this.failedLanguages.add(lang); + }); + return promise; + } + + private enabled: boolean = false; + private current?: { + readonly callback: () => void; + readonly pending: Set; + }; + + "before:highlight"(context: { language: string }) { + const { language: lang } = context; + if (this.loadedLanguages.has(lang)) return; + context.language = "plaintext"; // Silences "Could not find language" + if (!this.enabled || this.failedLanguages.has(lang)) { + return; + } + if (!this.current) { + console.warn(`Lazy highlightjs "${lang}" without callback.`); + this.loadLanguage(lang); + return; + } + const { callback, pending } = this.current; + pending.add(lang); + this.loadLanguage(lang) + .catch(() => {}) + .finally(() => { + if (pending.delete(lang) && !pending.size) { + // last remaining language removed, can call callback + requestAnimationFrame(() => { + callback(); + }); + } + }); + } + + public render( + md: MarkdownIt, + text: string, + shouldRerender: () => void, + ): string { + if (this.current) throw "no nesting"; + if (shouldRerender) { + this.current = { callback: shouldRerender, pending: new Set() }; + } + const result = md.render(text); + this.current = undefined; + return result; + } + + public enableLazyLoading() { + // When the server renders in a lazy language, the client will replace it + // with a plaintext render until the language is loaded. + console.assert(isBrowser(), "No lazy loading on server."); + this.enabled = true; + } + + public disableLazyLoading() { + this.enabled = false; + } +} + +export const lazyHighlightjs = new LazyHighlightjs(hljs); + +export async function verifyHighlighjsImports(): Promise { + const report = new ImportReport(); + let langs = + lazySyntaxHighlighters === "*" + ? ["dockerfile", "pgsql", "django", "nginx"] + : lazySyntaxHighlighters; + if (lazySyntaxHighlighters === "*") { + langs = langs.filter(l => !bundledSyntaxHighlighters.includes(l)); + // Avoid confusions about how few highlighters are enabled. + report.message = `Only testing ${langs.length} samples.`; + } + const promises = langs.map(lang => + lazyLoad(lang) + .then(x => { + if (x && x instanceof Function && x(hljs).name) { + report.success.push(lang); + } else { + throw "unexpected format"; + } + }) + .catch(err => report.error.push({ id: lang, error: err })), + ); + await Promise.all(promises); + return report; +} diff --git a/src/shared/markdown.ts b/src/shared/markdown.ts index 4bca0765..6ab35ee3 100644 --- a/src/shared/markdown.ts +++ b/src/shared/markdown.ts @@ -14,10 +14,11 @@ import markdown_it_html5_embed from "markdown-it-html5-embed"; import markdown_it_ruby from "markdown-it-ruby"; import markdown_it_sub from "markdown-it-sub"; import markdown_it_sup from "markdown-it-sup"; -import markdown_it_highlightjs from "markdown-it-highlightjs"; +import markdown_it_highlightjs from "markdown-it-highlightjs/core"; import Renderer from "markdown-it/lib/renderer"; import Token from "markdown-it/lib/token"; import { instanceLinkRegex, relTags } from "./config"; +import { lazyHighlightjs } from "./lazy-highlightjs"; export let Tribute: any; @@ -44,12 +45,12 @@ if (isBrowser()) { Tribute = require("tributejs"); } -export function mdToHtml(text: string) { - return { __html: md.render(text) }; +export function mdToHtml(text: string, rerender: () => void) { + return { __html: lazyHighlightjs.render(md, text, rerender) }; } -export function mdToHtmlNoImages(text: string) { - return { __html: mdNoImages.render(text) }; +export function mdToHtmlNoImages(text: string, rerender: () => void) { + return { __html: lazyHighlightjs.render(mdNoImages, text, rerender) }; } export function mdToHtmlInline(text: string) { @@ -74,6 +75,14 @@ const spoilerConfig = { }, }; +const highlightjsConfig = { + inline: true, + hljs: lazyHighlightjs.hljs, + auto: true, + code: true, + ignoreIllegals: true, +}; + const html5EmbedConfig = { html5embed: { useImageSyntax: true, // Enables video/audio embed with ![]() syntax (default) @@ -170,7 +179,7 @@ export function setupMarkdown() { .use(markdown_it_footnote) .use(markdown_it_html5_embed, html5EmbedConfig) .use(markdown_it_container, "spoiler", spoilerConfig) - .use(markdown_it_highlightjs, { inline: true }) + .use(markdown_it_highlightjs, highlightjsConfig) .use(markdown_it_ruby) .use(localInstanceLinkParser) .use(markdown_it_bidi); @@ -184,7 +193,7 @@ export function setupMarkdown() { .use(markdown_it_footnote) .use(markdown_it_html5_embed, html5EmbedConfig) .use(markdown_it_container, "spoiler", spoilerConfig) - .use(markdown_it_highlightjs, { inline: true }) + .use(markdown_it_highlightjs, highlightjsConfig) .use(localInstanceLinkParser) .use(markdown_it_bidi) // .use(markdown_it_emoji, { diff --git a/tsconfig.json b/tsconfig.json index aa3115fa..aa4f70de 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,6 +25,7 @@ } }, "include": [ + "src/shared/build-config.d.ts", "src/**/*.ts", "src/**/*.tsx", "node_modules/inferno/dist/index.d.ts" diff --git a/webpack.config.js b/webpack.config.js index 7f0fb9ff..b89dc8f1 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -4,6 +4,10 @@ const MiniCssExtractPlugin = require("mini-css-extract-plugin"); const nodeExternals = require("webpack-node-externals"); const CopyPlugin = require("copy-webpack-plugin"); const { ServiceWorkerPlugin } = require("service-worker-webpack"); +const { + bundledSyntaxHighlighters, + lazySyntaxHighlighters, +} = require("./src/shared/build-config"); const banner = ` hash:[contentHash], chunkhash:[chunkhash], name:[name], filebase:[base], query:[query], file:[file] @@ -12,6 +16,35 @@ const banner = ` @license magnet:?xt=urn:btih:0b31508aeb0634b347b8270c7bee4d411b5d4109&dn=agpl-3.0.txt AGPL v3.0 `; +const contextPlugin = (() => { + const check = x => x.match(/^[0-9a-zA-Z-]+$/); + const eagerNames = bundledSyntaxHighlighters.filter(check).join("|"); + const eagerHljs = new RegExp(`^[.][/\\\\](${eagerNames})[.]js\$`); + let lazyHljs; + if (lazySyntaxHighlighters === "*") { + lazyHljs = new RegExp(`^[.][/\\\\](?!(${eagerNames})[.]js\$)[^.]+[.]js\$`); + } else { + const lazyNames = lazySyntaxHighlighters.filter(check).join("|"); + lazyHljs = new RegExp(`^[.][/\\\\](${lazyNames})[.]js\$`); + } + + // Plugin will be used for all parameterized dynamic imports. + return new webpack.ContextReplacementPlugin(/.*/, options => { + if (/^highlight.js\/lib\/languages$/.test(options.request)) { + if (options.mode == "eager") { + options.regExp = eagerHljs; + } else { + options.regExp = lazyHljs; + } + } else if (/^date-fns\/locale$/.test(options.request)) { + } else { + return; + } + options.recursive = false; + options.request = resolve(__dirname, "node_modules/" + options.request); + }); +})(); + module.exports = (env, argv) => { const mode = argv.mode; @@ -63,12 +96,7 @@ module.exports = (env, argv) => { new webpack.BannerPlugin({ banner, }), - // helps import("date-fns/locale/${x}.mjs") find "date-fns/locale" - new webpack.ContextReplacementPlugin( - /date-fns\/locale/, - resolve(__dirname, "node_modules/date-fns/locale"), - false, - ), + contextPlugin, ], };