Lazy loading less common languages for syntax highlighting (#2388)

* Use fewer syntax highlighter languages.

Reduces client.js size by about 250kB (800kB uncompressed)

Common languages:
bash, c, cpp, csharp, css, diff, go, graphql, ini, java, javascript,
json, kotlin, less, lua, makefile, markdown, objectivec, perl,
php-template, php, plaintext, python-repl, python, r, ruby, rust, scss,
shell, sql, swift, typescript, vbnet, wasm, xml, yaml

Additionally enabled languages:
dockerfile, pgsql

* Configurable syntax highlighter languages

Allows to individually enable languages.

* Lazy load syntax highlighter languages

Allows to enable additional languages that will not be autodetected.

* Include highlight.js in dynamic import check

.eslintignore

@@ -1,5 +1,6 @@
 generate_translations.js
 webpack.config.js
+src/shared/build-config.js
 src/api_tests
 **/*.png
 **/*.css

package.json

@@ -48,6 +48,7 @@
     "emoji-mart": "^5.5.2",
     "emoji-short-name": "^2.0.0",
     "express": "~4.18.2",
+    "highlight.js": "^11.9.0",
     "history": "^5.3.0",
     "html-to-text": "^9.0.5",
     "i18next": "^23.10.0",

pnpm-lock.yaml

@@ -74,6 +74,9 @@ dependencies:
   express:
     specifier: ~4.18.2
     version: 4.18.2
+  highlight.js:
+    specifier: ^11.9.0
+    version: 11.9.0
   history:
     specifier: ^5.3.0
     version: 5.3.0

src/client/index.tsx

@@ -2,6 +2,7 @@ import { initializeSite } from "@utils/app";
 import { hydrate } from "inferno-hydrate";
 import { BrowserRouter } from "inferno-router";
 import { App } from "../shared/components/app/app";
+import { lazyHighlightjs } from "../shared/lazy-highlightjs";
 import { loadUserLanguage } from "../shared/services/I18NextService";
 import { verifyDynamicImports } from "../shared/dynamic-imports";
 
@@ -17,6 +18,8 @@ async function startClient() {
 
   initializeSite(window.isoData.site_res);
 
+  lazyHighlightjs.enableLazyLoading();
+
   await loadUserLanguage();
 
   const wrapper = (

src/shared/build-config.d.ts

@@ -0,0 +1,2 @@
+export const bundledSyntaxHighlighters: ["plaintext", ...string[]];
+export const lazySyntaxHighlighters: string[] | "*";

src/shared/build-config.js

@@ -0,0 +1,26 @@
+// Don't import/require things here. This file is also imported in
+// webpack.config.js. Needs dev server restart to apply changes.
+
+/** Bundled highlighters can be autodetected in markdown.
+ * @type ["plaintext", ...string[]] **/
+// prettier-ignore
+const bundledSyntaxHighlighters = [
+  "plaintext",
+  // The 'Common' set of highlight.js languages.
+  "bash", "c", "cpp", "csharp", "css", "diff", "go", "graphql", "ini", "java",
+  "javascript", "json", "kotlin", "less", "lua", "makefile", "markdown",
+  "objectivec", "perl", "php-template", "php", "python-repl", "python", "r",
+  "ruby", "rust", "scss", "shell", "sql", "swift", "typescript", "vbnet",
+  "wasm", "xml", "yaml",
+];
+
+/** Lazy highlighters can't be autodetected, they have to be explicitly specified
+ * as the language. (e.g. ```dockerfile ...)
+ * "*" enables all non-bundled languages
+ * @type string[] | "*" **/
+const lazySyntaxHighlighters = "*";
+
+module.exports = {
+  bundledSyntaxHighlighters,
+  lazySyntaxHighlighters,
+};

src/shared/components/comment/comment-node.tsx

@@ -305,8 +305,12 @@ export class CommentNode extends Component<CommentNodeProps, CommentNodeState> {
                       className="md-div"
                       dangerouslySetInnerHTML={
                         this.props.hideImages
-                          ? mdToHtmlNoImages(this.commentUnlessRemoved)
+                          ? mdToHtmlNoImages(this.commentUnlessRemoved, () =>
-                          : mdToHtml(this.commentUnlessRemoved)
+                              this.forceUpdate(),
+                            )
+                          : mdToHtml(this.commentUnlessRemoved, () =>
+                              this.forceUpdate(),
+                            )
                       }
                     />
                   )}

src/shared/components/common/markdown-textarea.tsx

@@ -250,7 +250,9 @@ export class MarkdownTextArea extends Component<
                 {this.state.previewMode && this.state.content && (
                   <div
                     className="card border-secondary card-body md-div"
-                    dangerouslySetInnerHTML={mdToHtml(this.state.content)}
+                    dangerouslySetInnerHTML={mdToHtml(this.state.content, () =>
+                      this.forceUpdate(),
+                    )}
                   />
                 )}
                 {this.state.imageUploadStatus &&

src/shared/components/common/registration-application.tsx

@@ -68,7 +68,12 @@ export class RegistrationApplication extends Component<
           <MomentTime showAgo published={ra.published} />
         </div>
         <div>{I18NextService.i18n.t("answer")}:</div>
-        <div className="md-div" dangerouslySetInnerHTML={mdToHtml(ra.answer)} />
+        <div
+          className="md-div"
+          dangerouslySetInnerHTML={mdToHtml(ra.answer, () =>
+            this.forceUpdate(),
+          )}
+        />
 
         {a.admin && (
           <div>
@@ -88,7 +93,9 @@ export class RegistrationApplication extends Component<
                     {I18NextService.i18n.t("deny_reason")}:{" "}
                     <div
                       className="md-div d-inline-flex"
-                      dangerouslySetInnerHTML={mdToHtml(ra.deny_reason)}
+                      dangerouslySetInnerHTML={mdToHtml(ra.deny_reason, () =>
+                        this.forceUpdate(),
+                      )}
                     />
                   </div>
                 )}

src/shared/components/community/sidebar.tsx

@@ -271,7 +271,10 @@ export class Sidebar extends Component<SidebarProps, SidebarState> {
     const desc = this.props.community_view.community.description;
     return (
       desc && (
-        <div className="md-div" dangerouslySetInnerHTML={mdToHtml(desc)} />
+        <div
+          className="md-div"
+          dangerouslySetInnerHTML={mdToHtml(desc, () => this.forceUpdate())}
+        />
       )
     );
   }

src/shared/components/home/home.tsx

@@ -401,7 +401,9 @@ export class Home extends Component<any, HomeState> {
               {tagline && (
                 <div
                   id="tagline"
-                  dangerouslySetInnerHTML={mdToHtml(tagline)}
+                  dangerouslySetInnerHTML={mdToHtml(tagline, () =>
+                    this.forceUpdate(),
+                  )}
                 ></div>
               )}
               <div className="d-block d-md-none">{this.mobileView}</div>

src/shared/components/home/legal.tsx

@@ -32,7 +32,10 @@ export class Legal extends Component<any, LegalState> {
           path={this.context.router.route.match.url}
         />
         {legal && (
-          <div className="md-div" dangerouslySetInnerHTML={mdToHtml(legal)} />
+          <div
+            className="md-div"
+            dangerouslySetInnerHTML={mdToHtml(legal, () => this.forceUpdate())}
+          />
         )}
       </div>
     );

src/shared/components/home/signup.tsx

@@ -221,6 +221,7 @@ export class Signup extends Component<any, State> {
                     className="md-div"
                     dangerouslySetInnerHTML={mdToHtml(
                       siteView.local_site.application_question,
+                      () => this.forceUpdate(),
                     )}
                   />
                 )}

src/shared/components/home/site-sidebar.tsx

@@ -99,7 +99,10 @@ export class SiteSidebar extends Component<SiteSidebarProps, SiteSidebarState> {
 
   siteSidebar(sidebar: string) {
     return (
-      <div className="md-div" dangerouslySetInnerHTML={mdToHtml(sidebar)} />
+      <div
+        className="md-div"
+        dangerouslySetInnerHTML={mdToHtml(sidebar, () => this.forceUpdate())}
+      />
     );
   }
 

src/shared/components/person/profile.tsx

@@ -586,7 +586,9 @@ export class Profile extends Component<
                 <div className="d-flex align-items-center mb-2">
                   <div
                     className="md-div"
-                    dangerouslySetInnerHTML={mdToHtml(pv.person.bio)}
+                    dangerouslySetInnerHTML={mdToHtml(pv.person.bio, () =>
+                      this.forceUpdate(),
+                    )}
                   />
                 </div>
               )}

src/shared/components/post/post-listing.tsx

@@ -177,7 +177,10 @@ export class PostListing extends Component<PostListingProps, PostListingState> {
         {this.state.viewSource ? (
           <pre>{body}</pre>
         ) : (
-          <div className="md-div" dangerouslySetInnerHTML={mdToHtml(body)} />
+          <div
+            className="md-div"
+            dangerouslySetInnerHTML={mdToHtml(body, () => this.forceUpdate())}
+          />
         )}
       </article>
     ) : (

src/shared/components/private_message/private-message-report.tsx

@@ -52,7 +52,9 @@ export class PrivateMessageReport extends Component<Props, State> {
           {I18NextService.i18n.t("message")}:
           <div
             className="md-div"
-            dangerouslySetInnerHTML={mdToHtml(pmr.original_pm_text)}
+            dangerouslySetInnerHTML={mdToHtml(pmr.original_pm_text, () =>
+              this.forceUpdate(),
+            )}
           />
         </div>
         <div>

src/shared/components/private_message/private-message.tsx

@@ -135,7 +135,10 @@ export class PrivateMessage extends Component<
               ) : (
                 <div
                   className="md-div"
-                  dangerouslySetInnerHTML={mdToHtml(this.messageUnlessRemoved)}
+                  dangerouslySetInnerHTML={mdToHtml(
+                    this.messageUnlessRemoved,
+                    () => this.forceUpdate(),
+                  )}
                 />
               )}
               <ul className="list-inline mb-0 text-muted fw-bold">

src/shared/dynamic-imports.ts

@@ -1,14 +1,17 @@
 import { verifyTranslationImports } from "./services/I18NextService";
 import { verifyDateFnsImports } from "@utils/app/setup-date-fns";
+import { verifyHighlighjsImports } from "./lazy-highlightjs";
 
 export class ImportReport {
   error: Array<{ id: string; error: Error | string | undefined }> = [];
   success: string[] = [];
+  message?: string;
 }
 
 export type ImportReportCollection = {
   translation?: ImportReport;
   "date-fns"?: ImportReport;
+  "highlight.js"?: ImportReport;
 };
 
 function collect(
@@ -22,12 +25,15 @@ function collect(
     for (const { id, error } of report.error) {
       console.warn(`${kind} "${id}" failed: ${error}`);
     }
+    const message = report.message ? ` (${report.message})` : "";
     const good = report.success.length;
     const bad = report.error.length;
     if (bad) {
-      console.error(`${bad} out of ${bad + good} ${kind} imports failed.`);
+      console.error(
+        `${bad} out of ${bad + good} ${kind} imports failed.` + message,
+      );
     } else {
-      console.log(`${good} ${kind} imports verified.`);
+      console.log(`${good} ${kind} imports verified.` + message);
     }
   }
 }
@@ -45,5 +51,8 @@ export async function verifyDynamicImports(
   await verifyDateFnsImports().then(report =>
     collect(verbose, "date-fns", collection, report),
   );
+  await verifyHighlighjsImports().then(report =>
+    collect(verbose, "highlight.js", collection, report),
+  );
   return collection;
 }

src/shared/lazy-highlightjs.ts

@@ -0,0 +1,145 @@
+import { HLJSApi, HLJSPlugin, LanguageFn } from "highlight.js";
+import hljs from "highlight.js/lib/core";
+import {
+  bundledSyntaxHighlighters,
+  lazySyntaxHighlighters,
+} from "./build-config";
+import { isBrowser } from "@utils/browser";
+import { default as MarkdownIt } from "markdown-it";
+import { ImportReport } from "./dynamic-imports";
+
+async function lazyLoad(lang: string): Promise<LanguageFn> {
+  return import(
+    /* webpackChunkName: "hljs-[request]" */
+    `highlight.js/lib/languages/${lang}.js`
+  ).then(x => x.default);
+}
+
+class LazyHighlightjs implements HLJSPlugin {
+  public hljs: HLJSApi;
+
+  private loadedLanguages: Set<string> = new Set();
+  private failedLanguages: Set<string> = new Set();
+
+  constructor(hljs: HLJSApi) {
+    this.hljs = hljs;
+    this.hljs.addPlugin(this);
+
+    // For consistent autodetection behaviour.
+    this.hljs.configure({ languages: bundledSyntaxHighlighters });
+
+    for (const lang of bundledSyntaxHighlighters) {
+      //"eager" means bundled, imports are resolved pseudo synchronously
+      import(
+        /* webpackMode: "eager" */
+        `highlight.js/lib/languages/${lang}.js`
+      )
+        .then(x => {
+          hljs.registerLanguage(lang, x.default);
+          this.loadedLanguages.add(lang);
+        })
+        .catch(err => {
+          console.error(`Syntax highlighter "${lang}" failed:`, err);
+          this.failedLanguages.add(lang);
+        });
+    }
+  }
+
+  private loadLanguage(lang: string): Promise<LanguageFn> {
+    const promise = lazyLoad(lang);
+    promise
+      .then(x => {
+        this.hljs.registerLanguage(lang, x);
+        this.loadedLanguages.add(lang);
+      })
+      .catch(() => {
+        this.failedLanguages.add(lang);
+      });
+    return promise;
+  }
+
+  private enabled: boolean = false;
+  private current?: {
+    readonly callback: () => void;
+    readonly pending: Set<string>;
+  };
+
+  "before:highlight"(context: { language: string }) {
+    const { language: lang } = context;
+    if (this.loadedLanguages.has(lang)) return;
+    context.language = "plaintext"; // Silences "Could not find language"
+    if (!this.enabled || this.failedLanguages.has(lang)) {
+      return;
+    }
+    if (!this.current) {
+      console.warn(`Lazy highlightjs "${lang}" without callback.`);
+      this.loadLanguage(lang);
+      return;
+    }
+    const { callback, pending } = this.current;
+    pending.add(lang);
+    this.loadLanguage(lang)
+      .catch(() => {})
+      .finally(() => {
+        if (pending.delete(lang) && !pending.size) {
+          // last remaining language removed, can call callback
+          requestAnimationFrame(() => {
+            callback();
+          });
+        }
+      });
+  }
+
+  public render(
+    md: MarkdownIt,
+    text: string,
+    shouldRerender: () => void,
+  ): string {
+    if (this.current) throw "no nesting";
+    if (shouldRerender) {
+      this.current = { callback: shouldRerender, pending: new Set() };
+    }
+    const result = md.render(text);
+    this.current = undefined;
+    return result;
+  }
+
+  public enableLazyLoading() {
+    // When the server renders in a lazy language, the client will replace it
+    // with a plaintext render until the language is loaded.
+    console.assert(isBrowser(), "No lazy loading on server.");
+    this.enabled = true;
+  }
+
+  public disableLazyLoading() {
+    this.enabled = false;
+  }
+}
+
+export const lazyHighlightjs = new LazyHighlightjs(hljs);
+
+export async function verifyHighlighjsImports(): Promise<ImportReport> {
+  const report = new ImportReport();
+  let langs =
+    lazySyntaxHighlighters === "*"
+      ? ["dockerfile", "pgsql", "django", "nginx"]
+      : lazySyntaxHighlighters;
+  if (lazySyntaxHighlighters === "*") {
+    langs = langs.filter(l => !bundledSyntaxHighlighters.includes(l));
+    // Avoid confusions about how few highlighters are enabled.
+    report.message = `Only testing ${langs.length} samples.`;
+  }
+  const promises = langs.map(lang =>
+    lazyLoad(lang)
+      .then(x => {
+        if (x && x instanceof Function && x(hljs).name) {
+          report.success.push(lang);
+        } else {
+          throw "unexpected format";
+        }
+      })
+      .catch(err => report.error.push({ id: lang, error: err })),
+  );
+  await Promise.all(promises);
+  return report;
+}

src/shared/markdown.ts

@@ -14,10 +14,11 @@ import markdown_it_html5_embed from "markdown-it-html5-embed";
 import markdown_it_ruby from "markdown-it-ruby";
 import markdown_it_sub from "markdown-it-sub";
 import markdown_it_sup from "markdown-it-sup";
-import markdown_it_highlightjs from "markdown-it-highlightjs";
+import markdown_it_highlightjs from "markdown-it-highlightjs/core";
 import Renderer from "markdown-it/lib/renderer";
 import Token from "markdown-it/lib/token";
 import { instanceLinkRegex, relTags } from "./config";
+import { lazyHighlightjs } from "./lazy-highlightjs";
 
 export let Tribute: any;
 
@@ -44,12 +45,12 @@ if (isBrowser()) {
   Tribute = require("tributejs");
 }
 
-export function mdToHtml(text: string) {
+export function mdToHtml(text: string, rerender: () => void) {
-  return { __html: md.render(text) };
+  return { __html: lazyHighlightjs.render(md, text, rerender) };
 }
 
-export function mdToHtmlNoImages(text: string) {
+export function mdToHtmlNoImages(text: string, rerender: () => void) {
-  return { __html: mdNoImages.render(text) };
+  return { __html: lazyHighlightjs.render(mdNoImages, text, rerender) };
 }
 
 export function mdToHtmlInline(text: string) {
@@ -74,6 +75,14 @@ const spoilerConfig = {
   },
 };
 
+const highlightjsConfig = {
+  inline: true,
+  hljs: lazyHighlightjs.hljs,
+  auto: true,
+  code: true,
+  ignoreIllegals: true,
+};
+
 const html5EmbedConfig = {
   html5embed: {
     useImageSyntax: true, // Enables video/audio embed with ![]() syntax (default)
@@ -170,7 +179,7 @@ export function setupMarkdown() {
     .use(markdown_it_footnote)
     .use(markdown_it_html5_embed, html5EmbedConfig)
     .use(markdown_it_container, "spoiler", spoilerConfig)
-    .use(markdown_it_highlightjs, { inline: true })
+    .use(markdown_it_highlightjs, highlightjsConfig)
     .use(markdown_it_ruby)
     .use(localInstanceLinkParser)
     .use(markdown_it_bidi);
@@ -184,7 +193,7 @@ export function setupMarkdown() {
     .use(markdown_it_footnote)
     .use(markdown_it_html5_embed, html5EmbedConfig)
     .use(markdown_it_container, "spoiler", spoilerConfig)
-    .use(markdown_it_highlightjs, { inline: true })
+    .use(markdown_it_highlightjs, highlightjsConfig)
     .use(localInstanceLinkParser)
     .use(markdown_it_bidi)
     // .use(markdown_it_emoji, {

tsconfig.json

@@ -25,6 +25,7 @@
     }
   },
   "include": [
+    "src/shared/build-config.d.ts",
     "src/**/*.ts",
     "src/**/*.tsx",
     "node_modules/inferno/dist/index.d.ts"

webpack.config.js

@@ -4,6 +4,10 @@ const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 const nodeExternals = require("webpack-node-externals");
 const CopyPlugin = require("copy-webpack-plugin");
 const { ServiceWorkerPlugin } = require("service-worker-webpack");
+const {
+  bundledSyntaxHighlighters,
+  lazySyntaxHighlighters,
+} = require("./src/shared/build-config");
 
 const banner = `
   hash:[contentHash], chunkhash:[chunkhash], name:[name], filebase:[base], query:[query], file:[file]
@@ -12,6 +16,35 @@ const banner = `
   @license magnet:?xt=urn:btih:0b31508aeb0634b347b8270c7bee4d411b5d4109&dn=agpl-3.0.txt AGPL v3.0
   `;
 
+const contextPlugin = (() => {
+  const check = x => x.match(/^[0-9a-zA-Z-]+$/);
+  const eagerNames = bundledSyntaxHighlighters.filter(check).join("|");
+  const eagerHljs = new RegExp(`^[.][/\\\\](${eagerNames})[.]js\$`);
+  let lazyHljs;
+  if (lazySyntaxHighlighters === "*") {
+    lazyHljs = new RegExp(`^[.][/\\\\](?!(${eagerNames})[.]js\$)[^.]+[.]js\$`);
+  } else {
+    const lazyNames = lazySyntaxHighlighters.filter(check).join("|");
+    lazyHljs = new RegExp(`^[.][/\\\\](${lazyNames})[.]js\$`);
+  }
+
+  // Plugin will be used for all parameterized dynamic imports.
+  return new webpack.ContextReplacementPlugin(/.*/, options => {
+    if (/^highlight.js\/lib\/languages$/.test(options.request)) {
+      if (options.mode == "eager") {
+        options.regExp = eagerHljs;
+      } else {
+        options.regExp = lazyHljs;
+      }
+    } else if (/^date-fns\/locale$/.test(options.request)) {
+    } else {
+      return;
+    }
+    options.recursive = false;
+    options.request = resolve(__dirname, "node_modules/" + options.request);
+  });
+})();
+
 module.exports = (env, argv) => {
   const mode = argv.mode;
 
@@ -63,12 +96,7 @@ module.exports = (env, argv) => {
       new webpack.BannerPlugin({
         banner,
       }),
-      // helps import("date-fns/locale/${x}.mjs") find "date-fns/locale"
+      contextPlugin,
-      new webpack.ContextReplacementPlugin(
-        /date-fns\/locale/,
-        resolve(__dirname, "node_modules/date-fns/locale"),
-        false,
-      ),
     ],
   };