feat: add first-class support for CJK (#8530)

scripts/buildPackage.js

@@ -97,8 +97,7 @@ const createESMBrowserBuild = async () => {
 //   );
 // });
 
-const rawConfig = {
-  entryPoints: ["index.tsx"],
+const rawConfigCommon = {
   bundle: true,
   format: "esm",
   plugins: [sassPlugin()],
@@ -107,28 +106,55 @@ const rawConfig = {
     ".woff2": "file",
   },
   packages: "external",
+  // chunks are always external, so they are not bundled within and get build separately
+  external: ["*.chunk"],
 };
 
-const createESMRawBuild = async () => {
-  // Development unminified build with source maps
-  await build({
-    ...rawConfig,
+const rawConfigIndex = {
+  ...rawConfigCommon,
+  entryPoints: ["index.tsx"],
+};
+
+const rawConfigChunks = {
+  ...rawConfigCommon,
+  // create a separate chunk for each
+  entryPoints: ["**/*.chunk.ts"],
+};
+
+function buildDev(chunkConfig) {
+  const config = {
+    ...chunkConfig,
     sourcemap: true,
-    outdir: "dist/dev",
     define: {
       "import.meta.env": JSON.stringify({ DEV: true }),
     },
-  });
+    outdir: "dist/dev",
+  };
 
-  // production minified build without sourcemaps
-  await build({
-    ...rawConfig,
+  return build(config);
+}
+
+function buildProd(chunkConfig) {
+  const config = {
+    ...chunkConfig,
     minify: true,
-    outdir: "dist/prod",
     define: {
       "import.meta.env": JSON.stringify({ PROD: true }),
     },
-  });
+    outdir: "dist/prod",
+  };
+
+  return build(config);
+}
+
+const createESMRawBuild = async () => {
+  // development unminified build with source maps
+  await buildDev(rawConfigIndex);
+  await buildDev(rawConfigChunks);
+
+  // production minified buld without sourcemaps
+  await buildProd(rawConfigIndex);
+  await buildProd(rawConfigChunks);
 };
 
 createESMRawBuild();

scripts/buildUtils.js

@@ -82,7 +82,6 @@ const rawConfig = {
   entryPoints: ["index.ts"],
   bundle: true,
   format: "esm",
-  packages: "external",
 };
 
 // const BASE_PATH = `${path.resolve(`${__dirname}/..`)}`;

scripts/buildWasm.js

@@ -8,12 +8,12 @@ const wasmModules = [
   {
     pkg: `../node_modules/fonteditor-core`,
     src: `./wasm/woff2.wasm`,
-    dest: `../packages/excalidraw/fonts/wasm/woff2.wasm.ts`,
+    dest: `../packages/excalidraw/fonts/wasm/woff2-wasm.ts`,
   },
   {
     pkg: `../node_modules/harfbuzzjs`,
     src: `./wasm/hb-subset.wasm`,
-    dest: `../packages/excalidraw/fonts/wasm/hb-subset.wasm.ts`,
+    dest: `../packages/excalidraw/fonts/wasm/hb-subset-wasm.ts`,
   },
 ];
 
@@ -35,7 +35,7 @@ for (const { pkg, src, dest } of wasmModules) {
   const licenseContent = fs.readFileSync(licensePath, "utf-8") || "";
   const base64 = fs.readFileSync(sourcePath, "base64");
   const content = `// GENERATED CODE -- DO NOT EDIT!
-/* eslint-disable prettier/prettier */
+/* eslint-disable */
 // @ts-nocheck
 
 /**

scripts/woff2/woff2-esbuild-plugins.js

@@ -11,7 +11,7 @@ const { Font } = require("fonteditor-core");
  * 2. convert all the imported fonts (including those from cdn) at build time into .ttf (since Resvg does not support woff2, neither inlined dataurls - https://github.com/RazrFalcon/resvg/issues/541)
  *    - merging multiple woff2 into one ttf (for same families with different unicode ranges)
  *    - deduplicating glyphs due to the merge process
- *    - merging emoji font for each
+ *    - merging fallback font for each
  *    - printing out font metrics
  *
  * @returns {import("esbuild").Plugin}
@@ -93,7 +93,6 @@ module.exports.woff2ServerPlugin = (options = {}) => {
         },
       );
 
-      // TODO: strip away some unnecessary glyphs
       build.onEnd(async () => {
         if (!generateTtf) {
           return;
@@ -109,15 +108,48 @@ module.exports.woff2ServerPlugin = (options = {}) => {
           return;
         }
 
+        const xiaolaiPath = path.resolve(
+          __dirname,
+          "./assets/Xiaolai-Regular.ttf",
+        );
+        const emojiPath = path.resolve(
+          __dirname,
+          "./assets/NotoEmoji-Regular.ttf",
+        );
+
+        // need to use the same em size as built-in fonts, otherwise pyftmerge throws (modified manually with font forge)
+        const emojiPath_2048 = path.resolve(
+          __dirname,
+          "./assets/NotoEmoji-Regular-2048.ttf",
+        );
+
+        const xiaolaiFont = Font.create(fs.readFileSync(xiaolaiPath), {
+          type: "ttf",
+        });
+        const emojiFont = Font.create(fs.readFileSync(emojiPath), {
+          type: "ttf",
+        });
+
         const sortedFonts = Array.from(fonts.entries()).sort(
           ([family1], [family2]) => (family1 > family2 ? 1 : -1),
         );
 
         // for now we are interested in the regular families only
         for (const [family, { Regular }] of sortedFonts) {
-          const baseFont = Regular[0];
+          if (family.includes("Xiaolai")) {
+            // don't generate ttf for Xiaolai, as we have it hardcoded as one ttf
+            continue;
+          }
 
-          const tempFilePaths = Regular.map((_, index) =>
+          const fallbackFontsPaths = [];
+          const shouldIncludeXiaolaiFallback = family.includes("Excalifont");
+
+          if (shouldIncludeXiaolaiFallback) {
+            fallbackFontsPaths.push(xiaolaiPath);
+          }
+
+          const baseFont = Regular[0];
+          const tempPaths = Regular.map((_, index) =>
             path.resolve(outputDir, `temp_${family}_${index}.ttf`),
           );
 
@@ -128,45 +160,28 @@ module.exports.woff2ServerPlugin = (options = {}) => {
             }
 
             // write down the buffer
-            fs.writeFileSync(tempFilePaths[index], font.write({ type: "ttf" }));
+            fs.writeFileSync(tempPaths[index], font.write({ type: "ttf" }));
           }
 
-          const emojiFilePath = path.resolve(
-            __dirname,
-            "./assets/NotoEmoji-Regular.ttf",
-          );
-
-          const emojiBuffer = fs.readFileSync(emojiFilePath);
-          const emojiFont = Font.create(emojiBuffer, { type: "ttf" });
-
-          // hack so that:
-          // - emoji font has same metrics as the base font, otherwise pyftmerge throws due to different unitsPerEm
-          // - emoji font glyphs are adjusted based to the base font glyphs, otherwise the glyphs don't match
-          const patchedEmojiFont = Font.create({
-            ...baseFont.data,
-            glyf: baseFont.find({ unicode: [65] }), // adjust based on the "A" glyph (does not have to be first)
-          }).merge(emojiFont, { adjustGlyf: true });
-
-          const emojiTempFilePath = path.resolve(
-            outputDir,
-            `temp_${family}_Emoji.ttf`,
-          );
-          fs.writeFileSync(
-            emojiTempFilePath,
-            patchedEmojiFont.write({ type: "ttf" }),
-          );
-
           const mergedFontPath = path.resolve(outputDir, `${family}.ttf`);
 
+          if (baseFont.data.head.unitsPerEm === 2048) {
+            fallbackFontsPaths.push(emojiPath_2048);
+          } else {
+            fallbackFontsPaths.push(emojiPath);
+          }
+
+          // drop Vertical related metrics, otherwise it does not allow us to merge the fonts
+          // vhea (Vertical Header Table)
+          // vmtx (Vertical Metrics Table)
           execSync(
-            `pyftmerge --output-file="${mergedFontPath}" "${tempFilePaths.join(
+            `pyftmerge --drop-tables=vhea,vmtx --output-file="${mergedFontPath}" "${tempPaths.join(
               '" "',
-            )}" "${emojiTempFilePath}"`,
+            )}" "${fallbackFontsPaths.join('" "')}"`,
           );
 
           // cleanup
-          fs.rmSync(emojiTempFilePath);
-          for (const path of tempFilePaths) {
+          for (const path of tempPaths) {
             fs.rmSync(path);
           }
 
@@ -177,13 +192,22 @@ module.exports.woff2ServerPlugin = (options = {}) => {
             hinting: true,
           });
 
-          // keep copyright & licence per both fonts, as per the OFL licence
+          const getNameField = (field) => {
+            const base = baseFont.data.name[field];
+            const xiaolai = xiaolaiFont.data.name[field];
+            const emoji = emojiFont.data.name[field];
+
+            return shouldIncludeXiaolaiFallback
+              ? `${base} & ${xiaolai} & ${emoji}`
+              : `${base} & ${emoji}`;
+          };
+
           mergedFont.set({
             ...mergedFont.data,
             name: {
               ...mergedFont.data.name,
-              copyright: `${baseFont.data.name.copyright} & ${emojiFont.data.name.copyright}`,
-              licence: `${baseFont.data.name.licence} & ${emojiFont.data.name.licence}`,
+              copyright: getNameField("copyright"),
+              licence: getNameField("licence"),
             },
           });
 
@@ -194,7 +218,7 @@ module.exports.woff2ServerPlugin = (options = {}) => {
           console.info(`Generated "${family}"`);
           if (Regular.length > 1) {
             console.info(
-              `- by merging ${Regular.length} woff2 files and 1 emoji ttf file`,
+              `- by merging ${Regular.length} woff2 fonts and related fallback fonts`,
             );
           }
           console.info(