Don't bundle locale data, but add to static (#10119)

.github/workflows/ci.yaml

@@ -30,7 +30,7 @@ jobs:
         env:
           CI: true
       - name: Build resources
-        run: ./node_modules/.bin/gulp gen-icons-json build-translations gather-gallery-demos
+        run: ./node_modules/.bin/gulp gen-icons-json build-translations build-locale-data gather-gallery-demos
       - name: Run eslint
         run: yarn run lint:eslint
       - name: Run tsc
@@ -53,6 +53,8 @@ jobs:
         run: yarn install
         env:
           CI: true
+      - name: Build resources
+        run: ./node_modules/.bin/gulp build-translations build-locale-data
       - name: Run Tests
         run: yarn run test
   build:

.gitignore

@@ -3,7 +3,6 @@
 
 # build
 build
-build-translations/*
 hass_frontend/*
 dist
 

.prettierignore

@@ -1,5 +1,4 @@
 build
-build-translations/*
 translations/*
 node_modules/*
 hass_frontend/*

build-scripts/gulp/app.js

@@ -5,6 +5,7 @@ const env = require("../env");
 
 require("./clean.js");
 require("./translations.js");
+require("./locale-data.js");
 require("./gen-icons-json.js");
 require("./gather-static.js");
 require("./compress.js");
@@ -26,7 +27,8 @@ gulp.task(
       "gen-icons-json",
       "gen-pages-dev",
       "gen-index-app-dev",
-      "build-translations"
+      "build-translations",
+      "build-locale-data"
     ),
     "copy-static-app",
     env.useWDS()
@@ -44,7 +46,7 @@ gulp.task(
       process.env.NODE_ENV = "production";
     },
     "clean",
-    gulp.parallel("gen-icons-json", "build-translations"),
+    gulp.parallel("gen-icons-json", "build-translations", "build-locale-data"),
     "copy-static-app",
     env.useRollup() ? "rollup-prod-app" : "webpack-prod-app",
     // Don't compress running tests

build-scripts/gulp/cast.js

@@ -18,7 +18,7 @@ gulp.task(
     },
     "clean-cast",
     "translations-enable-merge-backend",
-    gulp.parallel("gen-icons-json", "build-translations"),
+    gulp.parallel("gen-icons-json", "build-translations", "build-locale-data"),
     "copy-static-cast",
     "gen-index-cast-dev",
     env.useRollup() ? "rollup-dev-server-cast" : "webpack-dev-server-cast"
@@ -33,7 +33,7 @@ gulp.task(
     },
     "clean-cast",
     "translations-enable-merge-backend",
-    gulp.parallel("gen-icons-json", "build-translations"),
+    gulp.parallel("gen-icons-json", "build-translations", "build-locale-data"),
     "copy-static-cast",
     env.useRollup() ? "rollup-prod-cast" : "webpack-prod-cast",
     "gen-index-cast-prod"

build-scripts/gulp/demo.js

@@ -20,7 +20,12 @@ gulp.task(
     },
     "clean-demo",
     "translations-enable-merge-backend",
-    gulp.parallel("gen-icons-json", "gen-index-demo-dev", "build-translations"),
+    gulp.parallel(
+      "gen-icons-json",
+      "gen-index-demo-dev",
+      "build-translations",
+      "build-locale-data"
+    ),
     "copy-static-demo",
     env.useRollup() ? "rollup-dev-server-demo" : "webpack-dev-server-demo"
   )
@@ -35,7 +40,7 @@ gulp.task(
     "clean-demo",
     // Cast needs to be backwards compatible and older HA has no translations
     "translations-enable-merge-backend",
-    gulp.parallel("gen-icons-json", "build-translations"),
+    gulp.parallel("gen-icons-json", "build-translations", "build-locale-data"),
     "copy-static-demo",
     env.useRollup() ? "rollup-prod-demo" : "webpack-prod-demo",
     "gen-index-demo-prod"

build-scripts/gulp/gallery.js

@@ -51,6 +51,7 @@ gulp.task(
     gulp.parallel(
       "gen-icons-json",
       "build-translations",
+      "build-locale-data",
       "gather-gallery-demos"
     ),
     "copy-static-gallery",
@@ -70,6 +71,7 @@ gulp.task(
     gulp.parallel(
       "gen-icons-json",
       "build-translations",
+      "build-locale-data",
       "gather-gallery-demos"
     ),
     "copy-static-gallery",

build-scripts/gulp/gather-static.js

@@ -22,11 +22,18 @@ function copyTranslations(staticDir) {
 
   // Translation output
   fs.copySync(
-    polyPath("build-translations/output"),
+    polyPath("build/translations/output"),
     staticPath("translations")
   );
 }
 
+function copyLocaleData(staticDir) {
+  const staticPath = genStaticPath(staticDir);
+
+  // Locale data output
+  fs.copySync(polyPath("build/locale-data"), staticPath("locale-data"));
+}
+
 function copyMdiIcons(staticDir) {
   const staticPath = genStaticPath(staticDir);
 
@@ -84,6 +91,11 @@ function copyMapPanel(staticDir) {
   );
 }
 
+gulp.task("copy-locale-data", async () => {
+  const staticDir = paths.app_output_static;
+  copyLocaleData(staticDir);
+});
+
 gulp.task("copy-translations-app", async () => {
   const staticDir = paths.app_output_static;
   copyTranslations(staticDir);
@@ -94,6 +106,11 @@ gulp.task("copy-translations-supervisor", async () => {
   copyTranslations(staticDir);
 });
 
+gulp.task("copy-locale-data-supervisor", async () => {
+  const staticDir = paths.hassio_output_static;
+  copyLocaleData(staticDir);
+});
+
 gulp.task("copy-static-app", async () => {
   const staticDir = paths.app_output_static;
   // Basic static files
@@ -103,6 +120,7 @@ gulp.task("copy-static-app", async () => {
   copyPolyfills(staticDir);
   copyFonts(staticDir);
   copyTranslations(staticDir);
+  copyLocaleData(staticDir);
   copyMdiIcons(staticDir);
 
   // Panel assets
@@ -123,6 +141,7 @@ gulp.task("copy-static-demo", async () => {
   copyMapPanel(paths.demo_output_static);
   copyFonts(paths.demo_output_static);
   copyTranslations(paths.demo_output_static);
+  copyLocaleData(paths.demo_output_static);
   copyMdiIcons(paths.demo_output_static);
 });
 
@@ -137,6 +156,7 @@ gulp.task("copy-static-cast", async () => {
   copyMapPanel(paths.cast_output_static);
   copyFonts(paths.cast_output_static);
   copyTranslations(paths.cast_output_static);
+  copyLocaleData(paths.cast_output_static);
   copyMdiIcons(paths.cast_output_static);
 });
 
@@ -152,5 +172,6 @@ gulp.task("copy-static-gallery", async () => {
   copyMapPanel(paths.gallery_output_static);
   copyFonts(paths.gallery_output_static);
   copyTranslations(paths.gallery_output_static);
+  copyLocaleData(paths.gallery_output_static);
   copyMdiIcons(paths.gallery_output_static);
 });

build-scripts/gulp/hassio.js

@@ -24,6 +24,8 @@ gulp.task(
     "gen-index-hassio-dev",
     "build-supervisor-translations",
     "copy-translations-supervisor",
+    "build-locale-data",
+    "copy-locale-data-supervisor",
     env.useRollup() ? "rollup-watch-hassio" : "webpack-watch-hassio"
   )
 );
@@ -38,6 +40,8 @@ gulp.task(
     "gen-icons-json",
     "build-supervisor-translations",
     "copy-translations-supervisor",
+    "build-locale-data",
+    "copy-locale-data-supervisor",
     env.useRollup() ? "rollup-prod-hassio" : "webpack-prod-hassio",
     "gen-index-hassio-prod",
     ...// Don't compress running tests

build-scripts/gulp/locale-data.js

@@ -0,0 +1,77 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+
+const del = require("del");
+const path = require("path");
+const gulp = require("gulp");
+const fs = require("fs");
+const merge = require("gulp-merge-json");
+const rename = require("gulp-rename");
+const transform = require("gulp-json-transform");
+const paths = require("../paths");
+
+const outDir = "build/locale-data";
+
+gulp.task("clean-locale-data", () => del([outDir]));
+
+gulp.task("ensure-locale-data-build-dir", (done) => {
+  if (!fs.existsSync(outDir)) {
+    fs.mkdirSync(outDir, { recursive: true });
+  }
+  done();
+});
+
+const modules = {
+  "intl-relativetimeformat": "RelativeTimeFormat",
+  "intl-datetimeformat": "DateTimeFormat",
+  "intl-numberformat": "NumberFormat",
+};
+
+gulp.task("create-locale-data", (done) => {
+  const translationMeta = JSON.parse(
+    fs.readFileSync(
+      path.join(paths.translations_src, "translationMetadata.json")
+    )
+  );
+  Object.entries(modules).forEach(([module, className]) => {
+    Object.keys(translationMeta).forEach((lang) => {
+      try {
+        const localeData = String(
+          fs.readFileSync(
+            require.resolve(`@formatjs/${module}/locale-data/${lang}.js`)
+          )
+        )
+          .replace(
+            new RegExp(
+              `\\/\\*\\s*@generated\\s*\\*\\/\\s*\\/\\/\\s*prettier-ignore\\s*if\\s*\\(Intl\\.${className}\\s*&&\\s*typeof\\s*Intl\\.${className}\\.__addLocaleData\\s*===\\s*'function'\\)\\s*{\\s*Intl\\.${className}\\.__addLocaleData\\(`,
+              "im"
+            ),
+            ""
+          )
+          .replace(/\)\s*}/im, "");
+        // make sure we have valid JSON
+        JSON.parse(localeData);
+        if (!fs.existsSync(path.join(outDir, module))) {
+          fs.mkdirSync(path.join(outDir, module), { recursive: true });
+        }
+        fs.writeFileSync(
+          path.join(outDir, `${module}/${lang}.json`),
+          localeData
+        );
+      } catch (e) {
+        if (e.code !== "MODULE_NOT_FOUND") {
+          throw e;
+        }
+      }
+    });
+    done();
+  });
+});
+
+gulp.task(
+  "build-locale-data",
+  gulp.series(
+    "clean-locale-data",
+    "ensure-locale-data-build-dir",
+    "create-locale-data"
+  )
+);

build-scripts/gulp/translations.js

@@ -17,7 +17,7 @@ const paths = require("../paths");
 
 const inFrontendDir = "translations/frontend";
 const inBackendDir = "translations/backend";
-const workDir = "build-translations";
+const workDir = "build/translations";
 const fullDir = workDir + "/full";
 const coreDir = workDir + "/core";
 const outDir = workDir + "/output";
@@ -121,7 +121,7 @@ gulp.task("clean-translations", () => del([workDir]));
 
 gulp.task("ensure-translations-build-dir", (done) => {
   if (!fs.existsSync(workDir)) {
-    fs.mkdirSync(workDir);
+    fs.mkdirSync(workDir, { recursive: true });
   }
   done();
 });

build-scripts/gulp/webpack.js

@@ -148,7 +148,7 @@ gulp.task("webpack-watch-hassio", () => {
       isProdBuild: false,
       latestBuild: true,
     })
-  ).watch({ ignored: /build-translations/, poll: isWsl }, doneHandler());
+  ).watch({ ignored: /build/, poll: isWsl }, doneHandler());
 
   gulp.watch(
     path.join(paths.translations_src, "en.json"),

src/common/translations/localize.ts

@@ -4,6 +4,7 @@ import { shouldPolyfill as shouldPolyfillRelativeTime } from "@formatjs/intl-rel
 import { shouldPolyfill as shouldPolyfillDateTime } from "@formatjs/intl-datetimeformat/lib/should-polyfill";
 import IntlMessageFormat from "intl-messageformat";
 import { Resources } from "../../types";
+import { getLocalLanguage } from "../../util/hass-translation";
 
 export type LocalizeFunc = (key: string, ...args: any[]) => string;
 interface FormatType {
@@ -15,34 +16,32 @@ export interface FormatsType {
   time: FormatType;
 }
 
-const polyfillPluralRules = shouldPolyfillPluralRules();
+const loadedPolyfillLocale = new Set();
-const polyfillRelativeTime = shouldPolyfillRelativeTime();
-const polyfillDateTime = shouldPolyfillDateTime();
 
 const polyfills: Promise<any>[] = [];
 if (__BUILD__ === "latest") {
   if (shouldPolyfillLocale()) {
     polyfills.push(import("@formatjs/intl-locale/polyfill"));
   }
-  if (polyfillPluralRules) {
+  if (shouldPolyfillPluralRules()) {
     polyfills.push(import("@formatjs/intl-pluralrules/polyfill"));
+    polyfills.push(import("@formatjs/intl-pluralrules/locale-data/en"));
   }
-  if (polyfillRelativeTime) {
+  if (shouldPolyfillRelativeTime()) {
     polyfills.push(import("@formatjs/intl-relativetimeformat/polyfill"));
   }
-  if (polyfillDateTime) {
+  if (shouldPolyfillDateTime()) {
     polyfills.push(import("@formatjs/intl-datetimeformat/polyfill"));
   }
 }
 
-let polyfillLoaded = polyfills.length === 0;
+export const polyfillsLoaded =
-export const polyfillsLoaded = polyfillLoaded
+  polyfills.length === 0
-  ? undefined
+    ? undefined
-  : Promise.all(polyfills).then(() => {
+    : Promise.all(polyfills).then(() =>
-      polyfillLoaded = true;
+        // Load the default language
-      // Load English so it becomes the default
+        loadPolyfillLocales(getLocalLanguage())
-      return loadPolyfillLocales("en");
+      );
-    });
 
 /**
  * Adapted from Polymer app-localize-behavior.
@@ -71,11 +70,11 @@ export const computeLocalize = async (
   resources: Resources,
   formats?: FormatsType
 ): Promise<LocalizeFunc> => {
-  if (!polyfillLoaded) {
+  if (polyfillsLoaded) {
     await polyfillsLoaded;
   }
 
-  loadPolyfillLocales(language);
+  await loadPolyfillLocales(language);
 
   // Everytime any of the parameters change, invalidate the strings cache.
   cache._localizationCache = {};
@@ -129,28 +128,44 @@ export const computeLocalize = async (
 };
 
 export const loadPolyfillLocales = async (language: string) => {
-  if (!polyfillsLoaded) {
+  if (loadedPolyfillLocale.has(language)) {
     return;
   }
-  await polyfillsLoaded;
+  loadedPolyfillLocale.add(language);
   try {
-    if (polyfillPluralRules) {
+    if (
-      await import(
+      Intl.NumberFormat &&
-        /* webpackExclude: /.+-.+\.js$/ */
+      // @ts-ignore
-        `@formatjs/intl-pluralrules/locale-data/${language}`
+      typeof Intl.NumberFormat.__addLocaleData === "function"
+    ) {
+      const result = await fetch(
+        `/static/locale-data/intl-numberformat/${language}.json`
       );
+      // @ts-ignore
+      Intl.NumberFormat.__addLocaleData(await result.json());
     }
-    if (polyfillRelativeTime) {
+    if (
-      await import(
+      // @ts-expect-error
-        /* webpackExclude: /.+-.+\.js$/ */
+      Intl.RelativeTimeFormat &&
-        `@formatjs/intl-relativetimeformat/locale-data/${language}`
+      // @ts-ignore
+      typeof Intl.RelativeTimeFormat.__addLocaleData === "function"
+    ) {
+      const result = await fetch(
+        `/static/locale-data/intl-relativetimeformat/${language}.json`
       );
+      // @ts-ignore
+      Intl.RelativeTimeFormat.__addLocaleData(await result.json());
     }
-    if (polyfillDateTime) {
+    if (
-      await import(
+      Intl.DateTimeFormat &&
-        /* webpackExclude: /.+-.+\.js$/ */
+      // @ts-ignore
-        `@formatjs/intl-datetimeformat/locale-data/${language}`
+      typeof Intl.DateTimeFormat.__addLocaleData === "function"
+    ) {
+      const result = await fetch(
+        `/static/locale-data/intl-datetimeformat/${language}.json`
       );
+      // @ts-ignore
+      Intl.DateTimeFormat.__addLocaleData(await result.json());
     }
   } catch (_e) {
     // Ignore

src/resources/translations-metadata.ts

@@ -1,4 +1,4 @@
-import * as translationMetadata_ from "../../build-translations/translationMetadata.json";
+import * as translationMetadata_ from "../../build/translations/translationMetadata.json";
 import { TranslationMetadata } from "../types.js";
 
 export const translationMetadata = (translationMetadata_ as any)