Port code for building i18n resources to Clojure build scripts (#15189)

* Port code for building i18n resources to Clojure build scripts; remove dep on gettext

* Use Metabase classloader

* Test fixes 



* Bump adoptopenjdk version

* Delete gettext dependency from Dockerfile

* Update developers-guide.md

* Address PR feedback

* Fix frontend singular msgstr format

Co-authored-by: Luis Paolini <paoliniluis@gmail.com>

.circleci/config.yml

@@ -679,6 +679,11 @@ jobs:
                 command: |
                   cd /home/circleci/metabase/metabase/bin/build-drivers && clojure -M:test
                 no_output_timeout: 15m
+            - run:
+                name: Run i18n script tests
+                command: |
+                  cd /home/circleci/metabase/metabase/bin/i18n && clojure -M:test
+                no_output_timeout: 15m
             - run:
                 name: Run build-mb build script tests
                 command: |

.github/workflows/i18n.yml

@@ -36,6 +36,11 @@ jobs:
 
     - run: sudo apt install gettext
 
+    - name: Install Clojure CLI
+      run: |
+        curl -O https://download.clojure.org/install/linux-install-1.10.1.708.sh &&
+        sudo bash ./linux-install-1.10.1.708.sh
+
     - run: ./bin/i18n/update-translation-template
       name: Check i18n tags/make sure template can be built
     - run: ./bin/i18n/build-translation-resources

.github/workflows/uberjar.yml

@@ -33,8 +33,6 @@ jobs:
       uses: actions/setup-java@v1
       with:
         java-version: 8
-    - name: Install gettext
-      run: sudo apt install gettext
     - name: Install Clojure CLI
       run: |
         curl -O https://download.clojure.org/install/linux-install-1.10.1.708.sh &&

.gitignore

@@ -49,6 +49,7 @@
 /resources/frontend_client/embed.html
 /resources/frontend_client/index.html
 /resources/frontend_client/public.html
+/resources/i18n/*.edn
 /resources/namespaces.edn
 /resources/sample-dataset.db.trace.db
 /resources/version.properties

Dockerfile

@@ -18,8 +18,7 @@ RUN yarn install --frozen-lockfile
 # STAGE 1.2: builder backend
 ###################
 
-# Build currently doesn't work on > Java 11 (i18n utils are busted) so build on 8 until we fix this
-FROM adoptopenjdk/openjdk8:alpine as backend
+FROM adoptopenjdk/openjdk11:alpine as backend
 
 ARG MB_EDITION=oss
 
@@ -45,8 +44,7 @@ RUN lein deps
 # STAGE 1.3: main builder
 ###################
 
-# Build currently doesn't work on > Java 11 (i18n utils are busted) so build on 8 until we fix this
-FROM adoptopenjdk/openjdk8:alpine as builder
+FROM adoptopenjdk/openjdk11:alpine as builder
 
 ARG MB_EDITION=oss
 
@@ -58,10 +56,9 @@ ENV FC_LANG en-US LC_CTYPE en_US.UTF-8
 # curl:    needed by script that installs Clojure CLI
 # git:     ./bin/version
 # yarn:    frontend building
-# gettext: translations
 # java-cacerts: installs updated cacerts to /etc/ssl/certs/java/cacerts
 
-RUN apk add --no-cache coreutils bash yarn git curl gettext java-cacerts
+RUN apk add --no-cache coreutils bash yarn git curl java-cacerts
 
 # lein:    backend dependencies and building
 RUN curl https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein -o /usr/local/bin/lein && \

bin/build-mb/deps.edn

@@ -3,6 +3,7 @@
  :deps
  {common/common               {:local/root "../common"}
   build-drivers/build-drivers {:local/root "../build-drivers"}
+  i18n/i18n                   {:local/root "../i18n"}
   org.flatland/ordered        {:mvn/version "1.5.7"}}
 
  :aliases

bin/build-mb/src/build.clj

@@ -4,15 +4,8 @@
             [clojure.string :as str]
             [environ.core :as env]
             [flatland.ordered.map :as ordered-map]
-            [metabuild-common.core :as u]
-            [metabuild-common.java :as java]))
-
-(defn- build-translation-resources!
-  []
-  (u/step "Build translation resources"
-    (java/check-java-8)
-    (u/sh {:dir u/project-root-directory} "./bin/i18n/build-translation-resources")
-    (u/announce "Translation resources built successfully.")))
+            [i18n.create-artifacts :as i18n]
+            [metabuild-common.core :as u]))
 
 (defn- edition-from-env-var []
   (case (env/env :mb-edition)
@@ -66,7 +59,7 @@
    :version      (fn [{:keys [version]}]
                    (version-info/generate-version-info-file! version))
    :translations (fn [_]
-                   (build-translation-resources!))
+                   (i18n/create-all-artifacts!))
    :frontend     (fn [{:keys [edition]}]
                    (build-frontend! edition))
    :drivers      (fn [{:keys [edition]}]

bin/common/src/metabuild_common/java.clj

-(ns metabuild-common.java
-  (:require [metabuild-common.core :as u]))
-
-(defn java-version
-  "Get `major.minor` version of the `java` command, e.g. `14.0` or `1.8` (Java 8)."
-  []
-  (when-let [[_ version] (re-find #"version \"(\d+\.\d+)\..*\"" (first (u/sh "java" "-version")))]
-    (Double/parseDouble version)))
-
-(defn check-java-8 []
-  (u/step "Verify Java version is Java 8"
-    (let [version (or (java-version)
-                      (throw (Exception. "Unable to determine Java major version.")))]
-      ;; TODO -- is it possible to invoke `jabba` or some other command programmatically, or prompt for a different
-      ;; `JAVA_HOME`/`PATH` to use?
-      (when-not (#{1.8 8} version)
-        (throw (Exception. "The Metabase build script currently requires Java 8 to run. Please change your Java version and try again.")))
-      (u/announce "Java version is Java 8."))))

bin/i18n/build-translation-frontend-resource

-#!/usr/bin/env node
-
-// This program compiles a ".po" translations file to a JSON version suitable for use on the frontend
-// It removes strings that aren't used on the frontend, and other extraneous information like comments
-
-const fs = require("fs");
-const _ = require("underscore");
-const gParser = require("gettext-parser");
-
-// NOTE: this function replace xgettext "{0}" style references with ttag "${ 0 }" style references
-function replaceReferences(str) {
-  return str.replace(/\{ *(\d+) *\}/g, "${ $1 }");
-}
-
-if (process.argv.length !== 4) {
-  console.log(
-    "USAGE: build-translation-frontend-resource input.po output.json"
-  );
-  process.exit(1);
-}
-
-const inputFile = process.argv[2];
-const outputFile = process.argv[3];
-
-console.log(`Compiling ${inputFile} for frontend...`);
-
-const translationObject = gParser.po.parse(fs.readFileSync(inputFile, "utf-8"));
-
-// NOTE: unsure why the headers are duplicated in a translation for "", but we don't need it
-delete translationObject.translations[""][""];
-
-let fuzzyCount = 0;
-let emptyCount = 0;
-for (const id in translationObject.translations[""]) {
-  const translation = translationObject.translations[""][id];
-  const { reference, flag } = translation.comments || {};
-  // delete the translation, we'll add it back in if needed
-  delete translationObject.translations[""][id];
-  if (
-    // only include translations used on the frontend
-    !/(^|\n)frontend\//.test(reference)
-  ) {
-    continue;
-  }
-  // don't include empty translations
-  if (_.isEqual(translation.msgstr, [""])) {
-    emptyCount++;
-    continue;
-  }
-  // don't include fuzzy translations
-  if (flag === "fuzzy") {
-    fuzzyCount++;
-    continue;
-  }
-  // remove comments
-  delete translation.comments;
-  // delete msgid since it's redundant, we have to add it back in on the frontend though
-  delete translation.msgid;
-  // replace references in translations
-  translation.msgstr = translation.msgstr.map(str => replaceReferences(str));
-  // replace references in msgid
-  translationObject.translations[""][replaceReferences(id)] = translation;
-}
-
-if (emptyCount > 0) {
-  console.warn(`+ Warning: removed ${emptyCount} empty translations`);
-}
-if (fuzzyCount > 0) {
-  console.warn(`+ Warning: removed ${fuzzyCount} fuzzy translations`);
-}
-
-fs.writeFileSync(outputFile, JSON.stringify(translationObject), "utf-8");

bin/i18n/build-translation-resources

-#!/bin/sh
+#! /usr/bin/env bash
 
-set -eu
+set -euo pipefail
 
-# gettext installed via homebrew is "keg-only", add it to the PATH
-if [ -d "/usr/local/opt/gettext/bin" ]; then
-  export PATH="/usr/local/opt/gettext/bin:$PATH"
-fi
+source "./bin/check-clojure-cli.sh"
+check_clojure_cli
 
-POT_NAME="locales/metabase.pot"
-LOCALES=$(find locales -type f -name "*.po" -exec basename {} .po \;)
-
-if [ -z "$LOCALES" ]; then
-  LOCALES_QUOTED=""
-else
-  LOCALES_QUOTED=" $(echo "$LOCALES" | awk '{ printf "\"%s\" ", $0 }')"
-fi
-
-FRONTEND_LANG_DIR="resources/frontend_client/app/locales"
-
-# backend
-# NOTE: include "en" even though we don't have a .po file for it because it's the default?
-cat << EOF > "resources/locales.clj"
-{
-  :locales  #{"en"$LOCALES_QUOTED}
-  :packages ["metabase"]
-  :bundle   "metabase.Messages"
-}
-EOF
-
-mkdir -p "$FRONTEND_LANG_DIR"
-
-for LOCALE in $LOCALES; do
-  LOCALE_FILE="locales/$LOCALE.po"
-  LOCALE_WITH_UNDERSCORE=$(echo "$LOCALE" | tr '-' '_')
-  # frontend
-  # NOTE: just copy these for now, but eventially precompile from .po to .json
-  ./bin/i18n/build-translation-frontend-resource \
-    "$LOCALE_FILE" \
-    "$FRONTEND_LANG_DIR/$LOCALE_WITH_UNDERSCORE.json"
-
-  # backend
-  msgfmt                            \
-    --java2                         \
-    -d "resources"                  \
-    -r "metabase.Messages" \
-    -l "$LOCALE_WITH_UNDERSCORE"    \
-    "$LOCALE_FILE"
-done
+cd bin/i18n
+clojure -M -m i18n.create-artifacts $@

bin/i18n/deps.edn

+{:paths ["src"]
+
+ :deps
+ {common/common                     {:local/root "../common"}
+  cheshire/cheshire                 {:mvn/version "5.8.1"}
+  clj-http/clj-http                 {:mvn/version "3.9.1"}
+  org.fedorahosted.tennera/jgettext {:mvn/version "0.15.1"}}
+
+ :aliases
+ {:test {:extra-paths ["test"]
+         :extra-deps  {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner.git"
+                                                  :sha     "209b64504cb3bd3b99ecfec7937b358a879f55c1"}}
+         :main-opts   ["-m" "cognitect.test-runner"]}}}

bin/i18n/src/i18n/common.clj

+(ns i18n.common
+  (:require [clojure.java.io :as io]
+            [clojure.string :as str]
+            [metabuild-common.core :as u])
+  (:import [org.fedorahosted.tennera.jgettext Catalog HeaderFields Message PoParser]))
+
+(defn locales
+  "Set of all locales for which we have i18n bundles.
+
+    (locales) ; -> #{\"nl\" \"pt\" \"zh\" \"tr\" \"it\" \"fa\" ...}"
+  []
+  (set (for [^java.io.File file (.listFiles (io/file (u/filename u/project-root-directory "locales")))
+             :let               [file-name (.getName file)]
+             :when              (str/ends-with? file-name ".po")]
+         (str/replace file-name #"\.po$" ""))))
+
+(defn locale-source-po-filename [locale]
+  (u/filename u/project-root-directory "locales" (format "%s.po" locale)))
+
+;; see https://github.com/zanata/jgettext/tree/master/src/main/java/org/fedorahosted/tennera/jgettext
+
+(defn- catalog ^Catalog [locale]
+  (let [parser (PoParser.)]
+    (.parseCatalog parser (io/file (locale-source-po-filename "es")))))
+
+(defn po-headers [locale]
+  (when-let [^Message message (.locateHeader (catalog locale))]
+    (let [header-fields (HeaderFields/wrap (.getMsgstr message))]
+      (into {} (for [^String k (.getKeys header-fields)]
+                 [k (.getValue header-fields k)])))))
+
+(defn po-messages-seq [locale]
+  (for [^Message message (iterator-seq (.iterator (catalog locale)))
+        ;; remove any empty translations
+        :when            (not (str/blank? (.getMsgid message)))]
+    {:id                (.getMsgid message)
+     :id-plural         (.getMsgidPlural message)
+     :str               (.getMsgstr message)
+     :str-plural        (seq (remove str/blank? (.getMsgstrPlural message)))
+     :fuzzy?            (.isFuzzy message)
+     :plural?           (.isPlural message)
+     :source-references (seq (remove str/blank? (.getSourceReferences message)))
+     :comment           (.getMsgctxt message)}))
+
+(defn po-contents [locale]
+  {:headers  (po-headers locale)
+   :messages (po-messages-seq locale)})
+
+(defn print-message-count-xform [rf]
+  (let [num-messages (volatile! 0)]
+    (fn
+      ([]
+       (rf))
+      ([result]
+       (u/announce "Wrote %d messages." @num-messages)
+       (rf result))
+      ([result message]
+       (vswap! num-messages inc)
+       (rf result message)))))

bin/i18n/src/i18n/create_artifacts.clj

+(ns i18n.create-artifacts
+  (:require [clojure.pprint :as pprint]
+            [i18n.common :as i18n]
+            [i18n.create-artifacts.backend :as backend]
+            [i18n.create-artifacts.frontend :as frontend]
+            [metabuild-common.core :as u]))
+
+;; TODO -- shouldn't this be `locales.edn`?
+(defn- locales-dot-clj []
+  {:locales  (conj (i18n/locales) "en")
+   :packages ["metabase"]
+   :bundle   "metabase.Messages"})
+
+(defn- generate-locales-dot-clj! []
+  (u/step "Create resources/locales.clj"
+    (let [file (u/filename u/project-root-directory "resources" "locales.clj")]
+      (u/delete-file-if-exists! file)
+      (spit file (with-out-str (pprint/pprint (locales-dot-clj))))
+      (u/assert-file-exists file))))
+
+(defn- create-artifacts-for-locale! [locale]
+  (u/step (format "Create artifacts for locale %s" (pr-str locale))
+    (frontend/create-artifact-for-locale! locale)
+    (backend/create-artifact-for-locale! locale)
+    (u/announce "Artifacts for locale %s created successfully." (pr-str locale))))
+
+(defn- create-artifacts-for-all-locales! []
+  (doseq [locale (i18n/locales)]
+    (create-artifacts-for-locale! locale)))
+
+(defn create-all-artifacts! []
+  (u/step "Create i18n artifacts"
+    (generate-locales-dot-clj!)
+    (create-artifacts-for-all-locales!)
+    (u/announce "Translation resources built successfully.")))
+
+(defn -main []
+  (create-all-artifacts!))

bin/i18n/src/i18n/create_artifacts/backend.clj

+(ns i18n.create-artifacts.backend
+  (:require [clojure.java.io :as io]
+            [clojure.string :as str]
+            [i18n.common :as i18n]
+            [metabuild-common.core :as u])
+  (:import [java.io FileOutputStream OutputStreamWriter]
+           java.nio.charset.StandardCharsets))
+
+(defn- backend-message? [{:keys [source-references]}]
+  (some (fn [path]
+          (some
+           (fn [dir]
+             (str/starts-with? path dir))
+           ["src" "backend" "enterprise/backend" "shared"]))
+        source-references))
+
+(defn- ->edn [{:keys [messages]}]
+  (eduction
+   (filter backend-message?)
+   (remove :plural?)
+   i18n/print-message-count-xform
+   messages))
+
+(def target-directory
+  (u/filename u/project-root-directory "resources" "i18n"))
+
+(defn- target-filename [locale]
+  (u/filename target-directory (format "%s.edn" locale)))
+
+(defn- write-edn-file! [po-contents target-file]
+  (u/step "Write EDN file"
+    (with-open [os (FileOutputStream. (io/file target-file))
+                w  (OutputStreamWriter. os StandardCharsets/UTF_8)]
+      (.write w "{\n")
+      (doseq [{msg-id :id, msg-str :str} (->edn po-contents)]
+        (.write w (pr-str msg-id))
+        (.write w "\n")
+        (.write w (pr-str msg-str))
+        (.write w "\n\n"))
+      (.write w "}\n"))))
+
+(defn create-artifact-for-locale! [locale]
+  (let [target-file (target-filename locale)]
+    (u/step (format "Create backend artifact %s from %s" target-file (i18n/locale-source-po-filename locale))
+      (u/create-directory-unless-exists! target-directory)
+      (u/delete-file-if-exists! target-file)
+      (write-edn-file! (i18n/po-contents locale) target-file)
+      (u/assert-file-exists target-file))))

bin/i18n/src/i18n/create_artifacts/frontend.clj

+(ns i18n.create-artifacts.frontend
+  (:require [cheshire.core :as json]
+            [clojure.java.io :as io]
+            [clojure.string :as str]
+            [i18n.common :as i18n]
+            [metabuild-common.core :as u])
+  (:import [java.io FileOutputStream OutputStreamWriter]
+           java.nio.charset.StandardCharsets))
+
+(defn- frontend-message?
+  "Whether this i18n `message` comes from a frontend source file."
+  [{:keys [source-references]}]
+  (some #(str/includes? % "frontend")
+        source-references))
+
+(defn- ->ttag-reference
+  "Replace an xgettext `{0}` style reference with a ttag `${ 0 }` style reference."
+  [message-id]
+  (str/replace message-id #"\{\s*(\d+)\s*\}" "\\${ $1 }"))
+
+(defn- ->translations-map [messages]
+  {"" (into {}
+            (comp
+             ;; filter out i18n messages that aren't used on the FE client
+             (filter frontend-message?)
+             i18n/print-message-count-xform
+             (map (fn [message]
+                    [(->ttag-reference (:id message))
+                     (if (:plural? message)
+                       {:msgid_plural (:id-plural message)
+                        :msgstr       (:str-plural message)}
+                       {:msgstr [(:str message)]})])))
+            messages)})
+
+(defn- ->i18n-map
+  "Convert the contents of a `.po` file to map format used in the frontend client."
+  [po-contents]
+  {:charset      "utf-8"
+   :headers      (into {} (for [[k v] (:headers po-contents)]
+                            [(str/lower-case k) v]))
+   :translations (->translations-map (:messages po-contents))})
+
+(defn- i18n-map [locale]
+  (->i18n-map (i18n/po-contents locale)))
+
+(def ^:private target-directory
+  (u/filename u/project-root-directory "resources" "frontend_client" "app" "locales"))
+
+(defn- target-filename [locale]
+  (u/filename target-directory (format "%s.json" (str/replace locale #"-" "_"))))
+
+(defn create-artifact-for-locale! [locale]
+  (let [target-file (target-filename locale)]
+    (u/step (format "Create frontend artifact %s from %s" target-file (i18n/locale-source-po-filename locale))
+      (u/create-directory-unless-exists! target-directory)
+      (u/delete-file-if-exists! target-file)
+      (u/step "Write JSON"
+        (with-open [os (FileOutputStream. (io/file target-file))
+                    w  (OutputStreamWriter. os StandardCharsets/UTF_8)]
+          (json/generate-stream (i18n-map locale) w)))
+      (u/assert-file-exists target-file))))

bin/i18n/test/i18n/create_artifacts/backend_test.clj

+(ns i18n.create-artifacts.backend-test
+  (:require [clojure.string :as str]
+            [clojure.test :refer :all]
+            [i18n.create-artifacts.backend :as backend]
+            [i18n.create-artifacts.test-common :as test-common]))
+
+(deftest edn-test
+  (#'backend/write-edn-file! test-common/po-contents "/tmp/out.edn")
+  (is (= ["{"
+          "\"No table description yet\""
+          "\"No hay una descripción de la tabla\""
+          ""
+          "}"]
+         (some-> (slurp "/tmp/out.edn")
+                 (str/split-lines)))))

bin/i18n/test/i18n/create_artifacts/frontend_test.clj

+(ns i18n.create-artifacts.frontend-test
+  (:require [clojure.test :refer :all]
+            [i18n.create-artifacts.frontend :as frontend]
+            [i18n.create-artifacts.test-common :as test-common]))
+
+(deftest ->ttag-reference-test
+  (is (= "${ 0 } schemas"
+         (#'frontend/->ttag-reference "{0} schemas"))))
+
+(deftest ->i18n-map-test
+  (is (= {:charset      "utf-8"
+          :headers      {"mime-version"              "1.0"
+                         "content-type"              "text/plain; charset=UTF-8"
+                         "content-transfer-encoding" "8bit"
+                         "x-generator"               "POEditor.com"
+                         "project-id-version"        "Metabase"
+                         "language"                  "es"
+                         "plural-forms"              "nplurals=2; plural=(n != 1);"}
+          :translations {""
+                         {"No table description yet"
+                          {:msgstr ["No hay una descripción de la tabla"]}
+
+                          "${ 0 } Queryable Table"
+                          {:msgid_plural "{0} Queryable Tables"
+                           :msgstr       ["{0] Tabla Consultable" "{0] Tablas consultables"]}}}}
+         (#'frontend/->i18n-map test-common/po-contents))))

bin/i18n/test/i18n/create_artifacts/test_common.clj

+(ns i18n.create-artifacts.test-common)
+
+(def singular-message-frontend
+  {:id                "No table description yet",
+   :id-plural         nil
+   :str               "No hay una descripción de la tabla"
+   :str-plural        nil
+   :fuzzy?            false
+   :plural?           false
+   :source-references ["frontend/src/metabase/admin/datamodel/components/database/MetadataTable.jsx:136"]
+   :comment           nil})
+
+(def singular-message-backend
+  {:id                "No table description yet",
+   :id-plural         nil
+   :str               "No hay una descripción de la tabla"
+   :str-plural        nil
+   :fuzzy?            false
+   :plural?           false
+   :source-references ["src/metabase/models/table.clj"]
+   :comment           nil})
+
+(def plural-message-frontend
+  {:id                "{0} Queryable Table"
+   :id-plural         "{0} Queryable Tables"
+   :str               nil
+   :str-plural        ["{0] Tabla Consultable" "{0] Tablas consultables"]
+   :fuzzy?            false
+   :plural?           true
+   :source-references ["frontend/src/metabase/admin/datamodel/components/database/MetadataTableList.jsx:77"]
+   :comment           nil})
+
+(def messages
+  [singular-message-frontend
+   singular-message-backend
+   plural-message-frontend])
+
+(def po-contents
+  {:headers  {"MIME-Version"              "1.0",
+              "Content-Type"              "text/plain; charset=UTF-8",
+              "Content-Transfer-Encoding" "8bit",
+              "X-Generator"               "POEditor.com",
+              "Project-Id-Version"        "Metabase",
+              "Language"                  "es",
+              "Plural-Forms"              "nplurals=2; plural=(n != 1);"}
+   :messages messages})

bin/release/README.md

@@ -5,7 +5,7 @@
 1. Install Clojure CLI -- see [https://clojure.org/guides/getting_started]. Don't use `apt install clojure` as this
    installs a version that doesn't understand `deps.edn`.
 
-1. Install `git`, `node`, `yarn`, `awscli`, `docker`, `java`, `wget` `shasum`, and `gettext`
+1. Install `git`, `node`, `yarn`, `awscli`, `docker`, `java`, and `wget``
 
    1. For installing Docker on macOS you should use [Docker Desktop](https://docs.docker.com/docker-for-mac/install/).
       Make sure `docker ps` works from the terminal

bin/release/src/release/check_prereqs.clj

 (ns release.check-prereqs
   (:require [clojure.string :as str]
             [environ.core :as env]
-            [metabuild-common
-             [core :as u]
-             [java :as java]]))
+            [metabuild-common.core :as u]))
 
 (def ^:private required-commands
-  ["git" "node" "yarn" "aws" "docker" "java" "wget" "shasum" "gettext" "zip"])
+  ["git" "node" "yarn" "aws" "docker" "java" "wget" "zip"])
 
 (defn- check-for-required-commands []
   (u/step "Verify required external commands are available"
@@ -49,5 +47,4 @@
   (u/step "Check prereqs"
     (check-for-required-commands)
     (check-for-required-env-vars)
-    (check-docker-is-running)
-    (java/check-java-8)))
+    (check-docker-is-running)))