From aa211a57565b8aea4c7d2d0e38e684713f2a1f5a Mon Sep 17 00:00:00 2001
From: Cam Saul <1455846+camsaul@users.noreply.github.com>
Date: Wed, 17 Mar 2021 14:13:28 -0700
Subject: [PATCH] Port code for building i18n resources to Clojure build
scripts (#15189)
* Port code for building i18n resources to Clojure build scripts; remove dep on gettext
* Use Metabase classloader
* Test fixes :wrench:
* Bump adoptopenjdk version
* Delete gettext dependency from Dockerfile
* Update developers-guide.md
* Address PR feedback
* Fix frontend singular msgstr format
Co-authored-by: Luis Paolini <paoliniluis@gmail.com>
---
.circleci/config.yml | 5 ++
.github/workflows/i18n.yml | 5 ++
.github/workflows/uberjar.yml | 2 -
.gitignore | 1 +
Dockerfile | 9 +--
bin/build-mb/deps.edn | 1 +
bin/build-mb/src/build.clj | 13 +---
bin/common/src/metabuild_common/java.clj | 18 -----
bin/i18n/build-translation-frontend-resource | 72 ------------------
bin/i18n/build-translation-resources | 52 ++-----------
bin/i18n/deps.edn | 13 ++++
bin/i18n/src/i18n/common.clj | 59 +++++++++++++++
bin/i18n/src/i18n/create_artifacts.clj | 38 ++++++++++
.../src/i18n/create_artifacts/backend.clj | 48 ++++++++++++
.../src/i18n/create_artifacts/frontend.clj | 61 ++++++++++++++++
.../i18n/create_artifacts/backend_test.clj | 15 ++++
.../i18n/create_artifacts/frontend_test.clj | 26 +++++++
.../i18n/create_artifacts/test_common.clj | 46 ++++++++++++
bin/release/README.md | 2 +-
bin/release/src/release/check_prereqs.clj | 9 +--
docs/developers-guide.md | 7 +-
resources/locales.clj | 10 +--
src/metabase/util/i18n.clj | 2 +-
src/metabase/util/i18n/impl.clj | 73 +++++++++++--------
src/metabase/util/schema.clj | 12 +--
test/metabase/test/util/i18n.clj | 18 ++---
test/metabase/util/i18n/impl_test.clj | 23 +++---
27 files changed, 408 insertions(+), 232 deletions(-)
delete mode 100644 bin/common/src/metabuild_common/java.clj
delete mode 100755 bin/i18n/build-translation-frontend-resource
create mode 100644 bin/i18n/deps.edn
create mode 100644 bin/i18n/src/i18n/common.clj
create mode 100644 bin/i18n/src/i18n/create_artifacts.clj
create mode 100644 bin/i18n/src/i18n/create_artifacts/backend.clj
create mode 100644 bin/i18n/src/i18n/create_artifacts/frontend.clj
create mode 100644 bin/i18n/test/i18n/create_artifacts/backend_test.clj
create mode 100644 bin/i18n/test/i18n/create_artifacts/frontend_test.clj
create mode 100644 bin/i18n/test/i18n/create_artifacts/test_common.clj
diff --git a/.circleci/config.yml b/.circleci/config.yml
index abcf5f766e8..c8a6f9f26e6 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -679,6 +679,11 @@ jobs:
command: |
cd /home/circleci/metabase/metabase/bin/build-drivers && clojure -M:test
no_output_timeout: 15m
+ - run:
+ name: Run i18n script tests
+ command: |
+ cd /home/circleci/metabase/metabase/bin/i18n && clojure -M:test
+ no_output_timeout: 15m
- run:
name: Run build-mb build script tests
command: |
diff --git a/.github/workflows/i18n.yml b/.github/workflows/i18n.yml
index 7e14cf7369d..04606c99a52 100644
--- a/.github/workflows/i18n.yml
+++ b/.github/workflows/i18n.yml
@@ -36,6 +36,11 @@ jobs:
- run: sudo apt install gettext
+ - name: Install Clojure CLI
+ run: |
+ curl -O https://download.clojure.org/install/linux-install-1.10.1.708.sh &&
+ sudo bash ./linux-install-1.10.1.708.sh
+
- run: ./bin/i18n/update-translation-template
name: Check i18n tags/make sure template can be built
- run: ./bin/i18n/build-translation-resources
diff --git a/.github/workflows/uberjar.yml b/.github/workflows/uberjar.yml
index 5088c88e677..6b96b7fe98b 100644
--- a/.github/workflows/uberjar.yml
+++ b/.github/workflows/uberjar.yml
@@ -33,8 +33,6 @@ jobs:
uses: actions/setup-java@v1
with:
java-version: 8
- - name: Install gettext
- run: sudo apt install gettext
- name: Install Clojure CLI
run: |
curl -O https://download.clojure.org/install/linux-install-1.10.1.708.sh &&
diff --git a/.gitignore b/.gitignore
index 5f4fe89267b..1676473ceb0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,6 +49,7 @@
/resources/frontend_client/embed.html
/resources/frontend_client/index.html
/resources/frontend_client/public.html
+/resources/i18n/*.edn
/resources/namespaces.edn
/resources/sample-dataset.db.trace.db
/resources/version.properties
diff --git a/Dockerfile b/Dockerfile
index 7a9c4c6c391..a5b3b7019f4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -18,8 +18,7 @@ RUN yarn install --frozen-lockfile
# STAGE 1.2: builder backend
###################
-# Build currently doesn't work on > Java 11 (i18n utils are busted) so build on 8 until we fix this
-FROM adoptopenjdk/openjdk8:alpine as backend
+FROM adoptopenjdk/openjdk11:alpine as backend
ARG MB_EDITION=oss
@@ -45,8 +44,7 @@ RUN lein deps
# STAGE 1.3: main builder
###################
-# Build currently doesn't work on > Java 11 (i18n utils are busted) so build on 8 until we fix this
-FROM adoptopenjdk/openjdk8:alpine as builder
+FROM adoptopenjdk/openjdk11:alpine as builder
ARG MB_EDITION=oss
@@ -58,10 +56,9 @@ ENV FC_LANG en-US LC_CTYPE en_US.UTF-8
# curl: needed by script that installs Clojure CLI
# git: ./bin/version
# yarn: frontend building
-# gettext: translations
# java-cacerts: installs updated cacerts to /etc/ssl/certs/java/cacerts
-RUN apk add --no-cache coreutils bash yarn git curl gettext java-cacerts
+RUN apk add --no-cache coreutils bash yarn git curl java-cacerts
# lein: backend dependencies and building
RUN curl https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein -o /usr/local/bin/lein && \
diff --git a/bin/build-mb/deps.edn b/bin/build-mb/deps.edn
index 746faa697ca..9de9f869f40 100644
--- a/bin/build-mb/deps.edn
+++ b/bin/build-mb/deps.edn
@@ -3,6 +3,7 @@
:deps
{common/common {:local/root "../common"}
build-drivers/build-drivers {:local/root "../build-drivers"}
+ i18n/i18n {:local/root "../i18n"}
org.flatland/ordered {:mvn/version "1.5.7"}}
:aliases
diff --git a/bin/build-mb/src/build.clj b/bin/build-mb/src/build.clj
index 016ce9ee6e8..ebbba32cb3b 100644
--- a/bin/build-mb/src/build.clj
+++ b/bin/build-mb/src/build.clj
@@ -4,15 +4,8 @@
[clojure.string :as str]
[environ.core :as env]
[flatland.ordered.map :as ordered-map]
- [metabuild-common.core :as u]
- [metabuild-common.java :as java]))
-
-(defn- build-translation-resources!
- []
- (u/step "Build translation resources"
- (java/check-java-8)
- (u/sh {:dir u/project-root-directory} "./bin/i18n/build-translation-resources")
- (u/announce "Translation resources built successfully.")))
+ [i18n.create-artifacts :as i18n]
+ [metabuild-common.core :as u]))
(defn- edition-from-env-var []
(case (env/env :mb-edition)
@@ -66,7 +59,7 @@
:version (fn [{:keys [version]}]
(version-info/generate-version-info-file! version))
:translations (fn [_]
- (build-translation-resources!))
+ (i18n/create-all-artifacts!))
:frontend (fn [{:keys [edition]}]
(build-frontend! edition))
:drivers (fn [{:keys [edition]}]
diff --git a/bin/common/src/metabuild_common/java.clj b/bin/common/src/metabuild_common/java.clj
deleted file mode 100644
index 970c842cebb..00000000000
--- a/bin/common/src/metabuild_common/java.clj
+++ /dev/null
@@ -1,18 +0,0 @@
-(ns metabuild-common.java
- (:require [metabuild-common.core :as u]))
-
-(defn java-version
- "Get `major.minor` version of the `java` command, e.g. `14.0` or `1.8` (Java 8)."
- []
- (when-let [[_ version] (re-find #"version \"(\d+\.\d+)\..*\"" (first (u/sh "java" "-version")))]
- (Double/parseDouble version)))
-
-(defn check-java-8 []
- (u/step "Verify Java version is Java 8"
- (let [version (or (java-version)
- (throw (Exception. "Unable to determine Java major version.")))]
- ;; TODO -- is it possible to invoke `jabba` or some other command programmatically, or prompt for a different
- ;; `JAVA_HOME`/`PATH` to use?
- (when-not (#{1.8 8} version)
- (throw (Exception. "The Metabase build script currently requires Java 8 to run. Please change your Java version and try again.")))
- (u/announce "Java version is Java 8."))))
diff --git a/bin/i18n/build-translation-frontend-resource b/bin/i18n/build-translation-frontend-resource
deleted file mode 100755
index a1b5179014e..00000000000
--- a/bin/i18n/build-translation-frontend-resource
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env node
-
-// This program compiles a ".po" translations file to a JSON version suitable for use on the frontend
-// It removes strings that aren't used on the frontend, and other extraneous information like comments
-
-const fs = require("fs");
-const _ = require("underscore");
-const gParser = require("gettext-parser");
-
-// NOTE: this function replace xgettext "{0}" style references with ttag "${ 0 }" style references
-function replaceReferences(str) {
- return str.replace(/\{ *(\d+) *\}/g, "${ $1 }");
-}
-
-if (process.argv.length !== 4) {
- console.log(
- "USAGE: build-translation-frontend-resource input.po output.json"
- );
- process.exit(1);
-}
-
-const inputFile = process.argv[2];
-const outputFile = process.argv[3];
-
-console.log(`Compiling ${inputFile} for frontend...`);
-
-const translationObject = gParser.po.parse(fs.readFileSync(inputFile, "utf-8"));
-
-// NOTE: unsure why the headers are duplicated in a translation for "", but we don't need it
-delete translationObject.translations[""][""];
-
-let fuzzyCount = 0;
-let emptyCount = 0;
-for (const id in translationObject.translations[""]) {
- const translation = translationObject.translations[""][id];
- const { reference, flag } = translation.comments || {};
- // delete the translation, we'll add it back in if needed
- delete translationObject.translations[""][id];
- if (
- // only include translations used on the frontend
- !/(^|\n)frontend\//.test(reference)
- ) {
- continue;
- }
- // don't include empty translations
- if (_.isEqual(translation.msgstr, [""])) {
- emptyCount++;
- continue;
- }
- // don't include fuzzy translations
- if (flag === "fuzzy") {
- fuzzyCount++;
- continue;
- }
- // remove comments
- delete translation.comments;
- // delete msgid since it's redundant, we have to add it back in on the frontend though
- delete translation.msgid;
- // replace references in translations
- translation.msgstr = translation.msgstr.map(str => replaceReferences(str));
- // replace references in msgid
- translationObject.translations[""][replaceReferences(id)] = translation;
-}
-
-if (emptyCount > 0) {
- console.warn(`+ Warning: removed ${emptyCount} empty translations`);
-}
-if (fuzzyCount > 0) {
- console.warn(`+ Warning: removed ${fuzzyCount} fuzzy translations`);
-}
-
-fs.writeFileSync(outputFile, JSON.stringify(translationObject), "utf-8");
diff --git a/bin/i18n/build-translation-resources b/bin/i18n/build-translation-resources
index c6cec6a8a8a..bcac1a5d173 100755
--- a/bin/i18n/build-translation-resources
+++ b/bin/i18n/build-translation-resources
@@ -1,49 +1,9 @@
-#!/bin/sh
+#! /usr/bin/env bash
-set -eu
+set -euo pipefail
-# gettext installed via homebrew is "keg-only", add it to the PATH
-if [ -d "/usr/local/opt/gettext/bin" ]; then
- export PATH="/usr/local/opt/gettext/bin:$PATH"
-fi
+source "./bin/check-clojure-cli.sh"
+check_clojure_cli
-POT_NAME="locales/metabase.pot"
-LOCALES=$(find locales -type f -name "*.po" -exec basename {} .po \;)
-
-if [ -z "$LOCALES" ]; then
- LOCALES_QUOTED=""
-else
- LOCALES_QUOTED=" $(echo "$LOCALES" | awk '{ printf "\"%s\" ", $0 }')"
-fi
-
-FRONTEND_LANG_DIR="resources/frontend_client/app/locales"
-
-# backend
-# NOTE: include "en" even though we don't have a .po file for it because it's the default?
-cat << EOF > "resources/locales.clj"
-{
- :locales #{"en"$LOCALES_QUOTED}
- :packages ["metabase"]
- :bundle "metabase.Messages"
-}
-EOF
-
-mkdir -p "$FRONTEND_LANG_DIR"
-
-for LOCALE in $LOCALES; do
- LOCALE_FILE="locales/$LOCALE.po"
- LOCALE_WITH_UNDERSCORE=$(echo "$LOCALE" | tr '-' '_')
- # frontend
- # NOTE: just copy these for now, but eventially precompile from .po to .json
- ./bin/i18n/build-translation-frontend-resource \
- "$LOCALE_FILE" \
- "$FRONTEND_LANG_DIR/$LOCALE_WITH_UNDERSCORE.json"
-
- # backend
- msgfmt \
- --java2 \
- -d "resources" \
- -r "metabase.Messages" \
- -l "$LOCALE_WITH_UNDERSCORE" \
- "$LOCALE_FILE"
-done
+cd bin/i18n
+clojure -M -m i18n.create-artifacts $@
diff --git a/bin/i18n/deps.edn b/bin/i18n/deps.edn
new file mode 100644
index 00000000000..5b6dc746eae
--- /dev/null
+++ b/bin/i18n/deps.edn
@@ -0,0 +1,13 @@
+{:paths ["src"]
+
+ :deps
+ {common/common {:local/root "../common"}
+ cheshire/cheshire {:mvn/version "5.8.1"}
+ clj-http/clj-http {:mvn/version "3.9.1"}
+ org.fedorahosted.tennera/jgettext {:mvn/version "0.15.1"}}
+
+ :aliases
+ {:test {:extra-paths ["test"]
+ :extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner.git"
+ :sha "209b64504cb3bd3b99ecfec7937b358a879f55c1"}}
+ :main-opts ["-m" "cognitect.test-runner"]}}}
diff --git a/bin/i18n/src/i18n/common.clj b/bin/i18n/src/i18n/common.clj
new file mode 100644
index 00000000000..28d0c123e07
--- /dev/null
+++ b/bin/i18n/src/i18n/common.clj
@@ -0,0 +1,59 @@
+(ns i18n.common
+ (:require [clojure.java.io :as io]
+ [clojure.string :as str]
+ [metabuild-common.core :as u])
+ (:import [org.fedorahosted.tennera.jgettext Catalog HeaderFields Message PoParser]))
+
+(defn locales
+ "Set of all locales for which we have i18n bundles.
+
+ (locales) ; -> #{\"nl\" \"pt\" \"zh\" \"tr\" \"it\" \"fa\" ...}"
+ []
+ (set (for [^java.io.File file (.listFiles (io/file (u/filename u/project-root-directory "locales")))
+ :let [file-name (.getName file)]
+ :when (str/ends-with? file-name ".po")]
+ (str/replace file-name #"\.po$" ""))))
+
+(defn locale-source-po-filename [locale]
+ (u/filename u/project-root-directory "locales" (format "%s.po" locale)))
+
+;; see https://github.com/zanata/jgettext/tree/master/src/main/java/org/fedorahosted/tennera/jgettext
+
+(defn- catalog ^Catalog [locale]
+ (let [parser (PoParser.)]
+ (.parseCatalog parser (io/file (locale-source-po-filename "es")))))
+
+(defn po-headers [locale]
+ (when-let [^Message message (.locateHeader (catalog locale))]
+ (let [header-fields (HeaderFields/wrap (.getMsgstr message))]
+ (into {} (for [^String k (.getKeys header-fields)]
+ [k (.getValue header-fields k)])))))
+
+(defn po-messages-seq [locale]
+ (for [^Message message (iterator-seq (.iterator (catalog locale)))
+ ;; remove any empty translations
+ :when (not (str/blank? (.getMsgid message)))]
+ {:id (.getMsgid message)
+ :id-plural (.getMsgidPlural message)
+ :str (.getMsgstr message)
+ :str-plural (seq (remove str/blank? (.getMsgstrPlural message)))
+ :fuzzy? (.isFuzzy message)
+ :plural? (.isPlural message)
+ :source-references (seq (remove str/blank? (.getSourceReferences message)))
+ :comment (.getMsgctxt message)}))
+
+(defn po-contents [locale]
+ {:headers (po-headers locale)
+ :messages (po-messages-seq locale)})
+
+(defn print-message-count-xform [rf]
+ (let [num-messages (volatile! 0)]
+ (fn
+ ([]
+ (rf))
+ ([result]
+ (u/announce "Wrote %d messages." @num-messages)
+ (rf result))
+ ([result message]
+ (vswap! num-messages inc)
+ (rf result message)))))
diff --git a/bin/i18n/src/i18n/create_artifacts.clj b/bin/i18n/src/i18n/create_artifacts.clj
new file mode 100644
index 00000000000..23cf7de8ad3
--- /dev/null
+++ b/bin/i18n/src/i18n/create_artifacts.clj
@@ -0,0 +1,38 @@
+(ns i18n.create-artifacts
+ (:require [clojure.pprint :as pprint]
+ [i18n.common :as i18n]
+ [i18n.create-artifacts.backend :as backend]
+ [i18n.create-artifacts.frontend :as frontend]
+ [metabuild-common.core :as u]))
+
+;; TODO -- shouldn't this be `locales.edn`?
+(defn- locales-dot-clj []
+ {:locales (conj (i18n/locales) "en")
+ :packages ["metabase"]
+ :bundle "metabase.Messages"})
+
+(defn- generate-locales-dot-clj! []
+ (u/step "Create resources/locales.clj"
+ (let [file (u/filename u/project-root-directory "resources" "locales.clj")]
+ (u/delete-file-if-exists! file)
+ (spit file (with-out-str (pprint/pprint (locales-dot-clj))))
+ (u/assert-file-exists file))))
+
+(defn- create-artifacts-for-locale! [locale]
+ (u/step (format "Create artifacts for locale %s" (pr-str locale))
+ (frontend/create-artifact-for-locale! locale)
+ (backend/create-artifact-for-locale! locale)
+ (u/announce "Artifacts for locale %s created successfully." (pr-str locale))))
+
+(defn- create-artifacts-for-all-locales! []
+ (doseq [locale (i18n/locales)]
+ (create-artifacts-for-locale! locale)))
+
+(defn create-all-artifacts! []
+ (u/step "Create i18n artifacts"
+ (generate-locales-dot-clj!)
+ (create-artifacts-for-all-locales!)
+ (u/announce "Translation resources built successfully.")))
+
+(defn -main []
+ (create-all-artifacts!))
diff --git a/bin/i18n/src/i18n/create_artifacts/backend.clj b/bin/i18n/src/i18n/create_artifacts/backend.clj
new file mode 100644
index 00000000000..80cd9bb6d28
--- /dev/null
+++ b/bin/i18n/src/i18n/create_artifacts/backend.clj
@@ -0,0 +1,48 @@
+(ns i18n.create-artifacts.backend
+ (:require [clojure.java.io :as io]
+ [clojure.string :as str]
+ [i18n.common :as i18n]
+ [metabuild-common.core :as u])
+ (:import [java.io FileOutputStream OutputStreamWriter]
+ java.nio.charset.StandardCharsets))
+
+(defn- backend-message? [{:keys [source-references]}]
+ (some (fn [path]
+ (some
+ (fn [dir]
+ (str/starts-with? path dir))
+ ["src" "backend" "enterprise/backend" "shared"]))
+ source-references))
+
+(defn- ->edn [{:keys [messages]}]
+ (eduction
+ (filter backend-message?)
+ (remove :plural?)
+ i18n/print-message-count-xform
+ messages))
+
+(def target-directory
+ (u/filename u/project-root-directory "resources" "i18n"))
+
+(defn- target-filename [locale]
+ (u/filename target-directory (format "%s.edn" locale)))
+
+(defn- write-edn-file! [po-contents target-file]
+ (u/step "Write EDN file"
+ (with-open [os (FileOutputStream. (io/file target-file))
+ w (OutputStreamWriter. os StandardCharsets/UTF_8)]
+ (.write w "{\n")
+ (doseq [{msg-id :id, msg-str :str} (->edn po-contents)]
+ (.write w (pr-str msg-id))
+ (.write w "\n")
+ (.write w (pr-str msg-str))
+ (.write w "\n\n"))
+ (.write w "}\n"))))
+
+(defn create-artifact-for-locale! [locale]
+ (let [target-file (target-filename locale)]
+ (u/step (format "Create backend artifact %s from %s" target-file (i18n/locale-source-po-filename locale))
+ (u/create-directory-unless-exists! target-directory)
+ (u/delete-file-if-exists! target-file)
+ (write-edn-file! (i18n/po-contents locale) target-file)
+ (u/assert-file-exists target-file))))
diff --git a/bin/i18n/src/i18n/create_artifacts/frontend.clj b/bin/i18n/src/i18n/create_artifacts/frontend.clj
new file mode 100644
index 00000000000..cfa3ddd6d92
--- /dev/null
+++ b/bin/i18n/src/i18n/create_artifacts/frontend.clj
@@ -0,0 +1,61 @@
+(ns i18n.create-artifacts.frontend
+ (:require [cheshire.core :as json]
+ [clojure.java.io :as io]
+ [clojure.string :as str]
+ [i18n.common :as i18n]
+ [metabuild-common.core :as u])
+ (:import [java.io FileOutputStream OutputStreamWriter]
+ java.nio.charset.StandardCharsets))
+
+(defn- frontend-message?
+ "Whether this i18n `message` comes from a frontend source file."
+ [{:keys [source-references]}]
+ (some #(str/includes? % "frontend")
+ source-references))
+
+(defn- ->ttag-reference
+ "Replace an xgettext `{0}` style reference with a ttag `${ 0 }` style reference."
+ [message-id]
+ (str/replace message-id #"\{\s*(\d+)\s*\}" "\\${ $1 }"))
+
+(defn- ->translations-map [messages]
+ {"" (into {}
+ (comp
+ ;; filter out i18n messages that aren't used on the FE client
+ (filter frontend-message?)
+ i18n/print-message-count-xform
+ (map (fn [message]
+ [(->ttag-reference (:id message))
+ (if (:plural? message)
+ {:msgid_plural (:id-plural message)
+ :msgstr (:str-plural message)}
+ {:msgstr [(:str message)]})])))
+ messages)})
+
+(defn- ->i18n-map
+ "Convert the contents of a `.po` file to map format used in the frontend client."
+ [po-contents]
+ {:charset "utf-8"
+ :headers (into {} (for [[k v] (:headers po-contents)]
+ [(str/lower-case k) v]))
+ :translations (->translations-map (:messages po-contents))})
+
+(defn- i18n-map [locale]
+ (->i18n-map (i18n/po-contents locale)))
+
+(def ^:private target-directory
+ (u/filename u/project-root-directory "resources" "frontend_client" "app" "locales"))
+
+(defn- target-filename [locale]
+ (u/filename target-directory (format "%s.json" (str/replace locale #"-" "_"))))
+
+(defn create-artifact-for-locale! [locale]
+ (let [target-file (target-filename locale)]
+ (u/step (format "Create frontend artifact %s from %s" target-file (i18n/locale-source-po-filename locale))
+ (u/create-directory-unless-exists! target-directory)
+ (u/delete-file-if-exists! target-file)
+ (u/step "Write JSON"
+ (with-open [os (FileOutputStream. (io/file target-file))
+ w (OutputStreamWriter. os StandardCharsets/UTF_8)]
+ (json/generate-stream (i18n-map locale) w)))
+ (u/assert-file-exists target-file))))
diff --git a/bin/i18n/test/i18n/create_artifacts/backend_test.clj b/bin/i18n/test/i18n/create_artifacts/backend_test.clj
new file mode 100644
index 00000000000..929a4b9a17c
--- /dev/null
+++ b/bin/i18n/test/i18n/create_artifacts/backend_test.clj
@@ -0,0 +1,15 @@
+(ns i18n.create-artifacts.backend-test
+ (:require [clojure.string :as str]
+ [clojure.test :refer :all]
+ [i18n.create-artifacts.backend :as backend]
+ [i18n.create-artifacts.test-common :as test-common]))
+
+(deftest edn-test
+ (#'backend/write-edn-file! test-common/po-contents "/tmp/out.edn")
+ (is (= ["{"
+ "\"No table description yet\""
+ "\"No hay una descripción de la tabla\""
+ ""
+ "}"]
+ (some-> (slurp "/tmp/out.edn")
+ (str/split-lines)))))
diff --git a/bin/i18n/test/i18n/create_artifacts/frontend_test.clj b/bin/i18n/test/i18n/create_artifacts/frontend_test.clj
new file mode 100644
index 00000000000..0fe37b03bd3
--- /dev/null
+++ b/bin/i18n/test/i18n/create_artifacts/frontend_test.clj
@@ -0,0 +1,26 @@
+(ns i18n.create-artifacts.frontend-test
+ (:require [clojure.test :refer :all]
+ [i18n.create-artifacts.frontend :as frontend]
+ [i18n.create-artifacts.test-common :as test-common]))
+
+(deftest ->ttag-reference-test
+ (is (= "${ 0 } schemas"
+ (#'frontend/->ttag-reference "{0} schemas"))))
+
+(deftest ->i18n-map-test
+ (is (= {:charset "utf-8"
+ :headers {"mime-version" "1.0"
+ "content-type" "text/plain; charset=UTF-8"
+ "content-transfer-encoding" "8bit"
+ "x-generator" "POEditor.com"
+ "project-id-version" "Metabase"
+ "language" "es"
+ "plural-forms" "nplurals=2; plural=(n != 1);"}
+ :translations {""
+ {"No table description yet"
+ {:msgstr ["No hay una descripción de la tabla"]}
+
+ "${ 0 } Queryable Table"
+ {:msgid_plural "{0} Queryable Tables"
+ :msgstr ["{0] Tabla Consultable" "{0] Tablas consultables"]}}}}
+ (#'frontend/->i18n-map test-common/po-contents))))
diff --git a/bin/i18n/test/i18n/create_artifacts/test_common.clj b/bin/i18n/test/i18n/create_artifacts/test_common.clj
new file mode 100644
index 00000000000..26b18a17158
--- /dev/null
+++ b/bin/i18n/test/i18n/create_artifacts/test_common.clj
@@ -0,0 +1,46 @@
+(ns i18n.create-artifacts.test-common)
+
+(def singular-message-frontend
+ {:id "No table description yet",
+ :id-plural nil
+ :str "No hay una descripción de la tabla"
+ :str-plural nil
+ :fuzzy? false
+ :plural? false
+ :source-references ["frontend/src/metabase/admin/datamodel/components/database/MetadataTable.jsx:136"]
+ :comment nil})
+
+(def singular-message-backend
+ {:id "No table description yet",
+ :id-plural nil
+ :str "No hay una descripción de la tabla"
+ :str-plural nil
+ :fuzzy? false
+ :plural? false
+ :source-references ["src/metabase/models/table.clj"]
+ :comment nil})
+
+(def plural-message-frontend
+ {:id "{0} Queryable Table"
+ :id-plural "{0} Queryable Tables"
+ :str nil
+ :str-plural ["{0] Tabla Consultable" "{0] Tablas consultables"]
+ :fuzzy? false
+ :plural? true
+ :source-references ["frontend/src/metabase/admin/datamodel/components/database/MetadataTableList.jsx:77"]
+ :comment nil})
+
+(def messages
+ [singular-message-frontend
+ singular-message-backend
+ plural-message-frontend])
+
+(def po-contents
+ {:headers {"MIME-Version" "1.0",
+ "Content-Type" "text/plain; charset=UTF-8",
+ "Content-Transfer-Encoding" "8bit",
+ "X-Generator" "POEditor.com",
+ "Project-Id-Version" "Metabase",
+ "Language" "es",
+ "Plural-Forms" "nplurals=2; plural=(n != 1);"}
+ :messages messages})
diff --git a/bin/release/README.md b/bin/release/README.md
index be1876e9916..ec8777ee258 100644
--- a/bin/release/README.md
+++ b/bin/release/README.md
@@ -5,7 +5,7 @@
1. Install Clojure CLI -- see [https://clojure.org/guides/getting_started]. Don't use `apt install clojure` as this
installs a version that doesn't understand `deps.edn`.
-1. Install `git`, `node`, `yarn`, `awscli`, `docker`, `java`, `wget` `shasum`, and `gettext`
+1. Install `git`, `node`, `yarn`, `awscli`, `docker`, `java`, and `wget``
1. For installing Docker on macOS you should use [Docker Desktop](https://docs.docker.com/docker-for-mac/install/).
Make sure `docker ps` works from the terminal
diff --git a/bin/release/src/release/check_prereqs.clj b/bin/release/src/release/check_prereqs.clj
index 5c922f71d7c..74c8236d0d5 100644
--- a/bin/release/src/release/check_prereqs.clj
+++ b/bin/release/src/release/check_prereqs.clj
@@ -1,12 +1,10 @@
(ns release.check-prereqs
(:require [clojure.string :as str]
[environ.core :as env]
- [metabuild-common
- [core :as u]
- [java :as java]]))
+ [metabuild-common.core :as u]))
(def ^:private required-commands
- ["git" "node" "yarn" "aws" "docker" "java" "wget" "shasum" "gettext" "zip"])
+ ["git" "node" "yarn" "aws" "docker" "java" "wget" "zip"])
(defn- check-for-required-commands []
(u/step "Verify required external commands are available"
@@ -49,5 +47,4 @@
(u/step "Check prereqs"
(check-for-required-commands)
(check-for-required-env-vars)
- (check-docker-is-running)
- (java/check-java-8)))
+ (check-docker-is-running)))
diff --git a/docs/developers-guide.md b/docs/developers-guide.md
index 2d57c0d109b..a5b8a651fb1 100644
--- a/docs/developers-guide.md
+++ b/docs/developers-guide.md
@@ -25,18 +25,17 @@ If you have problems with your development environment, make sure that you are n
These are the set of tools which are required in order to complete any build of the Metabase code. Follow the links to download and install them on your own before continuing.
1. [Clojure (ttps://clojure.org)](https://clojure.org/guides/getting_started) - install the latest release by following the guide depending on your OS
-2. [Java Development Kit JDK (https://adoptopenjdk.net/releases.html)](https://adoptopenjdk.net/releases.html) - you need to install JDK 8 (./operations-guide/java-versions.md)
+2. [Java Development Kit JDK (https://adoptopenjdk.net/releases.html)](https://adoptopenjdk.net/releases.html) - you need to install JDK 11 (./operations-guide/java-versions.md)
3. [Node.js (http://nodejs.org/)](http://nodejs.org/) - latest LTS release
4. [Yarn package manager for Node.js](https://yarnpkg.com/) - latest release of version 1.x - you can install it in any OS by doing `npm install --global yarn`
5. [Leiningen (http://leiningen.org/)](http://leiningen.org/) - latest release
-6. [GetText package (https://www.gnu.org/software/gettext/)](https://www.gnu.org/software/gettext/) - latest
On a most recent stable Ubuntu/Debian, all the tools above, with the exception of Clojure, can be installed by using:
```
-sudo apt install gettext openjdk-8-jdk nodejs leiningen && sudo npm install --global yarn
+sudo apt install openjdk-11-jdk nodejs leiningen && sudo npm install --global yarn
```
-If you have multiple JDK versions installed in your machine, be sure to switch your JDK before building by doing `sudo update-alternatives --config java` and selecting Java 8 in the menu
+If you have multiple JDK versions installed in your machine, be sure to switch your JDK before building by doing `sudo update-alternatives --config java` and selecting Java 11 in the menu
If you are developing on Windows, make sure to use Ubuntu on Windows and follow instructions for Ubuntu/Linux instead of installing ordinary Windows versions.
diff --git a/resources/locales.clj b/resources/locales.clj
index 6e37aa8f477..ee96c61f308 100644
--- a/resources/locales.clj
+++ b/resources/locales.clj
@@ -1,5 +1,5 @@
-{
- :locales #{"en" "nl" "zh" "zh-HK" "fr" "ja" "bg" "de" "nb" "es" "cs" "ca" "pl" "zh-TW" "vi" "it" "ru" "sv" "fa" "sk" "pt" "tr" }
- :packages ["metabase"]
- :bundle "metabase.Messages"
-}
+{:locales
+ #{"nl" "pt" "en" "zh" "tr" "it" "fa" "vi" "zh-TW" "pl" "ca" "sv"
+ "zh-HK" "fr" "de" "nb" "ru" "sk" "es" "ja" "cs" "bg"},
+ :packages ["metabase"],
+ :bundle "metabase.Messages"}
diff --git a/src/metabase/util/i18n.clj b/src/metabase/util/i18n.clj
index 0e8c667c0ab..7b321700f19 100644
--- a/src/metabase/util/i18n.clj
+++ b/src/metabase/util/i18n.clj
@@ -78,7 +78,7 @@
(defn- localized-to-json
"Write a UserLocalizedString or SiteLocalizedString to the `json-generator`. This is intended for
- `json-gen/add-encoder`. Ideallys we'd implement those protocols directly as it's faster, but currently that doesn't
+ `json-gen/add-encoder`. Ideally we'd implement those protocols directly as it's faster, but currently that doesn't
work with Cheshire"
[localized-string json-generator]
(json-gen/write-string json-generator (str localized-string)))
diff --git a/src/metabase/util/i18n/impl.clj b/src/metabase/util/i18n/impl.clj
index a43b05ed86f..3ddae4ae0e0 100644
--- a/src/metabase/util/i18n/impl.clj
+++ b/src/metabase/util/i18n/impl.clj
@@ -8,7 +8,7 @@
[metabase.plugins.classloader :as classloader]
[potemkin.types :as p.types])
(:import java.text.MessageFormat
- [java.util Locale MissingResourceException ResourceBundle]
+ java.util.Locale
org.apache.commons.lang3.LocaleUtils))
(p.types/defprotocol+ CoerceToLocale
@@ -66,43 +66,56 @@
(when (seq (.getCountry a-locale))
(locale (.getLanguage a-locale)))))
-(def ^:private ^:const ^String i18n-bundle-name "metabase.Messages")
+(defn- locale-edn-resource
+ "The resource URL for the edn file containing translations for `locale-or-name`. These files are built by the
+ scripts in `bin/i18n` from `.po` files from POEditor.
-(defn- bundle* [^Locale locale]
- (try
- (ResourceBundle/getBundle i18n-bundle-name locale (classloader/the-classloader))
- (catch MissingResourceException _
- (log/error (format "Error translating to %s: no resource bundle" locale)))))
+ (locale-edn-resources \"es\") ;-> #object[java.net.URL \"file:/home/cam/metabase/resources/metabase/es.edn\"]"
+ ^java.net.URL [locale-or-name]
+ (when-let [a-locale (locale locale-or-name)]
+ (let [locale-name (-> (normalized-locale-string (str a-locale))
+ (str/replace #"_" "-"))
+ filename (format "i18n/%s.edn" locale-name)]
+ (io/resource filename (classloader/the-classloader)))))
+
+(defn- translations* [a-locale]
+ (when-let [resource (locale-edn-resource a-locale)]
+ (edn/read-string (slurp resource))))
-(defn- bundle
- "Get the Metabase i18n resource bundle associated with `locale`. Returns `nil` if no such bundle can be found."
- ^ResourceBundle [locale-or-name]
- (when-let [locale (locale locale-or-name)]
- (bundle* locale)))
+(def ^:private ^{:arglists '([locale-or-name])} translations
+ "Fetch a map of original untranslated message format string -> translated message format string for `locale-or-name`
+ by reading the corresponding EDN resource file. Does not include translations for parent locale(s). Memoized.
-(defn translated-format-string
- "Find the translated version of `format-string` in the bundle for `locale-or-name`, or `nil` if none can be found.
- Does not search 'parent' (country-only) locale bundle."
+ (translations \"es\") ;-> {\"Username\" \"Nombre Usuario\", ...}"
+ (comp (memoize translations*) locale))
+
+(defn- translated-format-string*
+ "Find the translated version of `format-string` for `locale-or-name`, or `nil` if none can be found.
+ Does not search 'parent' (language-only) translations."
^String [locale-or-name format-string]
(when (seq format-string)
(when-let [locale (locale locale-or-name)]
- (when-let [bundle (bundle locale)]
- (try
- (.getString bundle format-string)
- ;; no translated version available
- (catch MissingResourceException _))))))
+ (when-let [translations (translations locale)]
+ (get translations format-string)))))
+
+(defn- translated-format-string
+ "Find the translated version of `format-string` for `locale-or-name`, or `nil` if none can be found. Searches parent
+ (language-only) translations if none exist for a language + country locale."
+ ^String [locale-or-name format-string]
+ (when-let [a-locale (locale locale-or-name)]
+ (or (when (= (.getLanguage a-locale) "en")
+ format-string)
+ (translated-format-string* a-locale format-string)
+ (when-let [parent-locale (parent-locale a-locale)]
+ (log/tracef "No translated string found, trying parent locale %s" (pr-str parent-locale))
+ (translated-format-string* parent-locale format-string))
+ format-string)))
(defn- message-format ^MessageFormat [locale-or-name ^String format-string]
- (if-let [locale (locale locale-or-name)]
- (let [^String translated (or (when (= (.getLanguage locale) "en")
- format-string)
- (translated-format-string locale format-string)
- (when-let [parent-locale (parent-locale locale)]
- (log/tracef "No translated string found, trying parent locale %s" (pr-str parent-locale))
- (translated-format-string parent-locale format-string))
- format-string)]
- (MessageFormat. translated locale))
- (MessageFormat. format-string)))
+ (or (when-let [a-locale (locale locale-or-name)]
+ (when-let [^String translated (translated-format-string a-locale format-string)]
+ (MessageFormat. translated a-locale)))
+ (MessageFormat. format-string)))
(defn translate
"Find the translated version of `format-string` for a `locale-or-name`, then format it. Translates using the resource
diff --git a/src/metabase/util/schema.clj b/src/metabase/util/schema.clj
index f142c2fbe82..8a488615a62 100644
--- a/src/metabase/util/schema.clj
+++ b/src/metabase/util/schema.clj
@@ -40,7 +40,7 @@
(defn with-api-error-message
"Return `schema` with an additional `api-error-message` that will be used to explain the error if a parameter fails
validation."
- {:style/indent 1}
+ {:style/indent [:defn]}
[schema api-error-message]
(if-not (record? schema)
;; since this only works for record types, if `schema` isn't already one just wrap it in `s/named` to make it one
@@ -173,26 +173,26 @@
(def IntGreaterThanOrEqualToZero
"Schema representing an integer than must also be greater than or equal to zero."
(with-api-error-message
- (s/constrained s/Int (partial <= 0) (deferred-tru "Integer greater than or equal to zero"))
+ (s/constrained s/Int (partial <= 0) (deferred-tru "Integer greater than or equal to zero"))
(deferred-tru "value must be an integer greater than or equal to zero.")))
;; TODO - rename this to `PositiveInt`?
(def IntGreaterThanZero
"Schema representing an integer than must also be greater than zero."
(with-api-error-message
- (s/constrained s/Int (partial < 0) (deferred-tru "Integer greater than zero"))
+ (s/constrained s/Int (partial < 0) (deferred-tru "Integer greater than zero"))
(deferred-tru "value must be an integer greater than zero.")))
(def NonNegativeInt
"Schema representing an integer 0 or greater"
(with-api-error-message
- (s/constrained s/Int (partial <= 0) (deferred-tru "Integer greater than or equal to zero"))
+ (s/constrained s/Int (partial <= 0) (deferred-tru "Integer greater than or equal to zero"))
(deferred-tru "value must be an integer zero or greater.")))
(def PositiveNum
"Schema representing a numeric value greater than zero. This allows floating point numbers and integers."
(with-api-error-message
- (s/constrained s/Num (partial < 0) (deferred-tru "Number greater than zero"))
+ (s/constrained s/Num (partial < 0) (deferred-tru "Number greater than zero"))
(deferred-tru "value must be a number greater than zero.")))
(def KeywordOrString
@@ -220,7 +220,7 @@
(def EntityTypeKeywordOrString
"Validates entity type derivatives of `:entity/*`. Allows strings or keywords"
(with-api-error-message (s/pred #(isa? (keyword %) :entity/*) (deferred-tru "Valid entity type (keyword or string)"))
- (deferred-tru "value must be a valid entity type (keyword or string).")))
+ (deferred-tru "value must be a valid entity type (keyword or string).")))
(def Map
"Schema for a valid map."
diff --git a/test/metabase/test/util/i18n.clj b/test/metabase/test/util/i18n.clj
index 594b7aa0345..f9adc9f86ae 100644
--- a/test/metabase/test/util/i18n.clj
+++ b/test/metabase/test/util/i18n.clj
@@ -3,27 +3,19 @@
[metabase.util.i18n :as i18n]
[metabase.util.i18n.impl :as impl]))
-(defn map-bundle
- "Convert a Clojure map to a java `ResourceBundle`."
- ^java.util.ListResourceBundle [m]
- (when m
- (let [contents (to-array-2d (seq m))]
- (proxy [java.util.ListResourceBundle] []
- (getContents [] contents)))))
-
(defn do-with-mock-i18n-bundles [bundles thunk]
(t/testing (format "\nwith mock i18n bundles %s\n" (pr-str bundles))
- (let [locale->bundle (into {} (for [[locale m] bundles]
- [(impl/locale locale) (map-bundle m)]))]
- (with-redefs [impl/bundle* locale->bundle]
+ (let [locale->bundle (into {} (for [[locale-name bundle] bundles]
+ [(i18n/locale locale-name) bundle]))]
+ (with-redefs [impl/translations (comp locale->bundle i18n/locale)]
(thunk)))))
(defmacro with-mock-i18n-bundles
"Mock the i18n resource bundles for the duration of `body`.
(with-mock-i18n-bundles {\"es\" {\"Your database has been added!\" \"¡Tu base de datos ha sido añadida!\"}
- \"es-MX\" {\"I''m good thanks\" \"Está bien, gracias\"}}
- (/translate \"es-MX\" \"Your database has been added!\"))
+ \"es-MX\" {\"I''m good thanks\" \"Está bien, gracias\"}}
+ (translate \"es-MX\" \"Your database has been added!\"))
;; -> \"¡Tu base de datos ha sido añadida!\""
[bundles & body]
`(do-with-mock-i18n-bundles ~bundles (fn [] ~@body)))
diff --git a/test/metabase/util/i18n/impl_test.clj b/test/metabase/util/i18n/impl_test.clj
index 0e0e946a847..cc3cc60283a 100644
--- a/test/metabase/util/i18n/impl_test.clj
+++ b/test/metabase/util/i18n/impl_test.clj
@@ -1,7 +1,8 @@
(ns metabase.util.i18n.impl-test
(:require [clojure.test :refer :all]
[metabase.test :as mt]
- [metabase.util.i18n.impl :as impl]))
+ [metabase.util.i18n.impl :as impl])
+ (:import java.util.Locale))
(deftest normalized-locale-string-test
(doseq [[s expected] {"en" "en"
@@ -29,11 +30,11 @@
:str s
:keyword (keyword s))]]
(testing (pr-str (list 'locale x))
- (is (= (java.util.Locale/forLanguageTag (if language "en-US" "en"))
+ (is (= (Locale/forLanguageTag (if language "en-US" "en"))
(impl/locale x)))))
(testing "If something is already a Locale, `locale` should act as an identity fn"
- (is (= (java.util.Locale/forLanguageTag "en-US")
+ (is (= (Locale/forLanguageTag "en-US")
(impl/locale #locale "en-US")))))
(testing "nil"
@@ -58,11 +59,11 @@
(doseq [[locale expected] {nil nil
:es nil
"es" nil
- (java.util.Locale/forLanguageTag "es") nil
- "es-MX" (java.util.Locale/forLanguageTag "es")
- "es_MX" (java.util.Locale/forLanguageTag "es")
- :es/MX (java.util.Locale/forLanguageTag "es")
- (java.util.Locale/forLanguageTag "es-MX") (java.util.Locale/forLanguageTag "es")}]
+ (Locale/forLanguageTag "es") nil
+ "es-MX" (Locale/forLanguageTag "es")
+ "es_MX" (Locale/forLanguageTag "es")
+ :es/MX (Locale/forLanguageTag "es")
+ (Locale/forLanguageTag "es-MX") (Locale/forLanguageTag "es")}]
(testing locale
(is (= expected
(impl/parent-locale locale))))))
@@ -103,10 +104,8 @@
(mt/with-mock-i18n-bundles {"ba-DD" {"Bad translation {0}" "BaD TrAnSlAtIoN {a}"}}
(testing "Should fall back to original format string if translated one is busted"
(is (= "Bad translation 100"
- (mt/suppress-output
- (impl/translate "ba-DD" "Bad translation {0}" 100)))))
+ (impl/translate "ba-DD" "Bad translation {0}" 100))))
(testing "if the original format string is busted, should just return format-string as-is (better than nothing)"
(is (= "Bad original {a}"
- (mt/suppress-output
- (impl/translate "ba-DD" "Bad original {a}" 100)))))))
+ (impl/translate "ba-DD" "Bad original {a}" 100))))))
--
GitLab