From cfa6bf84656047f9acdf7db6486ec214c195be6f Mon Sep 17 00:00:00 2001
From: Cam Saul <1455846+camsaul@users.noreply.github.com>
Date: Fri, 6 Nov 2020 09:44:43 -0800
Subject: [PATCH] Mega build script improvements (#13638)
* Mega build script improvements
* Look for project.clj to determine root directory since we delete .git on CI for speed
* If .git folder is missing (i.e., in CI) fail gracefully
* Don't need to generate version file to run Cypress tests, right?
* Build uberjar step can cache version.properties
(so the Cypress test steps don't need to generate it again)
* Flush caches
* Update uberjar cache key
---
.circleci/config.yml | 40 +++--
Dockerfile | 2 +-
OSX/deps.edn | 16 +-
OSX/macos_release.clj | 22 +--
OSX/macos_release/common.clj | 151 +++---------------
OSX/release.sh | 9 ++
bin/build | 83 +---------
bin/build-drivers/build_drivers.clj | 8 +-
.../build_drivers/build_driver.clj | 2 +-
bin/build-drivers/build_drivers/checksum.clj | 10 +-
bin/build-drivers/build_drivers/common.clj | 40 ++---
.../build_drivers/install_driver_locally.clj | 4 +-
bin/build-drivers/build_drivers/metabase.clj | 14 +-
bin/build-drivers/deps.edn | 10 +-
bin/build-mb/build.clj | 92 +++++++++++
bin/build-mb/build/version_info.clj | 77 +++++++++
bin/build-mb/deps.edn | 6 +
bin/common/deps.edn | 7 +
bin/{ => common}/metabuild_common/aws.clj | 0
bin/{ => common}/metabuild_common/core.clj | 6 +-
.../metabuild_common/entrypoint.clj | 0
bin/{ => common}/metabuild_common/env.clj | 0
bin/{ => common}/metabuild_common/files.clj | 17 ++
bin/{ => common}/metabuild_common/input.clj | 8 +-
bin/common/metabuild_common/java.clj | 18 +++
bin/{ => common}/metabuild_common/output.clj | 0
bin/{ => common}/metabuild_common/shell.clj | 0
bin/{ => common}/metabuild_common/steps.clj | 0
bin/release/deps.edn | 20 ++-
bin/release/release/check_prereqs.clj | 22 +--
bin/release/release/common/upload.clj | 3 +-
bin/release/release/set_build_options.clj | 2 +-
bin/release/release/uberjar.clj | 38 +----
bin/remove-drivers.sh | 8 -
bin/verify-driver | 16 --
bin/version | 11 --
docs/developers-guide-osx.md | 35 ++--
src/metabase/config.clj | 26 +--
38 files changed, 370 insertions(+), 453 deletions(-)
create mode 100755 OSX/release.sh
create mode 100644 bin/build-mb/build.clj
create mode 100644 bin/build-mb/build/version_info.clj
create mode 100644 bin/build-mb/deps.edn
create mode 100644 bin/common/deps.edn
rename bin/{ => common}/metabuild_common/aws.clj (100%)
rename bin/{ => common}/metabuild_common/core.clj (86%)
rename bin/{ => common}/metabuild_common/entrypoint.clj (100%)
rename bin/{ => common}/metabuild_common/env.clj (100%)
rename bin/{ => common}/metabuild_common/files.clj (83%)
rename bin/{ => common}/metabuild_common/input.clj (90%)
create mode 100644 bin/common/metabuild_common/java.clj
rename bin/{ => common}/metabuild_common/output.clj (100%)
rename bin/{ => common}/metabuild_common/shell.clj (100%)
rename bin/{ => common}/metabuild_common/steps.clj (100%)
delete mode 100755 bin/remove-drivers.sh
delete mode 100755 bin/verify-driver
delete mode 100755 bin/version
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 97533d0389e..11dfd0fccc7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -201,15 +201,15 @@ commands:
steps:
- restore_cache:
keys:
- - be-deps-v2-{{ checksum "project.clj" }}
- - be-deps-v2-
+ - be-deps-v4-{{ checksum "project.clj" }}
+ - be-deps-v4-
restore-fe-deps-cache:
steps:
- restore_cache:
keys:
- - fe-deps-v2-{{ checksum "yarn.lock" }}
- - fe-deps-v2-
+ - fe-deps-v4-{{ checksum "yarn.lock" }}
+ - fe-deps-v4-
run-yarn-command:
parameters:
@@ -361,7 +361,7 @@ jobs:
- restore-be-deps-cache
- run: lein with-profile +include-all-drivers,+cloverage,+junit,+<< parameters.edition >> deps
- save_cache:
- key: be-deps-v2-{{ checksum "project.clj" }}
+ key: be-deps-v4-{{ checksum "project.clj" }}
paths:
- /home/circleci/.m2
@@ -495,7 +495,7 @@ jobs:
name: Save yarn checksum
command: md5sum yarn.lock > yarn.lock.checksum
- save_cache:
- key: fe-deps-v2-{{ checksum "yarn.lock" }}
+ key: fe-deps-v4-{{ checksum "yarn.lock" }}
paths:
- /home/circleci/.yarn
- /home/circleci/.yarn-cache
@@ -562,7 +562,7 @@ jobs:
# restore already-built uberjar
- restore_cache:
keys:
- - uberjar-<< parameters.edition >>-{{ checksum "./backend-checksums.txt" }}-{{ checksum "./frontend-checksums.txt" }}-{{ checksum "./modules-checksums.txt" }}
+ - uberjar-v4-<< parameters.edition >>-{{ checksum "./backend-checksums.txt" }}-{{ checksum "./frontend-checksums.txt" }}-{{ checksum "./modules-checksums.txt" }}
# restore the local maven installation of Metabase which is needed for building drivers
- restore_cache:
keys:
@@ -570,13 +570,13 @@ jobs:
# restore already-built drivers
- restore_cache:
keys:
- - drivers-v3-{{ checksum "./modules-checksums.txt" }}-{{ checksum "./backend-checksums.txt" }}
- - drivers-v3-{{ checksum "./modules-checksums.txt" }}
- - drivers-v3-
+ - drivers-v4-{{ checksum "./modules-checksums.txt" }}-{{ checksum "./backend-checksums.txt" }}
+ - drivers-v4-{{ checksum "./modules-checksums.txt" }}
+ - drivers-v4-
# restore already-built frontend
- restore_cache:
keys:
- - frontend-{{ checksum "./frontend-checksums.txt" }}
+ - frontend-v4-{{ checksum "./frontend-checksums.txt" }}
- run:
name: Install Clojure CLI
command: >
@@ -605,11 +605,14 @@ jobs:
no_output_timeout: 10m
- store_artifacts:
path: /home/circleci/metabase/metabase/target/uberjar/metabase.jar
- # Cache the built uberjar
+ - store_artifacts:
+ path: /home/circleci/metabase/metabase/resources/version.properties
+ # Cache the built uberjar & version.properties
- save_cache:
- key: uberjar-<< parameters.edition >>-{{ checksum "./backend-checksums.txt" }}-{{ checksum "./frontend-checksums.txt" }}-{{ checksum "./modules-checksums.txt" }}
+ key: uberjar-v4-<< parameters.edition >>-{{ checksum "./backend-checksums.txt" }}-{{ checksum "./frontend-checksums.txt" }}-{{ checksum "./modules-checksums.txt" }}
paths:
- /home/circleci/metabase/metabase/target/uberjar/metabase.jar
+ - /home/circleci/metabase/metabase/resources/version.properties
# Cache the maven installation of metabase-core
- save_cache:
key: metabase-core-{{ checksum "./backend-checksums.txt" }}
@@ -617,7 +620,7 @@ jobs:
- /home/circleci/.m2/repository/metabase-core
# Cache the built drivers
- save_cache:
- key: drivers-v3-{{ checksum "./modules-checksums.txt" }}-{{ checksum "./backend-checksums.txt" }}
+ key: drivers-v4-{{ checksum "./modules-checksums.txt" }}-{{ checksum "./backend-checksums.txt" }}
paths:
- /home/circleci/metabase/metabase/modules/drivers/bigquery/target
- /home/circleci/metabase/metabase/modules/drivers/druid/target
@@ -634,7 +637,7 @@ jobs:
- /home/circleci/metabase/metabase/modules/drivers/vertica/target
# Cache the built frontend
- save_cache:
- key: frontend-{{ checksum "./frontend-checksums.txt" }}
+ key: frontend-v4-{{ checksum "./frontend-checksums.txt" }}
paths:
- /home/circleci/metabase/metabase/resources/frontend_client
@@ -665,12 +668,7 @@ jobs:
before-steps:
- restore_cache:
keys:
- - uberjar-<< parameters.edition >>-{{ checksum "./backend-checksums.txt" }}
- - run:
- name: Generate version file
- environment:
- MB_EDITION: << parameters.edition >>
- command: ./bin/build version
+ - uberjar-v4-<< parameters.edition >>-{{ checksum "./backend-checksums.txt" }}
- store_artifacts:
path: /home/circleci/metabase/metabase/cypress
- store_test_results:
diff --git a/Dockerfile b/Dockerfile
index 55d9aaef4f8..f2020aed70b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -45,7 +45,7 @@ RUN yarn
ADD . .
# build the app
-RUN bin/build
+RUN INTERACTIVE=false bin/build
# import AWS RDS cert into /etc/ssl/certs/java/cacerts
ADD https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem .
diff --git a/OSX/deps.edn b/OSX/deps.edn
index 20c635d42cd..b1c58d89fe7 100644
--- a/OSX/deps.edn
+++ b/OSX/deps.edn
@@ -1,12 +1,10 @@
{:paths ["./"]
:deps
- {org.clojure/data.xml {:mvn/version "0.0.8"}
- cheshire {:mvn/version "5.8.1"}
- clj-http {:mvn/version "3.9.1"}
- clj-tagsoup {:mvn/version "0.3.0"}
- commons-io/commons-io {:mvn/version "2.6"}
- colorize {:mvn/version "0.1.1"}
- environ {:mvn/version "1.1.0"}
- hiccup {:mvn/version "1.0.5"}
- org.flatland/ordered {:mvn/version "1.5.7"}}}
+ {common/common {:local/root "../bin/common"}
+ org.clojure/data.xml {:mvn/version "0.0.8"}
+ cheshire/cheshire {:mvn/version "5.8.1"}
+ clj-http/clj-http {:mvn/version "3.9.1"}
+ clj-tagsoup/clj-tagsoup {:mvn/version "0.3.0"}
+ hiccup/hiccup {:mvn/version "1.0.5"}
+ org.flatland/ordered {:mvn/version "1.5.7"}}}
diff --git a/OSX/macos_release.clj b/OSX/macos_release.clj
index 5f3b7712ed5..2a486e83305 100644
--- a/OSX/macos_release.clj
+++ b/OSX/macos_release.clj
@@ -4,13 +4,11 @@
[macos-release
[build :as build]
[codesign :as codesign]
- [common :as c]
[create-dmg :as create-dmg]
[notarize :as notarize]
[sparkle-artifacts :as sparkle-artifacts]
- [upload :as upload]]))
-
-(set! *warn-on-reflection* true)
+ [upload :as upload]]
+ [metabuild-common.core :as u]))
(def ^:private steps*
(ordered-map/ordered-map
@@ -29,17 +27,13 @@
(thunk)))
(defn- do-steps! [steps]
- (c/announce "Running steps: %s" steps)
+ (u/announce "Running steps: %s" steps)
(doseq [step-name steps]
(do-step! step-name))
- (c/announce "Success."))
+ (u/announce "Success."))
(defn -main [& steps]
- (let [steps (or (seq steps)
- (keys steps*))]
- (try
- (do-steps! steps)
- (catch Throwable e
- (println (colorize/red (pr-str e)))
- (System/exit -1))))
- (System/exit 0))
+ (u/exit-when-finished-nonzero-on-exception
+ (let [steps (or (seq steps)
+ (keys steps*))]
+ (do-steps! steps))))
diff --git a/OSX/macos_release/common.clj b/OSX/macos_release/common.clj
index af64e1ebee0..4459042e06d 100644
--- a/OSX/macos_release/common.clj
+++ b/OSX/macos_release/common.clj
@@ -1,9 +1,23 @@
(ns macos-release.common
(:require [clojure.string :as str]
- [colorize.core :as colorize]
- [environ.core :as env])
- (:import [java.io BufferedReader File InputStreamReader]
- org.apache.commons.io.FileUtils))
+ [environ.core :as env]
+ [metabuild-common.core :as u]
+ [potemkin :as p])
+ (:import java.io.File))
+
+(comment u/keep-me)
+
+(p/import-vars
+ [u
+ announce
+ assert-file-exists
+ copy-file!
+ create-directory-unless-exists!
+ delete-file!
+ safe-println
+ sh
+ sh*
+ step])
(def ^String macos-source-dir
"e.g. /Users/cam/metabase/OSX"
@@ -20,142 +34,19 @@
"e.g. /Users/cam/metabase/osx-artifacts"
(str root-directory "/osx-artifacts"))
-;;; ---------------------------------------------------- Util Fns ----------------------------------------------------
-
-(def ^:dynamic *steps* [])
-
-(def ^:private step-indent (str/join (repeat 2 \space)))
-
-(defn- steps-indent []
- (str/join (repeat (count *steps*) step-indent)))
-
-(defn safe-println [& args]
- (locking println
- (print (steps-indent))
- (apply println args)))
-
-(defn announce
- "Like `println` + `format`, but outputs text in green. Use this for printing messages such as when starting build
- steps."
- ([s]
- (safe-println (colorize/magenta s)))
-
- ([format-string & args]
- (announce (apply format (str format-string) args))))
-
-(defn do-step [step thunk]
- (safe-println (colorize/green (str step)))
- (binding [*steps* (conj *steps* step)]
- (try
- (thunk)
- (catch Throwable e
- (throw (ex-info (str step) {} e))))))
-
-(defmacro step {:style/indent 1} [step & body]
- `(do-step ~step (fn [] ~@body)))
-
(defn exists? [^String filename]
(when filename
(.exists (File. filename))))
-(defn assert-file-exists
- "If file with `filename` exists, return `filename` as is; otherwise, throw Exception."
- ^String [filename & [message]]
- (when-not (exists? filename)
- (throw (ex-info (format "File %s does not exist. %s" (pr-str filename) (or message "")) {:filename filename})))
- (str filename))
-
-(defn create-directory-unless-exists! [^String dir]
- (when-not (exists? dir)
- (step (format "Creating directory %s..." dir)
- (.mkdirs (File. dir))))
- dir)
-
(defn artifact
"Return the full path of a file in the build artifacts directory."
^String [filename]
- (create-directory-unless-exists! artifacts-directory)
+ (when-not (u/file-exists? artifacts-directory)
+ (create-directory-unless-exists! artifacts-directory))
(str artifacts-directory "/" filename))
-(defn delete-file!
- "Delete a file or directory if it exists."
- ([^String filename]
- (step (format "Deleting %s..." filename)
- (if (exists? filename)
- (let [file (File. filename)]
- (if (.isDirectory file)
- (FileUtils/deleteDirectory file)
- (.delete file))
- (safe-println (format "Deleted %s." filename)))
- (safe-println (format "Don't need to delete %s, file does not exist." filename)))
- (assert (not (exists? filename)))))
-
- ([file & more]
- (dorun (map delete-file! (cons file more)))))
-
-(declare sh)
-
-(defn copy-file! [^String source ^String dest]
- (let [source-file (File. (assert-file-exists source))
- dest-file (File. dest)]
- ;; Use native `cp` rather than FileUtils or the like because codesigning is broken when you use those because they
- ;; don't preserve symlinks or something like that.
- (if (.isDirectory source-file)
- (step (format "Copying directory %s -> %s" source dest)
- (sh "cp" "-R" source dest))
- (step (format "Copying file %s -> %s" source dest)
- (sh "cp" source dest))))
- (assert-file-exists dest))
-
-(defn- read-lines [^java.io.BufferedReader reader {:keys [quiet? err?]}]
- (loop [lines []]
- (if-let [line (.readLine reader)]
- (do
- (when-not quiet?
- (safe-println (if err? (colorize/red line) line)))
- (recur (conj lines line)))
- lines)))
-
-(defn- deref-with-timeout [dereffable timeout-ms]
- (let [result (deref dereffable timeout-ms ::timed-out)]
- (when (= result ::timed-out)
- (throw (ex-info (format "Timed out after %d ms." timeout-ms) {})))
- result))
-
-(def ^:private command-timeout-ms (* 15 60 1000)) ; 15 minutes
-
-(defn sh*
- "Run a shell command. Like `clojure.java.shell/sh`, but prints output to stdout/stderr and returns results as a vector
- of lines."
- {:arglists '([cmd & args] [{:keys [dir quiet?]} cmd & args])}
- [& args]
- (step (colorize/blue (str "$ " (str/join " " (map (comp pr-str str) args))))
- (let [[opts & args] (if (map? (first args))
- args
- (cons nil args))
- {:keys [dir]} opts
- cmd-array (into-array (map str args))
- proc (.exec (Runtime/getRuntime) ^"[Ljava.lang.String;" cmd-array nil ^File (when dir (File. ^String dir)))]
- (with-open [out-reader (BufferedReader. (InputStreamReader. (.getInputStream proc)))
- err-reader (BufferedReader. (InputStreamReader. (.getErrorStream proc)))]
- (let [exit-code (future (.waitFor proc))
- out (future (read-lines out-reader opts))
- err (future (read-lines err-reader (assoc opts :err? true)))]
- {:exit (deref-with-timeout exit-code command-timeout-ms)
- :out (deref-with-timeout out command-timeout-ms)
- :err (deref-with-timeout err command-timeout-ms)})))))
-
-(defn sh
- "Run a shell command, returning its output if it returns zero or throwning an Exception if it returns non-zero."
- {:arglists '([cmd & args] [{:keys [dir quiet?]} cmd & args])}
- [& args]
- (let [{:keys [exit out err], :as response} (apply sh* args)]
- (if (zero? exit)
- (concat out err)
- (throw (ex-info (str/join "\n" (concat out err)) response)))))
-
(defn- version* []
- (let [[out] (sh (assert-file-exists (str root-directory "/bin/version")))
+ (let [[out] (sh "git" "describe" "--abbrev=0" "--tags")
[_ version] (re-find #"^v([\d.]+)" out)]
(when-not (seq version)
(throw (ex-info "Error parsing version." {:out out})))
diff --git a/OSX/release.sh b/OSX/release.sh
new file mode 100755
index 00000000000..f44195faa4f
--- /dev/null
+++ b/OSX/release.sh
@@ -0,0 +1,9 @@
+#! /usr/bin/env bash
+
+set -euo pipefail
+
+source "./bin/check-clojure-cli.sh"
+check_clojure_cli
+
+cd OSX/macos_release
+clojure -M -m macos-release $@
diff --git a/bin/build b/bin/build
index 0a83b8bac6a..7833eef8487 100755
--- a/bin/build
+++ b/bin/build
@@ -1,82 +1,9 @@
#!/usr/bin/env bash
-set -e
+set -euo pipefail
-MB_EDITION=${MB_EDITION:=oss}
+source "./bin/check-clojure-cli.sh"
+check_clojure_cli
-if [ "$MB_EDITION" != ee ] && [ "$MB_EDITION" != oss ]; then
- echo "MB_EDITION must be either 'ee' or 'oss'."
- exit 1
-fi
-
-# Generate the resources/version.properties file
-version() {
- VERSION_INFO=$(./bin/version)
- IFS=', ' read -a info <<< ${VERSION_INFO}
-
- echo "Tagging uberjar with version '$VERSION_INFO'..."
-
- # Ok, now generate the appropriate version.properties file.
- echo "tag=${info[0]}" > resources/version.properties
- echo "hash=${info[1]}" >> resources/version.properties
- echo "branch=${info[2]}" >> resources/version.properties
- echo "date=${info[3]}" >> resources/version.properties
-}
-
-frontend-deps() {
- echo "Running 'yarn' to download javascript dependencies..." &&
- yarn
-}
-
-frontend() {
- frontend-deps
- echo "Running 'webpack' with NODE_ENV=production assemble and minify frontend assets..." &&
- NODE_ENV=production ./node_modules/.bin/webpack --bail
-}
-
-frontend-fast() {
- frontend-deps
- echo "Running 'webpack' with NODE_ENV=development to assemble frontend assets..." &&
- NODE_ENV=development ./node_modules/.bin/webpack --bail --devtool eval
-}
-
-translations() {
- frontend-deps
- echo "Running './bin/i18n/build-translation-resources' to build translation resources..."
- if ! ./bin/i18n/build-translation-resources; then
- echo "Building translation resources failed, please install 'gettext', or build without translations by running './bin/build no-translations'."
- exit 1
- fi
-}
-
-drivers() {
- echo "Building Metabase drivers..."
- ./bin/build-drivers.sh
-}
-
-uberjar() {
- lein clean
- echo "Running 'lein with-profile +$MB_EDITION uberjar'..."
- lein with-profile +$MB_EDITION uberjar
-}
-
-all() {
- version && translations && frontend && drivers && uberjar
-}
-
-no-translations() {
- version && frontend && drivers && uberjar
-}
-
-# Default to running all but let someone specify one or more sub-tasks to run instead if desired
-# e.g.
-# ./bin/build # do everything
-# ./bin/build version # just update version.properties
-# ./bin/build version uberjar # just update version.properties and build uberjar
-if [ "$1" ]; then
- for cmd in "$@"; do
- $cmd
- done
-else
- all
-fi
+cd bin/build-mb
+clojure -M -m build $@
diff --git a/bin/build-drivers/build_drivers.clj b/bin/build-drivers/build_drivers.clj
index d65dfee59a5..7dba107c801 100644
--- a/bin/build-drivers/build_drivers.clj
+++ b/bin/build-drivers/build_drivers.clj
@@ -1,15 +1,13 @@
(ns build-drivers
"Entrypoint for `bin/build-drivers.sh`. Builds all drivers, if needed."
- (:require [build-drivers
- [build-driver :as build-driver]
- [common :as c]]
+ (:require [build-drivers.build-driver :as build-driver]
[clojure.java.io :as io]
[metabuild-common.core :as u]))
(defn- all-drivers []
- (map keyword (.list (io/file (c/filename c/project-root-directory "modules" "drivers")))))
+ (map keyword (.list (io/file (u/filename u/project-root-directory "modules" "drivers")))))
-(defn- build-drivers! []
+(defn build-drivers! []
(u/step "Building all drivers"
(doseq [driver (all-drivers)]
(build-driver/build-driver! driver))
diff --git a/bin/build-drivers/build_drivers/build_driver.clj b/bin/build-drivers/build_drivers/build_driver.clj
index c067cbc5589..8ce49da879f 100644
--- a/bin/build-drivers/build_drivers/build_driver.clj
+++ b/bin/build-drivers/build_drivers/build_driver.clj
@@ -81,7 +81,7 @@
([target source]
(u/step (format "Remove classes from %s that are present in %s and recompress" target source)
- (u/sh {:dir c/project-root-directory}
+ (u/sh {:dir u/project-root-directory}
"lein"
"strip-and-compress"
(u/assert-file-exists target)
diff --git a/bin/build-drivers/build_drivers/checksum.clj b/bin/build-drivers/build_drivers/checksum.clj
index b6f8e2ae6f9..eedd60ccc1e 100644
--- a/bin/build-drivers/build_drivers/checksum.clj
+++ b/bin/build-drivers/build_drivers/checksum.clj
@@ -25,15 +25,15 @@
(defn- metabase-source-paths []
(sort
(cons
- (c/filename c/project-root-directory "project.clj")
+ (u/filename u/project-root-directory "project.clj")
(mapcat (fn [dir]
(try
(u/find-files dir #(str/ends-with? % ".clj"))
(catch Throwable _
[])))
- [(c/filename c/project-root-directory "src")
- (c/filename c/project-root-directory "enterprise" "backend" "src")
- (c/filename c/project-root-directory "backend" "mbql")]))))
+ [(u/filename u/project-root-directory "src")
+ (u/filename u/project-root-directory "enterprise" "backend" "src")
+ (u/filename u/project-root-directory "backend" "mbql")]))))
(defn metabase-source-checksum
"Checksum of Metabase backend source files and `project.clj`."
@@ -58,7 +58,7 @@
(u/find-files (c/driver-project-dir driver)
(fn [path]
(or (and (str/ends-with? path ".clj")
- (not (str/starts-with? path (c/filename (c/driver-project-dir driver) "test"))))
+ (not (str/starts-with? path (u/filename (c/driver-project-dir driver) "test"))))
(str/ends-with? path ".yaml")))))
(defn driver-checksum
diff --git a/bin/build-drivers/build_drivers/common.clj b/bin/build-drivers/build_drivers/common.clj
index 953effd1260..2af68af2d07 100644
--- a/bin/build-drivers/build_drivers/common.clj
+++ b/bin/build-drivers/build_drivers/common.clj
@@ -1,28 +1,10 @@
(ns build-drivers.common
"Shared constants and functions related to source and artifact paths used throughout this code."
- (:require [clojure.string :as str]
- [environ.core :as env]
- [metabuild-common.core :as u])
- (:import java.io.File))
-
-;; since this file is used pretty much everywhere, this seemed like a good place to put this.
-(set! *warn-on-reflection* true)
-
-(when-not (str/ends-with? (env/env :user-dir) "/build-drivers")
- (throw (ex-info "Please run build-driver scripts from the `bin/build-drivers` directory e.g. `cd bin/build-drivers; clojure -m build-driver`"
- {:user-dir (env/env :user-dir)})))
-
-(defn ^:deprecated filename
- "DEPRECATED -- use u/filename instead."
- [& path-components]
- (apply u/filename path-components))
-
-(def ^String project-root-directory
- "e.g. /Users/cam/metabase"
- (.. (File. ^String (env/env :user-dir)) getParentFile getParentFile getAbsolutePath))
+ (:require [environ.core :as env]
+ [metabuild-common.core :as u]))
(def ^String maven-repository-path
- (filename (env/env :user-home) ".m2" "repository"))
+ (u/filename (env/env :user-home) ".m2" "repository"))
;;; -------------------------------------------------- Driver Paths --------------------------------------------------
@@ -30,7 +12,7 @@
(defn driver-project-dir
"e.g. \"/home/cam/metabase/modules/drivers/redshift\""
[driver]
- (filename project-root-directory "modules" "drivers" (name driver)))
+ (u/filename u/project-root-directory "modules" "drivers" (name driver)))
(defn driver-jar-name
"e.g. \"redshift.metabase-driver.jar\""
@@ -39,34 +21,34 @@
(defn driver-target-directory
[driver]
- (filename (driver-project-dir driver) "target"))
+ (u/filename (driver-project-dir driver) "target"))
(defn driver-jar-build-path
"e.g. \"/home/cam/metabase/modules/drivers/redshift/target/uberjar/redshift.metabase-driver.jar\""
[driver]
- (filename (driver-target-directory driver) "uberjar" (driver-jar-name driver)))
+ (u/filename (driver-target-directory driver) "uberjar" (driver-jar-name driver)))
(def ^String driver-jar-destination-directory
- (filename project-root-directory "resources" "modules"))
+ (u/filename u/project-root-directory "resources" "modules"))
(defn driver-jar-destination-path
"e.g. \"/home/cam/metabase/resources/modules/redshift.metabase-driver.jar\""
[driver]
- (filename driver-jar-destination-directory (driver-jar-name driver)))
+ (u/filename driver-jar-destination-directory (driver-jar-name driver)))
(defn driver-checksum-filename
"e.g. \"/home/cam/metabase/modules/drivers/redshift/target/checksum.txt\""
[driver]
- (filename (driver-project-dir driver) "target" "checksum.txt")) ; TODO - rename to checksum.md5
+ (u/filename (driver-project-dir driver) "target" "checksum.txt")) ; TODO - rename to checksum.md5
(defn driver-plugin-manifest-filename
"e.g. \"/home/cam/metabase/modules/drivers/bigquery/resources/plugin-manifest.yaml\""
[driver]
- (filename (driver-project-dir driver) "resources" "metabase-plugin.yaml"))
+ (u/filename (driver-project-dir driver) "resources" "metabase-plugin.yaml"))
;;; ------------------------------------------ Metabase Local Install Paths ------------------------------------------
(def ^String metabase-uberjar-path
"e.g. \"home/cam/metabase/target/uberjar/metabase.jar\""
- (filename project-root-directory "target" "uberjar" "metabase.jar"))
+ (u/filename u/project-root-directory "target" "uberjar" "metabase.jar"))
diff --git a/bin/build-drivers/build_drivers/install_driver_locally.clj b/bin/build-drivers/build_drivers/install_driver_locally.clj
index 484e880cc90..c2cb3140721 100644
--- a/bin/build-drivers/build_drivers/install_driver_locally.clj
+++ b/bin/build-drivers/build_drivers/install_driver_locally.clj
@@ -9,10 +9,10 @@
[metabuild-common.core :as u]))
(defn- driver-local-install-path [driver]
- (c/filename c/maven-repository-path "metabase" (format "%s-driver" (name driver))))
+ (u/filename c/maven-repository-path "metabase" (format "%s-driver" (name driver))))
(defn- driver-local-install-checksum-filename [driver]
- (c/filename (driver-local-install-path driver) "checksum.md5"))
+ (u/filename (driver-local-install-path driver) "checksum.md5"))
(defn clean!
"Delete local Maven installation of the library version of `driver`."
diff --git a/bin/build-drivers/build_drivers/metabase.clj b/bin/build-drivers/build_drivers/metabase.clj
index 5ba36429de0..9c209e10fbc 100644
--- a/bin/build-drivers/build_drivers/metabase.clj
+++ b/bin/build-drivers/build_drivers/metabase.clj
@@ -10,10 +10,10 @@
(str c/metabase-uberjar-path ".md5"))
(def ^String ^:private metabase-core-install-path
- (c/filename c/maven-repository-path "metabase-core"))
+ (u/filename c/maven-repository-path "metabase-core"))
(def ^String ^:private metabase-core-checksum-path
- (c/filename metabase-core-install-path "checksum.md5"))
+ (u/filename metabase-core-install-path "checksum.md5"))
(defn metabase-core-checksum-matches? []
(u/step "Determine whether Metabase source files checksum has changed since last install of metabase-core"
@@ -35,8 +35,8 @@
(u/announce "Up-to-date metabase-core already installed to local Maven repo")
(do
(delete-metabase-core-install!)
- (u/sh {:dir c/project-root-directory} "lein" "clean")
- (u/sh {:dir c/project-root-directory} "lein" "install-for-building-drivers")
+ (u/sh {:dir u/project-root-directory} "lein" "clean")
+ (u/sh {:dir u/project-root-directory} "lein" "install-for-building-drivers")
(u/step "Save checksum for local installation of metabase-core"
(spit metabase-core-checksum-path (checksum/metabase-source-checksum)))
(u/announce "metabase-core dep installed to local Maven repo successfully.")))))
@@ -57,7 +57,7 @@
(defn- delete-metabase-uberjar! []
(u/step "Delete exist metabase uberjar"
- (u/delete-file! (c/filename c/project-root-directory "target"))))
+ (u/delete-file! (u/filename u/project-root-directory "target"))))
(defn- build-metabase-uberjar! []
(u/step "Build Metabase uberjar if needed"
@@ -65,8 +65,8 @@
(u/announce "Update-to-date uberjar already built")
(do
(delete-metabase-uberjar!)
- (u/sh {:dir c/project-root-directory} "lein" "clean")
- (u/sh {:dir c/project-root-directory} "lein" "uberjar")
+ (u/sh {:dir u/project-root-directory} "lein" "clean")
+ (u/sh {:dir u/project-root-directory} "lein" "uberjar")
(u/step "Save checksum for Metabase uberar"
(spit uberjar-checksum-path (checksum/metabase-source-checksum)))
(u/announce "Metabase uberjar built successfully")))))
diff --git a/bin/build-drivers/deps.edn b/bin/build-drivers/deps.edn
index 0e9cf9483eb..bb291868c5a 100644
--- a/bin/build-drivers/deps.edn
+++ b/bin/build-drivers/deps.edn
@@ -1,13 +1,9 @@
-{:paths ["./" "../"]
+{:paths ["./"]
:deps
- {cheshire/cheshire {:mvn/version "5.8.1"}
+ {common/common {:local/root "../common"}
+ cheshire/cheshire {:mvn/version "5.8.1"}
commons-codec/commons-codec {:mvn/version "1.14"}
- commons-io/commons-io {:mvn/version "2.6"}
- colorize/colorize {:mvn/version "0.1.1"}
- environ/environ {:mvn/version "1.1.0"}
hiccup/hiccup {:mvn/version "1.0.5"}
io.forward/yaml {:mvn/version "1.0.9"}
- org.flatland/ordered {:mvn/version "1.5.7"}
- potemkin/potemkin {:mvn/version "0.4.5"}
stencil/stencil {:mvn/version "0.5.0"}}}
diff --git a/bin/build-mb/build.clj b/bin/build-mb/build.clj
new file mode 100644
index 00000000000..432d6a1b3f6
--- /dev/null
+++ b/bin/build-mb/build.clj
@@ -0,0 +1,92 @@
+(ns build
+ (:require [build-drivers :as build-drivers]
+ [build.version-info :as version-info]
+ [clojure.string :as str]
+ [environ.core :as env]
+ [flatland.ordered.map :as ordered-map]
+ [metabuild-common
+ [core :as u]
+ [java :as java]]))
+
+(defn- build-translation-resources!
+ []
+ (u/step "Build translation resources"
+ (java/check-java-8)
+ (u/sh {:dir u/project-root-directory} "./bin/i18n/build-translation-resources")
+ (u/announce "Translation resources built successfully.")))
+
+(defn- build-frontend! []
+ (u/step "Build frontend"
+ (u/step "Run 'yarn' to download javascript dependencies"
+ (u/sh {:dir u/project-root-directory} "yarn"))
+ (u/step "Run 'webpack' with NODE_ENV=production to assemble and minify frontend assets"
+ (u/sh {:dir u/project-root-directory
+ :env {"PATH" (env/env :path)
+ "HOME" (env/env :user-home)
+ "NODE_ENV" "production"}}
+ "./node_modules/.bin/webpack" "--bail"))
+ (u/announce "Frontend built successfully.")))
+
+(def uberjar-filename (u/filename u/project-root-directory "target" "uberjar" "metabase.jar"))
+
+(defn- build-uberjar! [edition]
+ {:pre [(#{:ce :ee} edition)]}
+ ;; clojure scripts currently use :ee vs :ce but everything else uses :ee vs :oss
+ (let [profile (case edition
+ :ee "ee"
+ :ce "oss")]
+ (u/delete-file-if-exists! uberjar-filename)
+ (u/step "Build uberjar"
+ (u/sh {:dir u/project-root-directory} "lein" "clean")
+ (u/sh {:dir u/project-root-directory} "lein" "with-profile" (str \+ profile) "uberjar")
+ (u/assert-file-exists uberjar-filename)
+ (u/announce "Uberjar built successfully."))))
+
+(def all-steps
+ (ordered-map/ordered-map
+ :version (fn [{:keys [version]}]
+ (version-info/generate-version-info-file! version))
+ :translations (fn [_]
+ (build-translation-resources!))
+ :frontend (fn [_]
+ (build-frontend!))
+ :drivers (fn [_]
+ (build-drivers/build-drivers!))
+ :uberjar (fn [{:keys [edition]}]
+ (build-uberjar! edition))))
+
+(defn build!
+ ([]
+ (build! nil))
+
+ ([{:keys [version edition steps]
+ :or {version (version-info/current-snapshot-version)
+ edition :ce
+ steps (keys all-steps)}}]
+ (u/step (format "Running build steps for %s version %s: %s"
+ (case edition
+ :ce "Community (OSS) Edition"
+ :ee "Enterprise Edition")
+ version
+ (str/join ", " (map name steps)))
+ (doseq [step-name steps
+ :let [step-fn (or (get all-steps (keyword step-name))
+ (throw (ex-info (format "Invalid step: %s" step-name)
+ {:step step-name
+ :valid-steps (keys all-steps)})))]]
+ (step-fn {:version version, :edition edition}))
+ (u/announce "All build steps finished."))))
+
+(defn- edition-from-env-var []
+ ;; MB_EDITION is either oss/ee, but the Clojure build scripts currently use :ce/:ee
+ (if-not (env/env :mb-edition)
+ :ce
+ (case (env/env :mb-edition)
+ "oss" :ce
+ "ee" :ee)))
+
+(defn -main [& steps]
+ (u/exit-when-finished-nonzero-on-exception
+ (build! (merge {:edition (edition-from-env-var)}
+ (when-let [steps (not-empty steps)]
+ {:steps steps})))))
diff --git a/bin/build-mb/build/version_info.clj b/bin/build-mb/build/version_info.clj
new file mode 100644
index 00000000000..011b3109325
--- /dev/null
+++ b/bin/build-mb/build/version_info.clj
@@ -0,0 +1,77 @@
+(ns build.version-info
+ (:require [clojure.string :as str]
+ [metabuild-common.core :as u]))
+
+(def version-properties-filename
+ (u/filename u/project-root-directory "resources" "version.properties"))
+
+(defn- shell-output-when-nonzero
+ "Call an external shell command, and return the first line that is output if it has a nonzero exit code. (Sometimes
+ `git` will fail, e.g. in CI where we delete the `.git` directory to reduce the workspace snapshot size.)"
+ [& args]
+ (let [{:keys [exit out]} (apply u/sh* args)]
+ (when (zero? exit)
+ (first out))))
+
+(defn git-hash []
+ ;; first 7 letters of hash should be enough; that's what GitHub uses
+ (or (shell-output-when-nonzero "git" "show-ref" "--head" "--hash=7" "head")
+ "?"))
+
+(defn git-branch []
+ (or (shell-output-when-nonzero "git" "symbolic-ref" "--short" "HEAD")
+ "?"))
+
+(defn git-last-commit-date []
+ (or (shell-output-when-nonzero "git" "log" "-1" "--pretty=%ad" "--date=short")
+ "?"))
+
+(defn- version-properties [version]
+ (str/join "\n" (for [[k v] {:tag (if-not (str/starts-with? version "v")
+ (str \v version)
+ version)
+ :hash (git-hash)
+ :branch (git-branch)
+ :date (git-last-commit-date)}]
+ (str (name k) \= v))))
+
+(defn most-recent-tag []
+ (shell-output-when-nonzero "git" "describe" "--abbrev=0" "--tags"))
+
+(defn most-recent-tag-parts []
+ (when-let [tag (most-recent-tag)]
+ (for [part (str/split tag #"\.")
+ :let [numeric-part (re-find #"\d+" part)]
+ :when (seq numeric-part)]
+ (Integer/parseInt numeric-part))))
+
+(defn current-snapshot-version
+ "Attempt to come up with a snapshot version for builds that aren't given explicit version info based on the most
+ recent tag. e.g.
+
+ v0.37.1 -> v0.37.2-SNAPSHOT
+
+ For builds from `master`, increment the minor version instead e.g.
+
+ v0.37.1 -> v0.38.0-SNAPSHOT"
+ []
+ (if-let [tag-parts (most-recent-tag-parts)]
+ (let [[major minor patch] tag-parts
+ major (or major 0)
+ [minor patch] (if (= (git-branch) "master")
+ [(inc (or minor 0)) 0]
+ [(or minor 0) (inc (or patch 0))])]
+ (format "v%d.%d.%d-SNAPSHOT" major minor patch))
+ "UNKNOWN"))
+
+(defn generate-version-info-file!
+ "Generate version.properties file"
+ ([]
+ (generate-version-info-file! (current-snapshot-version)))
+
+ ([version]
+ (u/delete-file-if-exists! version-properties-filename)
+ (u/step (format "Generate version.properties file for version %s" version)
+ (spit version-properties-filename (version-properties version))
+ (u/assert-file-exists version-properties-filename)
+ (u/announce "version.properties generated successfully."))))
diff --git a/bin/build-mb/deps.edn b/bin/build-mb/deps.edn
new file mode 100644
index 00000000000..434f9aeabe2
--- /dev/null
+++ b/bin/build-mb/deps.edn
@@ -0,0 +1,6 @@
+{:paths ["./"]
+
+ :deps
+ {common/common {:local/root "../common"}
+ build-drivers/build-drivers {:local/root "../build-drivers"}
+ org.flatland/ordered {:mvn/version "1.5.7"}}}
diff --git a/bin/common/deps.edn b/bin/common/deps.edn
new file mode 100644
index 00000000000..31ea40aa439
--- /dev/null
+++ b/bin/common/deps.edn
@@ -0,0 +1,7 @@
+{:paths ["./"]
+
+ :deps
+ {commons-io/commons-io {:mvn/version "2.6"}
+ colorize/colorize {:mvn/version "0.1.1"}
+ environ/environ {:mvn/version "1.1.0"}
+ potemkin/potemkin {:mvn/version "0.4.5"}}}
diff --git a/bin/metabuild_common/aws.clj b/bin/common/metabuild_common/aws.clj
similarity index 100%
rename from bin/metabuild_common/aws.clj
rename to bin/common/metabuild_common/aws.clj
diff --git a/bin/metabuild_common/core.clj b/bin/common/metabuild_common/core.clj
similarity index 86%
rename from bin/metabuild_common/core.clj
rename to bin/common/metabuild_common/core.clj
index c55aa1e36ed..db228554d51 100644
--- a/bin/metabuild_common/core.clj
+++ b/bin/common/metabuild_common/core.clj
@@ -10,6 +10,9 @@
[steps :as steps]]
[potemkin :as p]))
+;; since this file is used pretty much everywhere, this seemed like a good place to put this.
+(set! *warn-on-reflection* true)
+
(comment aws/keep-me
entrypoint/keep-me
build.env/env
@@ -38,7 +41,8 @@
delete-file-if-exists!
file-exists?
filename
- find-files]
+ find-files
+ project-root-directory]
[input
interactive?
diff --git a/bin/metabuild_common/entrypoint.clj b/bin/common/metabuild_common/entrypoint.clj
similarity index 100%
rename from bin/metabuild_common/entrypoint.clj
rename to bin/common/metabuild_common/entrypoint.clj
diff --git a/bin/metabuild_common/env.clj b/bin/common/metabuild_common/env.clj
similarity index 100%
rename from bin/metabuild_common/env.clj
rename to bin/common/metabuild_common/env.clj
diff --git a/bin/metabuild_common/files.clj b/bin/common/metabuild_common/files.clj
similarity index 83%
rename from bin/metabuild_common/files.clj
rename to bin/common/metabuild_common/files.clj
index 007831ec61a..3bfd9998647 100644
--- a/bin/metabuild_common/files.clj
+++ b/bin/common/metabuild_common/files.clj
@@ -1,5 +1,6 @@
(ns metabuild-common.files
(:require [clojure.string :as str]
+ [environ.core :as env]
[metabuild-common
[output :as out]
[shell :as sh]
@@ -94,3 +95,19 @@
;; -> \"usr/cam/.emacs.d/init.el\""
[& path-components]
(str/join File/separatorChar path-components))
+
+(def ^String project-root-directory
+ "Root directory of the Metabase repo, e.g. `/users/cam/metabase`. Determined by finding the directory that has
+ `project.clj` in it."
+ (loop [^File dir (File. ^String (env/env :user-dir))]
+ (cond
+ (file-exists? (filename (.getAbsolutePath dir) "project.clj"))
+ (.getAbsolutePath dir)
+
+ (.getParentFile dir)
+ (recur (.getParentFile dir))
+
+ :else
+ (throw (ex-info (format "Can't find project root directory: no parent directory of %s has a project.clj file"
+ (env/env :user-dir))
+ {:dir (env/env :user-dir)})))))
diff --git a/bin/metabuild_common/input.clj b/bin/common/metabuild_common/input.clj
similarity index 90%
rename from bin/metabuild_common/input.clj
rename to bin/common/metabuild_common/input.clj
index edb4ae7a2d7..dafc59c4f82 100644
--- a/bin/metabuild_common/input.clj
+++ b/bin/common/metabuild_common/input.clj
@@ -4,13 +4,13 @@
[metabuild-common.output :as out]))
(defn interactive?
- "Whether we're running these scripts interactively, and can prompt the user for input. By default, this is true, but
- if the env var `INTERACTIVE=false` is set, these scripts will not prompt for input. Be sure to set this when running
- scripts in CI or other places that automate them."
+ "Whether we're running these scripts interactively, and can prompt the user for input. By default, this is
+ true (except when running in CircleCI), but if the env var `INTERACTIVE=false` is set, these scripts will not prompt
+ for input. Be sure to set this when running scripts in CI or other places that automate them."
[]
(if-let [env-var (env/env :interactive)]
(Boolean/parseBoolean env-var)
- true))
+ (not (:circleci env/env))))
(defn read-line-with-prompt
"Prompt for and read a value from stdin. Accepts two options: `:default`, which is the default value to use if the
diff --git a/bin/common/metabuild_common/java.clj b/bin/common/metabuild_common/java.clj
new file mode 100644
index 00000000000..970c842cebb
--- /dev/null
+++ b/bin/common/metabuild_common/java.clj
@@ -0,0 +1,18 @@
+(ns metabuild-common.java
+ (:require [metabuild-common.core :as u]))
+
+(defn java-version
+ "Get `major.minor` version of the `java` command, e.g. `14.0` or `1.8` (Java 8)."
+ []
+ (when-let [[_ version] (re-find #"version \"(\d+\.\d+)\..*\"" (first (u/sh "java" "-version")))]
+ (Double/parseDouble version)))
+
+(defn check-java-8 []
+ (u/step "Verify Java version is Java 8"
+ (let [version (or (java-version)
+ (throw (Exception. "Unable to determine Java major version.")))]
+ ;; TODO -- is it possible to invoke `jabba` or some other command programmatically, or prompt for a different
+ ;; `JAVA_HOME`/`PATH` to use?
+ (when-not (#{1.8 8} version)
+ (throw (Exception. "The Metabase build script currently requires Java 8 to run. Please change your Java version and try again.")))
+ (u/announce "Java version is Java 8."))))
diff --git a/bin/metabuild_common/output.clj b/bin/common/metabuild_common/output.clj
similarity index 100%
rename from bin/metabuild_common/output.clj
rename to bin/common/metabuild_common/output.clj
diff --git a/bin/metabuild_common/shell.clj b/bin/common/metabuild_common/shell.clj
similarity index 100%
rename from bin/metabuild_common/shell.clj
rename to bin/common/metabuild_common/shell.clj
diff --git a/bin/metabuild_common/steps.clj b/bin/common/metabuild_common/steps.clj
similarity index 100%
rename from bin/metabuild_common/steps.clj
rename to bin/common/metabuild_common/steps.clj
diff --git a/bin/release/deps.edn b/bin/release/deps.edn
index 52716c01fb9..e4e3bb9f579 100644
--- a/bin/release/deps.edn
+++ b/bin/release/deps.edn
@@ -1,13 +1,11 @@
-{:paths ["./" "../"]
+{:paths ["./"]
:deps
- {cheshire/cheshire {:mvn/version "5.8.1"}
- clj-http/clj-http {:mvn/version "3.9.1"}
- commons-io/commons-io {:mvn/version "2.6"}
- colorize/colorize {:mvn/version "0.1.1"}
- enlive/enlive {:mvn/version "1.1.6"}
- environ/environ {:mvn/version "1.1.0"}
- hiccup/hiccup {:mvn/version "1.0.5"}
- org.flatland/ordered {:mvn/version "1.5.7"}
- potemkin/potemkin {:mvn/version "0.4.5"}
- stencil/stencil {:mvn/version "0.5.0"}}}
+ {common/common {:local/root "../common"}
+ build/build {:local/root "../build-mb"}
+ cheshire/cheshire {:mvn/version "5.8.1"}
+ clj-http/clj-http {:mvn/version "3.9.1"}
+ enlive/enlive {:mvn/version "1.1.6"}
+ hiccup/hiccup {:mvn/version "1.0.5"}
+ org.flatland/ordered {:mvn/version "1.5.7"}
+ stencil/stencil {:mvn/version "0.5.0"}}}
diff --git a/bin/release/release/check_prereqs.clj b/bin/release/release/check_prereqs.clj
index dfb9b292bf8..5c922f71d7c 100644
--- a/bin/release/release/check_prereqs.clj
+++ b/bin/release/release/check_prereqs.clj
@@ -1,7 +1,9 @@
(ns release.check-prereqs
(:require [clojure.string :as str]
[environ.core :as env]
- [metabuild-common.core :as u]))
+ [metabuild-common
+ [core :as u]
+ [java :as java]]))
(def ^:private required-commands
["git" "node" "yarn" "aws" "docker" "java" "wget" "shasum" "gettext" "zip"])
@@ -43,25 +45,9 @@
(throw (ex-info "Docker is not running. Please start it and try again." {})))
(u/announce "Docker is running.")))
-(defn- java-version
- "Get `major.minor` version of the `java` command, e.g. `14.0` or `1.8` (Java 8)."
- []
- (when-let [[_ version] (re-find #"version \"(\d+\.\d+)\..*\"" (first (u/sh "java" "-version")))]
- (Double/parseDouble version)))
-
-(defn- check-java-8 []
- (u/step "Verify Java version is Java 8"
- (let [version (or (java-version)
- (throw (Exception. "Unable to determine Java major version.")))]
- ;; TODO -- is it possible to invoke `jabba` or some other command programmatically, or prompt for a different
- ;; `JAVA_HOME`/`PATH` to use?
- (when-not (#{1.8 8} version)
- (throw (Exception. "The Metabase build script currently requires Java 8 to run. Please change your Java version and try again.")))
- (u/announce "Java version is Java 8."))))
-
(defn check-prereqs []
(u/step "Check prereqs"
(check-for-required-commands)
(check-for-required-env-vars)
(check-docker-is-running)
- (check-java-8)))
+ (java/check-java-8)))
diff --git a/bin/release/release/common/upload.clj b/bin/release/release/common/upload.clj
index 024874034d7..2e58f3c5575 100644
--- a/bin/release/release/common/upload.clj
+++ b/bin/release/release/common/upload.clj
@@ -1,5 +1,6 @@
(ns release.common.upload
- (:require [release.common :as c]))
+ (:require [metabuild-common.core :as u]
+ [release.common :as c]))
(defn upload-artifact!
"Upload an artifact to downloads.metabase.com and create a CloudFront invalidation."
diff --git a/bin/release/release/set_build_options.clj b/bin/release/release/set_build_options.clj
index dcc013bc065..1b12dbacab9 100644
--- a/bin/release/release/set_build_options.clj
+++ b/bin/release/release/set_build_options.clj
@@ -8,7 +8,7 @@
(loop []
(let [version (u/read-line-with-prompt "What version are we building (e.g. 0.36.0)?")
branch current-branch
- edition (case (first (c/version))
+ edition (case (first version)
\0 :ce
\1 :ee)]
(if-not (u/yes-or-no-prompt (format "Building %s version %s from branch %s. Is this correct?"
diff --git a/bin/release/release/uberjar.clj b/bin/release/release/uberjar.clj
index ee7535bc09e..48d0df7afe7 100644
--- a/bin/release/release/uberjar.clj
+++ b/bin/release/release/uberjar.clj
@@ -1,7 +1,6 @@
(ns release.uberjar
"Code related to building, pushing, and validating metabase.jar"
- (:require [clojure.string :as str]
- [environ.core :as env]
+ (:require [build :as build]
[metabuild-common.core :as u]
[release.common :as c]
[release.common
@@ -9,38 +8,11 @@
[http :as common.http]
[upload :as upload]]))
-(def ^:private bin-version-file
- (str c/root-directory "/bin/version"))
-
-(defn- update-version-info! []
- (u/step (format "Update %s if needed" (u/assert-file-exists bin-version-file))
- (u/step "Update version in bin/version"
- (let [lines (str/split-lines (slurp bin-version-file))
- updated-lines (for [line lines]
- (if (re-matches #".*VERSION=.*" line)
- (format "VERSION='v%s'" (c/version))
- line))]
- (if (= lines updated-lines)
- (u/announce "Correct version is already set.")
- (do
- (u/announce "Version set to 'v%s'" (c/version))
- (spit bin-version-file (str/join (interleave updated-lines (repeat "\n"))))
- (u/sh "chmod" "+x" bin-version-file)
- (u/step "Commit updated bin/version"
- (u/sh "git" "add" bin-version-file)
- (u/sh "git" "commit" "-m" (str \v (c/version)) bin-version-file))))))))
-
(defn build-uberjar! []
- (update-version-info!)
(u/step "Run bin/build to build uberjar"
- (u/delete-file! (str c/root-directory "/target"))
- (u/sh {:dir c/root-directory
- :env (merge {"JAVA_HOME" (env/env :java-home)
- "PATH" (env/env :path)
- "HOME" (env/env :user-home)}
- (when (= (c/edition) :ee)
- {"MB_EDITION" "ee"}))}
- "bin/build")
+ (u/delete-file-if-exists! (str c/root-directory "/target"))
+ (build/build! {:version (str \v (c/version))
+ :edition (c/edition)})
(u/step "Verify uberjar exists"
(u/assert-file-exists c/uberjar-path))))
@@ -54,7 +26,7 @@
(common.http/check-url-exists url)
(u/step (format "Check hash of %s" url)
(let [temp-location "/tmp/metabase.jar"]
- (u/delete-file! temp-location)
+ (u/delete-file-if-exists! temp-location)
(u/sh {:quiet? true} "wget" "--quiet" "--no-cache" "--output-document" temp-location url)
(let [uberjar-hash (hash/sha-256-sum c/uberjar-path)
url-hash (hash/sha-256-sum temp-location)]
diff --git a/bin/remove-drivers.sh b/bin/remove-drivers.sh
deleted file mode 100755
index c82c6db24a4..00000000000
--- a/bin/remove-drivers.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#! /usr/bin/env bash
-
-set -eu
-
-for file in `find resources plugins -name '*.metabase-driver.jar'`; do
- echo "Deleting $file..."
- rm "$file"
-done
diff --git a/bin/verify-driver b/bin/verify-driver
deleted file mode 100755
index 4e463736085..00000000000
--- a/bin/verify-driver
+++ /dev/null
@@ -1,16 +0,0 @@
-#! /usr/bin/env bash
-
-set -eo pipefail
-
-driver="$1"
-
-if [ ! "$driver" ]; then
- echo "Usage: ./bin/verify-driver [driver]"
- exit -1
-fi
-
-source "./bin/check-clojure-cli.sh"
-check_clojure_cli
-
-cd bin/build-drivers
-clojure -M -m verify-driver "$driver"
diff --git a/bin/version b/bin/version
deleted file mode 100755
index cb1ee17a443..00000000000
--- a/bin/version
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-VERSION='v0.37.0.2'
-
-# dynamically pull more interesting stuff from latest git commit
-HASH=$(git show-ref --head --hash=7 head) # first 7 letters of hash should be enough; that's what GitHub uses
-BRANCH=$(git rev-parse --abbrev-ref HEAD)
-DATE=$(git log -1 --pretty=%ad --date=short)
-
-# Return the version string used to describe this version of Metabase.
-echo "$VERSION $HASH $BRANCH $DATE"
diff --git a/docs/developers-guide-osx.md b/docs/developers-guide-osx.md
index da7ab9a6a5c..5da84029042 100644
--- a/docs/developers-guide-osx.md
+++ b/docs/developers-guide-osx.md
@@ -8,7 +8,7 @@ our [developers' guide](developers-guide.md).
<summary>
Steps
</summary>
-
+
### Building
The following steps need to be done before building the Mac App:
@@ -16,11 +16,11 @@ The following steps need to be done before building the Mac App:
1. Install XCode.
1. Add a JRE to the `/path/to/metabase/repo/OSX/Metabase/jre`
-
+
You must acquire a copy of a JRE (make sure you get a JRE rather than JDK) and move it to the correct location in the Mac App source directory so it can be included as part of the Mac App. To ship Java applications as Mac Apps, you must ship them with their own JRE. In this case we want to get a JRE from somewhere (more on this below) and move the `Contents/Home` directory from the JRE archive into `OSX/Metabase/jre`. (`OSX/Metabase` already exists inside the `metabase/metabase` repo.)
<details><summary>Option 1: Download from AdoptOpenJDK (currently broken -- do not use)</summary>
-
+
You can download a copy of a JRE from https://adoptopenjdk.net/releases.html?jvmVariant=hotspot — make sure you download a JRE rather than JDK. Move the `Contents/Home` directory from the JRE archive into `OSX/Metabase/jre`. (`OSX/Metabase` already exists inside the `metabase/metabase` repo.) For example:
```bash
@@ -33,7 +33,7 @@ The following steps need to be done before building the Mac App:
**VERY IMPORTANT!**
- Make sure the JRE version you use is one that is known to work successfully with notarization. We have found out the one linked above does not work.
+ Make sure the JRE version you use is one that is known to work successfully with notarization. We have found out the one linked above does not work.
I have found a nightly build that *does* work, but it's no longer available for download. Cam and Sameer both have copies of a JRE that is known to work. Refer to Option 2.
If you get notarization errors like
@@ -44,9 +44,9 @@ The following steps need to be done before building the Mac App:
Assuming the OpenJDK folks have resolved this issue going forward, you are fine to use whatever the latest JRE version available is. I have been using the HotSpot JRE instead of the OpenJ9 one but it ultimately shouldn't make a difference.
</details>
-
+
<details><summary>Option 2: Ask Cam or Sameer for known working JRE</summary>
-
+
Have Cam or Sameer ZIP up their `/path/to/metabase/repo/OSX/Metabase/jre` folder and send it to you. Don't try Option 1 until we know the issues are fixed
</details>
@@ -89,18 +89,18 @@ The following steps are prereqs for *releasing* the Mac App:
1) Add `Apple Developer ID Application Certificate` to your computer's keychain.
- 1) Generate a Certificate Signing Request from the Keychain Access app.
-
- 1) `Keychain Access` > `Certificate Assistant` > `Request a Certificate From a Certificate Authority`.
-
+ 1) Generate a Certificate Signing Request from the Keychain Access app.
+
+ 1) `Keychain Access` > `Certificate Assistant` > `Request a Certificate From a Certificate Authority`.
+
1) Enter the email associated with your Apple Developer account.
-
+
1) Leave "CA Email Address" blank
-
+
1) Choose "Save to Disk"
-
+
1) Have Sameer go to [the Apple Developer Site](https://developer.apple.com/account/mac/certificate/) and generate a `Developer ID Application` certificate for you by uploading the Certificate Signing Request you creating in the last step.
-
+
1) Load the generated certificate on your computer.
1) Export your Apple ID for building the app as `METABASE_MAC_APP_BUILD_APPLE_ID`. (This Apple ID must be part of the Metabase org in the Apple developer site. Ask Cam or Sameer to add you if it isn't.)
@@ -137,7 +137,7 @@ After following the configuration steps above, to build and release the app you
1. Make sure release is *published* on GitHub and release notes are ready. The script copies these for the update release notes.
-1. Make sure you're on the appropriate release branch locally. The script reads the version number from `./bin/version`
+1. Make sure you're on the appropriate release branch locally. The script reads the version number from the most recent tag
1. Copy latest uberjar to the Mac App build directory
@@ -148,8 +148,7 @@ After following the configuration steps above, to build and release the app you
1. Bundle entire app, and upload to s3
```bash
- cd OSX
- clojure -m macos-release
+ ./OSX/release.sh
```
-
+
**Important Note** Do not let your computer lock the screen while running the script — if the screen is locked, macOS will not allow the script to access your Apple Developer credentials from the Keychain (needed for notarization) and the script will fail.
diff --git a/src/metabase/config.clj b/src/metabase/config.clj
index b78315f6db5..672512c023c 100644
--- a/src/metabase/config.clj
+++ b/src/metabase/config.clj
@@ -1,7 +1,5 @@
(ns metabase.config
- (:require [clojure.java
- [io :as io]
- [shell :as shell]]
+ (:require [clojure.java.io :as io]
[clojure.string :as str]
[environ.core :as environ]
[metabase.plugins.classloader :as classloader])
@@ -68,19 +66,6 @@
(def ^Boolean is-test? "Are we running in `test` mode (i.e. via `lein test`)?" (= :test (config-kw :mb-run-mode)))
;;; Version stuff
-;; Metabase version is of the format `GIT-TAG (GIT-SHORT-HASH GIT-BRANCH)`
-
-(defn- version-info-from-shell-script []
- (try
- (let [[tag hash branch date] (-> (shell/sh "./bin/version") :out str/trim (str/split #" "))]
- {:tag (or tag "?")
- :hash (or hash "?")
- :branch (or branch "?")
- :date (or date "?")})
- ;; if ./bin/version fails (e.g., if we are developing on Windows) just return something so the whole thing doesn't
- ;; barf
- (catch Throwable _
- {:tag "?", :hash "?", :branch "?", :date "?"})))
(defn- version-info-from-properties-file []
(when-let [props-file (io/resource "version.properties")]
@@ -92,14 +77,11 @@
;; TODO - Can we make this `^:const`, so we don't have to read the file at launch when running from the uberjar?
(def mb-version-info
- "Information about the current version of Metabase.
- This comes from `resources/version.properties` for prod builds and is fetched from `git` via the `./bin/version`
- script for dev.
+ "Information about the current version of Metabase. Comes from `version.properties` which is generated by the build
+ script.
mb-version-info -> {:tag: \"v0.11.1\", :hash: \"afdf863\", :branch: \"about_metabase\", :date: \"2015-10-05\"}"
- (or (if is-prod?
- (version-info-from-properties-file)
- (version-info-from-shell-script))
+ (or (version-info-from-properties-file)
;; if version info is not defined for whatever reason
{}))
--
GitLab