From 9a82d7d939df39b23e0435bf1f27916d368284f8 Mon Sep 17 00:00:00 2001
From: Cam Saul <1455846+camsaul@users.noreply.github.com>
Date: Thu, 8 Oct 2020 13:51:39 -0700
Subject: [PATCH] New driver build scripts (#13383)
* New driver build scripts and fix Google driver build
* Install Clojure CLI in Circle CI
* Fix driver build script
* Build script fix :wrench:
* More dox
* Enforce minimum build Clojure CLI version and address feedback
* New metabuild-common directory for shared stuff. Address some PR feedback
---
.circleci/config.yml | 7 +-
.gitignore | 2 +-
bin/build-driver.sh | 235 +-----------------
bin/build-drivers.sh | 46 +---
bin/build-drivers/README.md | 45 ++++
bin/build-drivers/build_driver.clj | 10 +
bin/build-drivers/build_drivers.clj | 20 ++
.../build_drivers/build_driver.clj | 168 +++++++++++++
bin/build-drivers/build_drivers/checksum.clj | 76 ++++++
bin/build-drivers/build_drivers/common.clj | 75 ++++++
.../build_drivers/install_driver_locally.clj | 50 ++++
bin/build-drivers/build_drivers/metabase.clj | 88 +++++++
.../build_drivers/plugin_manifest.clj | 55 ++++
bin/build-drivers/build_drivers/verify.clj | 39 +++
bin/build-drivers/deps.edn | 13 +
bin/build-drivers/verify_driver.clj | 10 +
bin/check-clojure-cli.sh | 33 +++
bin/metabuild_common/core.clj | 38 +++
bin/metabuild_common/entrypoint.clj | 20 ++
bin/metabuild_common/files.clj | 79 ++++++
bin/metabuild_common/output.clj | 34 +++
bin/metabuild_common/shell.clj | 76 ++++++
bin/metabuild_common/steps.clj | 30 +++
bin/metabuild_common/util.clj | 0
bin/verify-driver | 46 +---
modules/drivers/bigquery/parents | 1 -
modules/drivers/googleanalytics/parents | 1 -
27 files changed, 982 insertions(+), 315 deletions(-)
create mode 100644 bin/build-drivers/README.md
create mode 100644 bin/build-drivers/build_driver.clj
create mode 100644 bin/build-drivers/build_drivers.clj
create mode 100644 bin/build-drivers/build_drivers/build_driver.clj
create mode 100644 bin/build-drivers/build_drivers/checksum.clj
create mode 100644 bin/build-drivers/build_drivers/common.clj
create mode 100644 bin/build-drivers/build_drivers/install_driver_locally.clj
create mode 100644 bin/build-drivers/build_drivers/metabase.clj
create mode 100644 bin/build-drivers/build_drivers/plugin_manifest.clj
create mode 100644 bin/build-drivers/build_drivers/verify.clj
create mode 100644 bin/build-drivers/deps.edn
create mode 100644 bin/build-drivers/verify_driver.clj
create mode 100755 bin/check-clojure-cli.sh
create mode 100644 bin/metabuild_common/core.clj
create mode 100644 bin/metabuild_common/entrypoint.clj
create mode 100644 bin/metabuild_common/files.clj
create mode 100644 bin/metabuild_common/output.clj
create mode 100644 bin/metabuild_common/shell.clj
create mode 100644 bin/metabuild_common/steps.clj
create mode 100644 bin/metabuild_common/util.clj
delete mode 100644 modules/drivers/bigquery/parents
delete mode 100644 modules/drivers/googleanalytics/parents
diff --git a/.circleci/config.yml b/.circleci/config.yml
index ba8048ccf2e..aa132bfed66 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -566,6 +566,12 @@ jobs:
- restore_cache:
keys:
- frontend-{{ checksum "./frontend-checksums.txt" }}
+ - run:
+ name: Install Clojure CLI
+ command: >
+ curl -O https://download.clojure.org/install/linux-install-1.10.1.708.sh &&
+ chmod +x linux-install-1.10.1.708.sh &&
+ sudo ./linux-install-1.10.1.708.sh
- run:
name: Build frontend if needed
command: >
@@ -942,7 +948,6 @@ workflows:
requires:
- build-uberjar
- fe-deps
-
- fe-tests-cypress:
name: fe-tests-cypress-1
requires:
diff --git a/.gitignore b/.gitignore
index 0e0906a0b71..772e1c944bd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,7 +8,7 @@
*.xcworkspacedata
.DS_Store
.\#*
-.cpcache
+.cpcache/
.eastwood
.idea/
.nrepl-port
diff --git a/bin/build-driver.sh b/bin/build-driver.sh
index 445e1bf832d..ed438085bb6 100755
--- a/bin/build-driver.sh
+++ b/bin/build-driver.sh
@@ -1,8 +1,6 @@
#! /usr/bin/env bash
-set -euo pipefail
-
-project_root=`pwd`
+set -eo pipefail
driver="$1"
@@ -11,231 +9,8 @@ if [ ! "$driver" ]; then
exit -1
fi
-driver_project_dir="$project_root/modules/drivers/$driver"
-driver_jar="$driver.metabase-driver.jar"
-dest_location="$project_root/resources/modules/$driver_jar"
-metabase_uberjar="$project_root/target/uberjar/metabase.jar"
-target_jar="$driver_project_dir/target/uberjar/$driver_jar"
-parents=''
-checksum_file="$driver_project_dir/target/checksum.txt"
-
-################################ DELETING OLD INCORRECTLY BUILT DRIVERS ###############################
-
-verify_existing_build() {
- verification_failed=''
- ./bin/verify-driver "$driver" || verification_failed=true
-
- if [ "$verification_failed" ]; then
- echo 'No existing build, or existing build is invalid. (Re)building driver.'
- # By removing the checksum it will force rebuilding the driver
- rm -f "$checksum_file"
- fi
-}
-
-
-######################################## CALCULATING CHECKSUMS ########################################
-
-md5_command=''
-if [ `command -v md5` ]; then
- md5_command=md5
-elif [ `command -v md5sum` ]; then
- md5_command=md5sum
-else
- echo "Don't know what command to use to calculate md5sums."
- exit -2
-fi
-
-# Calculate a checksum of all the driver source files. If we've already built the driver and the checksum is the same
-# there's no need to build the driver a second time
-calculate_checksum() {
- find "$driver_project_dir" -name '*.clj' -or -name '*.yaml' | sort | xargs cat | $md5_command
-}
-
-# Check whether the saved checksum for the driver sources from the last build is the same as the current one. If so,
-# we don't need to build again.
-checksum_is_same() {
- if [ -f "$checksum_file" ]; then
- old_checksum=`cat "$checksum_file"`
- current_checksum=`calculate_checksum`
- echo "Checksum of source files for previous build: $old_checksum"
- echo "Current checksum of source files: $current_checksum"
- if [ "$current_checksum" == "$old_checksum" ]; then
- # Make sure the target driver JAR actually exists as well!
- if [ -f "$target_jar" ]; then
- echo "$driver driver source unchanged since last build. Skipping re-build."
- return 0
- fi
- fi
- fi
- return 1
-}
-
-######################################## BUILDING THE DRIVER ########################################
-
-# Delete existing saved copies of the driver in the plugins and resources directories
-delete_old_drivers() {
- echo "Deleting old versions of $driver driver..."
- rm -f plugins/"$driver_jar"
- rm -f "$dest_location"
-}
-
-# Check if Metabase is installed locally for building drivers; install it if not
-install_metabase_core() {
- if [ ! "$(find ~/.m2/repository/metabase-core/metabase-core -name '*.jar')" ]; then
- echo "Building Metabase and installing locally..."
- lein clean
- lein install-for-building-drivers
- else
- echo "metabase-core already installed to local Maven repo."
- fi
-}
-
-
-# Build Metabase uberjar if needed, we'll need this for stripping duplicate classes
-build_metabase_uberjar() {
- if [ ! -f "$metabase_uberjar" ]; then
- echo 'Building Metabase uberjar...'
- lein uberjar
- else
- echo "Metabase uberjar already built."
- fi
-}
-
-# Take a look at the `parents` file listing the parents to build, if applicable
-build_parents() {
- echo "Building parent drivers (if needed)..."
-
- parents_list="$driver_project_dir"/parents
-
- if [ -f "$parents_list" ]; then
- parents=`cat "$parents_list"`
- echo "Found driver parents: $parents"
- fi
-
- # Check and see if we need to recursively build or install any of our parents before proceeding
- for parent in $parents; do
- if [ ! -f resources/modules/"$parent.metabase-driver.jar" ]; then
- echo "Building $parent..."
- ./bin/build-driver.sh "$parent"
- fi
-
- # Check whether built parent driver *JAR* exists in local Maven repo
- parent_install_dir="~/.m2/repository/metabase/$parent-driver"
- parent_installed_jar=''
- if [ -f "$parent_install_dir" ]; then
- parent_installed_jar=`find "$parent_install_dir" -name '*.jar'`
- fi
+source "./bin/check-clojure-cli.sh"
+check_clojure_cli
- if [ ! "$parent_installed_jar" ]; then
- parent_project_dir="$project_root/modules/drivers/$parent"
- echo "Installing $parent locally..."
- cd "$parent_project_dir"
- lein clean
- lein install-for-building-drivers
- cd "$project_root"
- else
- echo "$parent already installed to local Maven repo"
- fi
- done
-}
-
-# Build the driver uberjar itself
-build_driver_uberjar() {
- echo "Building $driver driver..."
-
- cd "$driver_project_dir"
-
- rm -rf target
-
- lein clean
- DEBUG=1 LEIN_SNAPSHOTS_IN_RELEASE=true lein uberjar
-
- cd "$project_root"
-
- if [ ! -f "$target_jar" ]; then
- echo "Error: could not find $target_jar. Build failed."
- return -3
- fi
-}
-
-# Strip out any classes in driver JAR found in core Metabase uberjar or parent JARs; recompress with higher compression ratio
-strip_and_compress() {
- # ok, first things first, strip out any classes also found in the core Metabase uberjar
- lein strip-and-compress "$target_jar"
-
- # next, remove any classes also found in any of the parent JARs
- for parent in $parents; do
- echo "Removing duplicate classes with $parent uberjar..."
- lein strip-and-compress "$target_jar" "resources/modules/$parent.metabase-driver.jar"
- done
-}
-
-# copy finished JAR to the resources dir
-copy_target_to_dest() {
- echo "Copying $target_jar -> $dest_location"
- cp "$target_jar" "$dest_location"
-}
-
-# check that JAR in resources dir looks correct
-verify_build () {
- verification_failed=''
- ./bin/verify-driver "$driver" || verification_failed=true
-
- if [ "$verification_failed" ]; then
- echo "./bin/build-driver.sh $driver FAILED."
- rm -f "$checksum_file"
- rm -f "$target_jar"
- rm -f "$dest_location"
- return -4
- fi
-}
-
-# Save the checksum for the newly built JAR
-save_checksum() {
- echo "Saving checksum for source files to $checksum_file"
- checksum=`calculate_checksum`
- echo "$checksum" > "$checksum_file"
-}
-
-# Runs all the steps needed to build the driver.
-build_driver() {
- verify_existing_build &&
- delete_old_drivers &&
- install_metabase_core &&
- build_metabase_uberjar &&
- build_parents &&
- build_driver_uberjar &&
- strip_and_compress &&
- copy_target_to_dest &&
- verify_build &&
- save_checksum
-}
-
-######################################## PUTTING IT ALL TOGETHER ########################################
-
-clean_local_repo() {
- echo "Deleting existing installed metabase-core and driver dependencies..."
- rm -rf ~/.m2/repository/metabase-core
- rm -rf ~/.m2/repository/metabase/*-driver
-}
-
-retry_clean_build() {
- echo "Building without cleaning failed. Retrying clean build..."
- clean_local_repo
- build_driver
-}
-
-mkdir -p resources/modules
-
-# run only a specific step with ./bin/build-driver.sh <driver> <step>
-if [ $# -eq 2 ]; then
- $2
-# Build driver if checksum has changed
-elif ! checksum_is_same; then
- echo "Checksum has changed."
- build_driver || retry_clean_build
-# Either way, always copy the target uberjar to the dest location
-else
- echo "Checksum is unchanged."
- (copy_target_to_dest && verify_build) || retry_clean_build
-fi
+cd bin/build-drivers
+clojure -M -m build-driver "$driver"
diff --git a/bin/build-drivers.sh b/bin/build-drivers.sh
index 163814628cf..f3b78e8f959 100755
--- a/bin/build-drivers.sh
+++ b/bin/build-drivers.sh
@@ -1,45 +1,9 @@
#! /usr/bin/env bash
-set -eo pipefail
+set -euo pipefail
-# If ran as `./bin/build-drivers.sh clean` then uninstall metabase-core from the local Maven repo and delete
-if [ "$1" == clean ]; then
- echo "Deleting existing installed metabase-core and driver dependencies..."
- rm -rf ~/.m2/repository/metabase-core
- rm -rf ~/.m2/repository/metabase/*-driver
+source "./bin/check-clojure-cli.sh"
+check_clojure_cli
- echo "Deleting built drivers in resources/modules..."
- rm -rf resources/modules
- echo "Deleting build Metabase uberjar..."
- rm -rf target
-
- for target in `find modules -name 'target' -type d`; do
- echo "Deleting $target..."
- rm -rf "$target"
- done
-fi
-
-# strip trailing slashes if `ls` is set to include them
-drivers=`ls modules/drivers/ | sed 's|/$||'`
-
-for driver in $drivers; do
- echo "Build: $driver"
-
- build_failed=''
- ./bin/build-driver.sh "$driver" || build_failed=true
-
- if [ "$build_failed" ]; then
- echo "Failed to build driver $driver."
- exit -1
- fi
-done
-
-# Double-check that all drivers were built successfully
-for driver in $drivers; do
- verification_failed=''
- ./bin/verify-driver "$driver" || verification_failed=true
-
- if [ "$verification_failed" ]; then
- exit -2
- fi
-done
+cd bin/build-drivers
+clojure -M -m build-drivers
diff --git a/bin/build-drivers/README.md b/bin/build-drivers/README.md
new file mode 100644
index 00000000000..0033455aba5
--- /dev/null
+++ b/bin/build-drivers/README.md
@@ -0,0 +1,45 @@
+# Build-drivers scripts
+
+Scripts for building Metabase driver plugins. You must install the [Clojure CLI
+tools](https://www.clojure.org/guides/getting_started) to use these.
+
+There are three main entrypoints. Shell script wrappers are provided for convenience and compatibility.
+
+### `build-drivers`
+
+Builds *all* drivers as needed. If drivers were recently built and no relevant source code changed, skips rebuild.
+
+```
+cd bin/build-drivers
+clojure -M -m build-drivers
+
+# or
+
+./bin/build-drivers.sh
+```
+
+### `build-driver`
+
+Build a single driver as needed. Builds parent drivers if needed first.
+
+```
+cd bin/build-driver redshift
+clojure -M -m build-driver redshift
+
+# or
+
+./bin/build-driver.sh redshift
+```
+
+### `verify-driver`
+
+Verify that a built driver looks correctly built.
+
+```
+cd bin/verify-driver redshift
+clojure -M -m verify-driver redshift
+
+# or
+
+./bin/verify-driver redshift
+```
diff --git a/bin/build-drivers/build_driver.clj b/bin/build-drivers/build_driver.clj
new file mode 100644
index 00000000000..a934d4cd26f
--- /dev/null
+++ b/bin/build-drivers/build_driver.clj
@@ -0,0 +1,10 @@
+(ns build-driver
+ "Entrypoint for `bin/build-driver.sh`. Builds a single driver, if needed."
+ (:require [build-drivers.build-driver :as build-driver]
+ [metabuild-common.core :as u]))
+
+(defn -main [& [driver]]
+ (u/exit-when-finished-nonzero-on-exception
+ (when-not (seq driver)
+ (throw (ex-info "Usage: clojure -m build-driver <driver>" {})))
+ (build-driver/build-driver! (keyword driver))))
diff --git a/bin/build-drivers/build_drivers.clj b/bin/build-drivers/build_drivers.clj
new file mode 100644
index 00000000000..d65dfee59a5
--- /dev/null
+++ b/bin/build-drivers/build_drivers.clj
@@ -0,0 +1,20 @@
+(ns build-drivers
+ "Entrypoint for `bin/build-drivers.sh`. Builds all drivers, if needed."
+ (:require [build-drivers
+ [build-driver :as build-driver]
+ [common :as c]]
+ [clojure.java.io :as io]
+ [metabuild-common.core :as u]))
+
+(defn- all-drivers []
+ (map keyword (.list (io/file (c/filename c/project-root-directory "modules" "drivers")))))
+
+(defn- build-drivers! []
+ (u/step "Building all drivers"
+ (doseq [driver (all-drivers)]
+ (build-driver/build-driver! driver))
+ (u/announce "Successfully built all drivers.")))
+
+(defn -main []
+ (u/exit-when-finished-nonzero-on-exception
+ (build-drivers!)))
diff --git a/bin/build-drivers/build_drivers/build_driver.clj b/bin/build-drivers/build_drivers/build_driver.clj
new file mode 100644
index 00000000000..bc34a167c75
--- /dev/null
+++ b/bin/build-drivers/build_drivers/build_driver.clj
@@ -0,0 +1,168 @@
+(ns build-drivers.build-driver
+ "Logic for building a single driver."
+ (:require [build-drivers
+ [checksum :as checksum]
+ [common :as c]
+ [install-driver-locally :as install-locally]
+ [metabase :as metabase]
+ [plugin-manifest :as manifest]
+ [verify :as verify]]
+ [colorize.core :as colorize]
+ [environ.core :as env]
+ [metabuild-common.core :as u]))
+
+(defn- copy-driver!
+ "Copy the driver JAR from its `target/` directory to `resources/modules`/."
+ [driver]
+ (u/step (format "Copy %s driver uberjar from %s -> %s"
+ driver
+ (u/assert-file-exists (c/driver-jar-build-path driver))
+ (c/driver-jar-destination-path driver))
+ (u/delete-file! (c/driver-jar-destination-path driver))
+ (u/create-directory-unless-exists! c/driver-jar-destination-directory)
+ (u/copy-file! (c/driver-jar-build-path driver)
+ (c/driver-jar-destination-path driver))))
+
+(defn- clean-driver-artifacts!
+ "Delete built JARs of `driver`."
+ [driver]
+ (u/step (format "Delete %s driver artifacts" driver)
+ (u/delete-file! (c/driver-target-directory driver))
+ (u/delete-file! (c/driver-jar-destination-path driver))))
+
+(defn- clean-parents!
+ "Delete built JARs and local Maven installations of the parent drivers of `driver`."
+ [driver]
+ (u/step (format "Clean %s parent driver artifacts" driver)
+ (doseq [parent (manifest/parent-drivers driver)]
+ (clean-driver-artifacts! parent)
+ (install-locally/clean! parent)
+ (clean-parents! parent))))
+
+(defn- clean-all!
+ "Delete all artifacts relating to building `driver`, including the driver JAR itself and installed
+ `metabase-core`/Metabase uberjar and any parent driver artifacts."
+ [driver]
+ (u/step "Clean all"
+ (clean-driver-artifacts! driver)
+ (clean-parents! driver)
+ (metabase/clean-metabase!)))
+
+(declare build-driver!)
+
+(defn- build-parents!
+ "Build and install to the local Maven repo any parent drivers of `driver` (e.g. `:google` is a parent of `:bigquery`).
+ The driver must be built as an uberjar so we can remove duplicate classes during the `strip-and-compress` stage; it
+ must be installed as a library so we can use it as a `:provided` dependency when building the child driver."
+ [driver]
+ (u/step (format "Build %s parent drivers" driver)
+ (doseq [parent (manifest/parent-drivers driver)]
+ (build-parents! parent)
+ (install-locally/install-locally! parent)
+ (build-driver! parent))
+ (u/announce "%s parents built successfully." driver)))
+
+(defn- strip-and-compress-uberjar!
+ "Remove any classes in compiled `driver` that are also present in the Metabase uberjar or parent drivers. The classes
+ will be available at runtime, and we don't want to make things unpredictable by including them more than once in
+ different drivers.
+
+ This is only needed because `lein uberjar` does not seem to reliably exclude classes from `:provided` Clojure
+ dependencies like `metabase-core` and the parent drivers."
+ ([driver]
+ (u/step (str (format "Strip out any classes in %s driver JAR found in core Metabase uberjar or parent JARs" driver)
+ " and recompress with higher compression ratio")
+ (let [uberjar (u/assert-file-exists (c/driver-jar-build-path driver))]
+ (u/step "strip out any classes also found in the core Metabase uberjar"
+ (strip-and-compress-uberjar! uberjar (u/assert-file-exists c/metabase-uberjar-path)))
+ (u/step "remove any classes also found in any of the parent JARs"
+ (doseq [parent (manifest/parent-drivers driver)]
+ (strip-and-compress-uberjar! uberjar (u/assert-file-exists (c/driver-jar-build-path parent))))))))
+
+ ([target source]
+ (u/step (format "Remove classes from %s that are present in %s and recompress" target source)
+ (u/sh {:dir c/project-root-directory}
+ "lein"
+ "strip-and-compress"
+ (u/assert-file-exists target)
+ (u/assert-file-exists source)))))
+
+(defn- build-uberjar! [driver]
+ (u/step (format "Build %s uberjar" driver)
+ (u/delete-file! (c/driver-target-directory driver))
+ (u/sh {:dir (c/driver-project-dir driver)} "lein" "clean")
+ (u/sh {:dir (c/driver-project-dir driver)
+ :env {"LEIN_SNAPSHOTS_IN_RELEASE" "true"
+ #_"DEBUG" #_"1"
+ "JAVA_HOME" (env/env :java-home)
+ "HOME" (env/env :user-home)}}
+ "lein" "uberjar")
+ (strip-and-compress-uberjar! driver)
+ (u/announce "%s uberjar build successfully." driver)))
+
+(defn- build-and-verify!
+ "Build `driver` and verify the built JAR. This function ignores any existing artifacts and will always rebuild."
+ [driver]
+ (u/step (str (colorize/green "Build ") (colorize/yellow driver) (colorize/green " driver"))
+ (clean-driver-artifacts! driver)
+ (metabase/build-metabase!)
+ (build-parents! driver)
+ (build-uberjar! driver)
+ (copy-driver! driver)
+ (verify/verify-driver driver)
+ (u/step (format "Save checksum for %s driver to %s" driver (c/driver-checksum-filename driver))
+ (spit (c/driver-checksum-filename driver) (checksum/driver-checksum driver)))))
+
+(defn- driver-checksum-matches?
+ "Check whether the saved checksum for the driver from the last build is the same as the current one. If so, we don't
+ need to build again. This checksum is based on driver sources as well as the checksums for Metabase sources and
+ parent drivers."
+ [driver]
+ (u/step (format "Determine whether %s driver source files have changed since last build" driver)
+ (let [existing-checksum (checksum/existing-driver-checksum driver)
+ current-checksum (checksum/driver-checksum driver)
+ same? (= existing-checksum current-checksum)]
+ (u/announce (if same?
+ "Checksum is the same. Do not need to rebuild driver."
+ "Checksum is different. Need to rebuild driver."))
+ same?)))
+
+(defn build-driver!
+ "Build `driver`, if needed."
+ [driver]
+ {:pre [(keyword? driver)]}
+ (u/step (str (colorize/green "Build ") (colorize/yellow driver) (colorize/green " driver if needed"))
+ ;; When we build a driver, we save a checksum of driver source code + metabase source code + parent drivers
+ ;; alongside the built driver JAR. The next time this script is called, we recalculate that checksum -- if the
+ ;; current checksum matches the saved one associated with the built driver JAR, we do not need to rebuild the
+ ;; driver. If anything relevant has changed, we have to rebuild the driver.
+ (if (driver-checksum-matches? driver)
+ ;; even if we're not rebuilding the driver, copy the artifact from `modules/drivers/<driver>/target/uberjar/`
+ ;; to `resources/modules` so we can be sure we have the most up-to-date version there.
+ (try
+ (verify/verify-driver driver)
+ (copy-driver! driver)
+ ;; if verification fails, delete all the existing artifacts and just rebuild the driver from scratch.
+ (catch Throwable e
+ (u/error "Error verifying existing driver:\n%s" (pr-str e))
+ (u/announce "Deleting existing driver artifacts and rebuilding.")
+ (clean-driver-artifacts! driver)
+ (build-driver! driver)))
+ ;; if checksum does not match, build and verify the driver
+ (try
+ (build-and-verify! driver)
+ ;; if building fails, clean everything, including metabase-core, the metabase uberjar, and parent
+ ;; dependencies, *then* retry.
+ (catch Throwable e
+ (u/announce "Cleaning ALL and retrying...")
+ (clean-all! driver)
+ (try
+ (build-and-verify! driver)
+ ;; if building the driver failed again, even after cleaning, delete anything that was built and then
+ ;; give up.
+ (catch Throwable e
+ (u/safe-println (colorize/red (format "Failed to build %s driver." driver)))
+ (clean-driver-artifacts! driver)
+ (throw e))))))
+ ;; if we make it this far, we've built the driver successfully.
+ (u/announce "Success.")))
diff --git a/bin/build-drivers/build_drivers/checksum.clj b/bin/build-drivers/build_drivers/checksum.clj
new file mode 100644
index 00000000000..b6f8e2ae6f9
--- /dev/null
+++ b/bin/build-drivers/build_drivers/checksum.clj
@@ -0,0 +1,76 @@
+(ns build-drivers.checksum
+ "Shared code for calculating and reading hex-encoded MD5 checksums for relevant files."
+ (:require [build-drivers
+ [common :as c]
+ [plugin-manifest :as manifest]]
+ [clojure.java.io :as io]
+ [clojure.string :as str]
+ [metabuild-common.core :as u])
+ (:import org.apache.commons.codec.digest.DigestUtils))
+
+(defn checksum-from-file
+ "Read a saved MD5 hash checksum from a file."
+ [filename]
+ (u/step (format "Read saved checksum from %s" filename)
+ (let [file (io/file filename)]
+ (if-not (.exists file)
+ (u/announce "%s does not exist" filename)
+ (when-let [[checksum-line] (not-empty (str/split-lines (slurp file)))]
+ (when-let [[_ checksum-hex] (re-matches #"(^[0-9a-f]{32}).*$" checksum-line)]
+ (u/safe-println (format "Saved checksum is %s" checksum-hex))
+ checksum-hex))))))
+
+;;; -------------------------------------------- Metabase source checksum --------------------------------------------
+
+(defn- metabase-source-paths []
+ (sort
+ (cons
+ (c/filename c/project-root-directory "project.clj")
+ (mapcat (fn [dir]
+ (try
+ (u/find-files dir #(str/ends-with? % ".clj"))
+ (catch Throwable _
+ [])))
+ [(c/filename c/project-root-directory "src")
+ (c/filename c/project-root-directory "enterprise" "backend" "src")
+ (c/filename c/project-root-directory "backend" "mbql")]))))
+
+(defn metabase-source-checksum
+ "Checksum of Metabase backend source files and `project.clj`."
+ ^String []
+ (let [paths (metabase-source-paths)]
+ (u/step (format "Calculate checksum for %d Metabase source files" (count paths))
+ (let [checksum (DigestUtils/md5Hex (str/join (map slurp paths)))]
+ (u/safe-println (format "Current checksum is %s" checksum))
+ checksum))))
+
+
+;;; ---------------------------------------------- Driver source files -----------------------------------------------
+
+(defn existing-driver-checksum
+ "Checksum from the relevant sources from last time we built `driver`."
+ [driver]
+ (checksum-from-file (c/driver-checksum-filename driver)))
+
+(defn- driver-source-paths
+ "Returns sequence of the source filenames for `driver`."
+ [driver]
+ (u/find-files (c/driver-project-dir driver)
+ (fn [path]
+ (or (and (str/ends-with? path ".clj")
+ (not (str/starts-with? path (c/filename (c/driver-project-dir driver) "test"))))
+ (str/ends-with? path ".yaml")))))
+
+(defn driver-checksum
+ "The driver checksum is based on a checksum of all the driver source files (`.clj` files and the plugin manifest YAML
+ file) combined with the checksums for `metabase-core` *and* the parent drivers. After building a driver, we save
+ this checksum. Next time the script is ran, we recalculate the checksum to determine whether anything relevant has
+ changed -- if it has, and the current checksum doesn't match the saved one, we need to rebuild the driver."
+ ^String [driver]
+ (let [source-paths (driver-source-paths driver)]
+ (u/step (format "Calculate checksum for %d files: %s ..." (count source-paths) (first source-paths))
+ (let [checksum (DigestUtils/md5Hex (str/join (concat [(metabase-source-checksum)]
+ (map driver-checksum (manifest/parent-drivers driver))
+ (map slurp (driver-source-paths driver)))))]
+ (u/safe-println (format "Current checksum is %s" checksum))
+ checksum))))
diff --git a/bin/build-drivers/build_drivers/common.clj b/bin/build-drivers/build_drivers/common.clj
new file mode 100644
index 00000000000..21f45bea25d
--- /dev/null
+++ b/bin/build-drivers/build_drivers/common.clj
@@ -0,0 +1,75 @@
+(ns build-drivers.common
+ "Shared constants and functions related to source and artifact paths used throughout this code."
+ (:require [clojure.string :as str]
+ [environ.core :as env])
+ (:import java.io.File))
+
+;; since this file is used pretty much everywhere, this seemed like a good place to put this.
+(set! *warn-on-reflection* true)
+
+(when-not (str/ends-with? (env/env :user-dir) "/build-drivers")
+ (throw (ex-info "Please run build-driver scripts from the `bin/build-drivers` directory e.g. `cd bin/build-drivers; clojure -m build-driver`"
+ {:user-dir (env/env :user-dir)})))
+
+(defn env-or-throw
+ "Fetch an env var value or throw an Exception if it is unset."
+ [k]
+ (or (get env/env k)
+ (throw (Exception. (format "%s is unset. Please set it and try again." (str/upper-case (str/replace (name k) #"-" "_")))))))
+
+(defn filename [& path-components]
+ (str/join File/separatorChar path-components))
+
+(def ^String project-root-directory
+ "e.g. /Users/cam/metabase"
+ (.. (File. ^String (env/env :user-dir)) getParentFile getParentFile getAbsolutePath))
+
+(def ^String maven-repository-path
+ (filename (env/env :user-home) ".m2" "repository"))
+
+
+;;; -------------------------------------------------- Driver Paths --------------------------------------------------
+
+(defn driver-project-dir
+ "e.g. \"/home/cam/metabase/modules/drivers/redshift\""
+ [driver]
+ (filename project-root-directory "modules" "drivers" (name driver)))
+
+(defn driver-jar-name
+ "e.g. \"redshift.metabase-driver.jar\""
+ [driver]
+ (format "%s.metabase-driver.jar" (name driver)))
+
+(defn driver-target-directory
+ [driver]
+ (filename (driver-project-dir driver) "target"))
+
+(defn driver-jar-build-path
+ "e.g. \"/home/cam/metabase/modules/drivers/redshift/target/uberjar/redshift.metabase-driver.jar\""
+ [driver]
+ (filename (driver-target-directory driver) "uberjar" (driver-jar-name driver)))
+
+(def ^String driver-jar-destination-directory
+ (filename project-root-directory "resources" "modules"))
+
+(defn driver-jar-destination-path
+ "e.g. \"/home/cam/metabase/resources/modules/redshift.metabase-driver.jar\""
+ [driver]
+ (filename driver-jar-destination-directory (driver-jar-name driver)))
+
+(defn driver-checksum-filename
+ "e.g. \"/home/cam/metabase/modules/drivers/redshift/target/checksum.txt\""
+ [driver]
+ (filename (driver-project-dir driver) "target" "checksum.txt")) ; TODO - rename to checksum.md5
+
+(defn driver-plugin-manifest-filename
+ "e.g. \"/home/cam/metabase/modules/drivers/bigquery/resources/plugin-manifest.yaml\""
+ [driver]
+ (filename (driver-project-dir driver) "resources" "metabase-plugin.yaml"))
+
+
+;;; ------------------------------------------ Metabase Local Install Paths ------------------------------------------
+
+(def ^String metabase-uberjar-path
+ "e.g. \"home/cam/metabase/target/uberjar/metabase.jar\""
+ (filename project-root-directory "target" "uberjar" "metabase.jar"))
diff --git a/bin/build-drivers/build_drivers/install_driver_locally.clj b/bin/build-drivers/build_drivers/install_driver_locally.clj
new file mode 100644
index 00000000000..484e880cc90
--- /dev/null
+++ b/bin/build-drivers/build_drivers/install_driver_locally.clj
@@ -0,0 +1,50 @@
+(ns build-drivers.install-driver-locally
+ "Logic related to installing a driver as a library in the local Maven repository so it can be used as a dependency
+ when building descandant drivers. Right now this is only used for `:google`, which is used by `:bigquery` and
+ `:googleanalytics`."
+ (:require [build-drivers
+ [checksum :as checksum]
+ [common :as c]]
+ [colorize.core :as colorize]
+ [metabuild-common.core :as u]))
+
+(defn- driver-local-install-path [driver]
+ (c/filename c/maven-repository-path "metabase" (format "%s-driver" (name driver))))
+
+(defn- driver-local-install-checksum-filename [driver]
+ (c/filename (driver-local-install-path driver) "checksum.md5"))
+
+(defn clean!
+ "Delete local Maven installation of the library version of `driver`."
+ [driver]
+ (u/step (format "Deleting existing Maven installation of %s driver" driver)
+ (u/delete-file! (driver-local-install-path driver))))
+
+(defn- local-install-checksum-matches?
+ "After installing the library version of `driver`, we save a checksum based on its sources; next time we call
+ `install-locally!`, we can recalculate the checksum; if the saved one matches the current one, we do not need to
+ reinstall."
+ [driver]
+ (u/step "Determine whether %s driver source files have changed since last local install"
+ (let [existing-checksum (checksum/checksum-from-file (driver-local-install-checksum-filename driver))
+ current-checksum (checksum/driver-checksum driver)
+ same? (= existing-checksum current-checksum)]
+ (u/announce (if same?
+ "Checksum is the same. Do not need to rebuild driver."
+ "Checksum is different. Need to rebuild driver."))
+ same?)))
+
+(defn install-locally!
+ "Install `driver` as a library in the local Maven repository IF NEEDED so descendant drivers can use it as a
+ `:provided` dependency when building. E.g. before building `:bigquery` we need to install `:google` as a library
+ locally."
+ [driver]
+ {:pre [(keyword? driver)]}
+ (u/step (str (colorize/green "Install ") (colorize/yellow driver) (colorize/green " driver to local Maven repo if needed"))
+ (if (local-install-checksum-matches? driver)
+ (u/announce "Already installed locally.")
+ (u/step (str (colorize/green "Install ") (colorize/yellow driver) (colorize/green " driver to local Maven repo"))
+ (u/sh {:dir (c/driver-project-dir driver)} "lein" "clean")
+ (u/sh {:dir (c/driver-project-dir driver)} "lein" "install-for-building-drivers")
+ (u/step (format "Save checksum to %s" driver (driver-local-install-checksum-filename driver))
+ (spit (driver-local-install-checksum-filename driver) (checksum/driver-checksum driver)))))))
diff --git a/bin/build-drivers/build_drivers/metabase.clj b/bin/build-drivers/build_drivers/metabase.clj
new file mode 100644
index 00000000000..5ba36429de0
--- /dev/null
+++ b/bin/build-drivers/build_drivers/metabase.clj
@@ -0,0 +1,88 @@
+(ns build-drivers.metabase
+ "Code for installing the main Metabase project as a library (`metabase-core`) in the local Maven repository, and for
+ building a Metabase uberjar. Both are needed when building drivers."
+ (:require [build-drivers
+ [checksum :as checksum]
+ [common :as c]]
+ [metabuild-common.core :as u]))
+
+(def ^String ^:private uberjar-checksum-path
+ (str c/metabase-uberjar-path ".md5"))
+
+(def ^String ^:private metabase-core-install-path
+ (c/filename c/maven-repository-path "metabase-core"))
+
+(def ^String ^:private metabase-core-checksum-path
+ (c/filename metabase-core-install-path "checksum.md5"))
+
+(defn metabase-core-checksum-matches? []
+ (u/step "Determine whether Metabase source files checksum has changed since last install of metabase-core"
+ (let [existing-checksum (checksum/checksum-from-file metabase-core-checksum-path)
+ current-checksum (checksum/metabase-source-checksum)
+ same? (= existing-checksum current-checksum)]
+ (u/announce (if same?
+ "Checksum is the same. Do not need to reinstall metabase-core locally."
+ "Checksum is different. Need to reinstall metabase-core locally."))
+ same?)))
+
+(defn- delete-metabase-core-install! []
+ (u/step "Delete local installation of metabase-core"
+ (u/delete-file! metabase-core-install-path)))
+
+(defn- install-metabase-core! []
+ (u/step "Install metabase-core locally if needed"
+ (if (metabase-core-checksum-matches?)
+ (u/announce "Up-to-date metabase-core already installed to local Maven repo")
+ (do
+ (delete-metabase-core-install!)
+ (u/sh {:dir c/project-root-directory} "lein" "clean")
+ (u/sh {:dir c/project-root-directory} "lein" "install-for-building-drivers")
+ (u/step "Save checksum for local installation of metabase-core"
+ (spit metabase-core-checksum-path (checksum/metabase-source-checksum)))
+ (u/announce "metabase-core dep installed to local Maven repo successfully.")))))
+
+(defn uberjar-checksum-matches?
+ "After installing/building Metabase we save a MD5 hex checksum of Metabase backend source files (including
+ `project.clj`). The next time we run `build-metabase!`, if the checksums have changed we know we need to
+ rebuild/reinstall."
+ []
+ (u/step "Determine whether Metabase source files checksum has changed since last build of uberjar"
+ (let [existing-checksum (checksum/checksum-from-file uberjar-checksum-path)
+ current-checksum (checksum/metabase-source-checksum)
+ same? (= existing-checksum current-checksum)]
+ (u/announce (if same?
+ "Checksum is the same. Do not need to rebuild Metabase uberjar."
+ "Checksum is different. Need to rebuild Metabase uberjar."))
+ same?)))
+
+(defn- delete-metabase-uberjar! []
+ (u/step "Delete exist metabase uberjar"
+ (u/delete-file! (c/filename c/project-root-directory "target"))))
+
+(defn- build-metabase-uberjar! []
+ (u/step "Build Metabase uberjar if needed"
+ (if (uberjar-checksum-matches?)
+ (u/announce "Update-to-date uberjar already built")
+ (do
+ (delete-metabase-uberjar!)
+ (u/sh {:dir c/project-root-directory} "lein" "clean")
+ (u/sh {:dir c/project-root-directory} "lein" "uberjar")
+ (u/step "Save checksum for Metabase uberar"
+ (spit uberjar-checksum-path (checksum/metabase-source-checksum)))
+ (u/announce "Metabase uberjar built successfully")))))
+
+(defn clean-metabase!
+ "Delete local Maven repository installation of the `metabase-core` library and delete the built Metabase uberjar."
+ []
+ (u/step "Clean local Metabase deps"
+ (delete-metabase-core-install!)
+ (delete-metabase-uberjar!)))
+
+(defn build-metabase!
+ "Install `metabase-core` as a library in the local Maven repo, and build the Metabase uberjar IF NEEDED. We need to do
+ both because `metabase-core` is used as a dependency for drivers, and the Metabase uberjar is checked to make sure
+ we don't ship duplicate classes in the driver JAR (as part of the `strip-and-compress` stage.)"
+ []
+ (u/step "Build metabase-core and install locally"
+ (install-metabase-core!)
+ (build-metabase-uberjar!)))
diff --git a/bin/build-drivers/build_drivers/plugin_manifest.clj b/bin/build-drivers/build_drivers/plugin_manifest.clj
new file mode 100644
index 00000000000..163c6096475
--- /dev/null
+++ b/bin/build-drivers/build_drivers/plugin_manifest.clj
@@ -0,0 +1,55 @@
+(ns build-drivers.plugin-manifest
+ "Code for reading the YAML plugin manifest for a driver. "
+ (:require [build-drivers.common :as c]
+ [metabuild-common.core :as u]
+ [yaml.core :as yaml]))
+
+(defn- plugin-manifest
+ "Read `driver` plugin manifest and return a map."
+ [driver]
+ {:post [(map? %)]}
+ (yaml/from-file (u/assert-file-exists (c/driver-plugin-manifest-filename driver))))
+
+(defn- driver-declarations [manifest]
+ ;; driver plugin manifest can have a single `:driver`, or multiple drivers, e.g. Spark SQL which also has the
+ ;; `:hive-like` abstract driver
+ (let [{driver-declaration :driver} manifest]
+ (if (map? driver-declaration)
+ [driver-declaration]
+ driver-declaration)))
+
+(defn- declared-drivers
+ "Sequence of all drivers declared in a plugin `manifest`. Usually only one driver, except for Spark SQL which declares
+ both `:hive-like` and `:sparksql`."
+ [manifest]
+ (map (comp keyword :name) (driver-declarations manifest)))
+
+(def ^:private metabase-core-drivers
+ "Drivers that ship as part of the core Metabase project (as opposed to a plugin) and thus do not need to be built."
+ #{:sql
+ :sql-jdbc
+ :mysql
+ :h2
+ :postgres})
+
+(defn parent-drivers
+ "Get the parent drivers of a driver for purposes of building a driver. Excludes drivers that ship as part of
+ `metabase-core`, since we don't need to worry about building those.
+
+ e.g.
+
+ (parent-drivers :googleanalytics) ;-> (:google)"
+ [driver]
+ (let [manifest (plugin-manifest driver)
+ declared (declared-drivers manifest)]
+ (or (not-empty
+ (for [{parent-declaration :parent} (driver-declarations manifest)
+ :let [parents (if (string? parent-declaration)
+ [parent-declaration]
+ parent-declaration)]
+ parent parents
+ :let [parent (keyword parent)]
+ :when (and (not (contains? (set declared) parent))
+ (not (contains? metabase-core-drivers parent)))]
+ parent))
+ (u/announce "%s does not have any parents" driver))))
diff --git a/bin/build-drivers/build_drivers/verify.clj b/bin/build-drivers/build_drivers/verify.clj
new file mode 100644
index 00000000000..16919cfbde8
--- /dev/null
+++ b/bin/build-drivers/build_drivers/verify.clj
@@ -0,0 +1,39 @@
+(ns build-drivers.verify
+ (:require [build-drivers.common :as c]
+ [colorize.core :as colorize]
+ [metabuild-common.core :as u]))
+
+(defn- verify-exists [driver]
+ (let [filename (c/driver-jar-destination-path driver)]
+ (u/step (format "Check %s exists" filename)
+ (if (u/file-exists? filename)
+ (u/announce "File exists.")
+ (throw (ex-info (format "Driver verification failed: %s does not exist" filename) {}))))))
+
+(defn- verify-has-init-class [driver]
+ (let [filename (c/driver-jar-destination-path driver)
+ driver-init-class (format "metabase/driver/%s__init.class" (munge (name driver)))]
+ (u/step (format "Check %s contains init class file %s" filename driver-init-class)
+ (if (some (partial = driver-init-class)
+ (u/sh {:quiet? true} "jar" "-tf" filename))
+ (u/announce "Driver init class file found.")
+ (throw (ex-info (format "Driver verification failed: init class file %s not found" driver-init-class) {}))))))
+
+(defn- verify-has-plugin-manifest [driver]
+ (let [filename (c/driver-jar-destination-path driver)
+ driver-init-class (format "metabase/driver/%s__init.class" (munge (name driver)))]
+ (u/step (format "Check %s contains metabase-plugin.yaml" filename)
+ (if (some (partial = "metabase-plugin.yaml")
+ (u/sh {:quiet? true} "jar" "-tf" filename))
+ (u/announce "Plugin manifest found.")
+ (throw (ex-info (format "Driver verification failed: plugin manifest missing" driver-init-class) {}))))))
+
+(defn verify-driver
+ "Run a series of checks to make sure `driver` was built correctly. Throws exception if any checks fail."
+ [driver]
+ {:pre [(keyword? driver)]}
+ (u/step (str (colorize/green "Verify ") (colorize/yellow driver) (colorize/green " driver"))
+ (verify-exists driver)
+ (verify-has-init-class driver)
+ (verify-has-plugin-manifest driver)
+ (u/announce (format "%s driver verification successful." driver))))
diff --git a/bin/build-drivers/deps.edn b/bin/build-drivers/deps.edn
new file mode 100644
index 00000000000..0e9cf9483eb
--- /dev/null
+++ b/bin/build-drivers/deps.edn
@@ -0,0 +1,13 @@
+{:paths ["./" "../"]
+
+ :deps
+ {cheshire/cheshire {:mvn/version "5.8.1"}
+ commons-codec/commons-codec {:mvn/version "1.14"}
+ commons-io/commons-io {:mvn/version "2.6"}
+ colorize/colorize {:mvn/version "0.1.1"}
+ environ/environ {:mvn/version "1.1.0"}
+ hiccup/hiccup {:mvn/version "1.0.5"}
+ io.forward/yaml {:mvn/version "1.0.9"}
+ org.flatland/ordered {:mvn/version "1.5.7"}
+ potemkin/potemkin {:mvn/version "0.4.5"}
+ stencil/stencil {:mvn/version "0.5.0"}}}
diff --git a/bin/build-drivers/verify_driver.clj b/bin/build-drivers/verify_driver.clj
new file mode 100644
index 00000000000..5964ba99c0e
--- /dev/null
+++ b/bin/build-drivers/verify_driver.clj
@@ -0,0 +1,10 @@
+(ns verify-driver
+ "Entrypoint for `bin/verify-driver`. Verify that a driver JAR looks correct."
+ (:require [build-drivers.verify :as verify]
+ [metabuild-common.core :as u]))
+
+(defn -main [& [driver]]
+ (u/exit-when-finished-nonzero-on-exception
+ (when-not (seq driver)
+ (throw (ex-info "Usage: clojure -m verify-driver <driver>" {})))
+ (verify/verify-driver (keyword driver))))
diff --git a/bin/check-clojure-cli.sh b/bin/check-clojure-cli.sh
new file mode 100755
index 00000000000..0fdb8cd5df4
--- /dev/null
+++ b/bin/check-clojure-cli.sh
@@ -0,0 +1,33 @@
+#! /usr/bin/env bash
+
+set -eou pipefail
+
+you_need_to_upgrade() {
+ echo "Clojure CLI must be at least version 1.10.1.708. Your version is $version."
+ echo "See https://www.clojure.org/guides/getting_started for upgrade instructions."
+ exit -3
+}
+
+check_clojure_cli() {
+ if [ ! `which clojure` ]; then
+ echo "Please install the Clojure command line tools. See https://www.clojure.org/guides/getting_started for instructions."
+ exit -2
+ fi
+
+ version=`clojure --help | grep Version`
+ minor_version=`echo "$version" | cut -d '.' -f 2`
+ patch_version=`echo "$version" | cut -d '.' -f 3`
+ build_version=`echo "$version" | cut -d '.' -f 4`
+
+ if [ "$minor_version" -lt "10" ]; then
+ you_need_to_upgrade
+ elif [ "$minor_version" -eq "10" ]; then
+ if [ "$patch_version" -lt "1" ]; then
+ you_need_to_upgrade
+ elif [ "$patch_version" -eq "1" ]; then
+ if [ "$build_version" -lt "708" ]; then
+ you_need_to_upgrade
+ fi
+ fi
+ fi
+}
diff --git a/bin/metabuild_common/core.clj b/bin/metabuild_common/core.clj
new file mode 100644
index 00000000000..023b96c7e16
--- /dev/null
+++ b/bin/metabuild_common/core.clj
@@ -0,0 +1,38 @@
+(ns metabuild-common.core
+ (:require [metabuild-common
+ [entrypoint :as entrypoint]
+ [files :as files]
+ [output :as output]
+ [shell :as shell]
+ [steps :as steps]]
+ [potemkin :as p]))
+
+(comment entrypoint/keep-me
+ files/keep-me
+ output/keep-me
+ shell/keep-me
+ steps/keep-me)
+
+(p/import-vars
+ [entrypoint
+ exit-when-finished-nonzero-on-exception]
+
+ [files
+ assert-file-exists
+ copy-file!
+ create-directory-unless-exists!
+ delete-file!
+ file-exists?
+ find-files]
+
+ [output
+ announce
+ error
+ safe-println]
+
+ [shell
+ sh
+ sh*]
+
+ [steps
+ step])
diff --git a/bin/metabuild_common/entrypoint.clj b/bin/metabuild_common/entrypoint.clj
new file mode 100644
index 00000000000..c7767ad3264
--- /dev/null
+++ b/bin/metabuild_common/entrypoint.clj
@@ -0,0 +1,20 @@
+(ns metabuild-common.entrypoint
+ (:require [clojure.pprint :as pprint]
+ [colorize.core :as colorize]))
+
+(defn do-exit-when-finished-nonzero-on-exception [thunk]
+ (try
+ (thunk)
+ (System/exit 0)
+ (catch Throwable e
+ (let [e-map (Throwable->map e)]
+ (println (colorize/red (str "Command failed: " (:cause e-map))))
+ (binding [pprint/*print-right-margin* 120]
+ (pprint/pprint e-map))))))
+
+(defmacro exit-when-finished-nonzero-on-exception
+ "Execute `body` and catch exceptions. If an Exception is thrown, exit with status code 0; if an exception was
+ thrown, print it and exit with a non-zero status code. Intended for use in `-main` functions."
+ {:style/indent 0}
+ [& body]
+ `(do-exit-when-finished-nonzero-on-exception (fn [] ~@body)))
diff --git a/bin/metabuild_common/files.clj b/bin/metabuild_common/files.clj
new file mode 100644
index 00000000000..efda641bfa4
--- /dev/null
+++ b/bin/metabuild_common/files.clj
@@ -0,0 +1,79 @@
+(ns metabuild-common.files
+ (:require [metabuild-common
+ [output :as out]
+ [shell :as sh]
+ [steps :as steps]])
+ (:import java.io.File
+ [java.nio.file Files FileVisitOption Path Paths]
+ java.util.function.BiPredicate
+ org.apache.commons.io.FileUtils))
+
+(defn file-exists?
+ "Does a file or directory with `filename` exist?"
+ [^String filename]
+ (when filename
+ (.exists (File. filename))))
+
+(defn assert-file-exists
+ "If file with `filename` exists, return `filename` as is; otherwise, throw Exception."
+ ^String [filename & [message]]
+ (when-not (file-exists? filename)
+ (throw (ex-info (format "File %s does not exist. %s" (pr-str filename) (or message "")) {:filename filename})))
+ (str filename))
+
+(defn create-directory-unless-exists! [^String dir]
+ (when-not (file-exists? dir)
+ (steps/step (format "Creating directory %s..." dir)
+ (.mkdirs (File. dir))))
+ dir)
+
+(defn delete-file!
+ "Delete a file or directory (recursively) if it exists."
+ ([^String filename]
+ (steps/step (format "Deleting %s..." filename)
+ (if (file-exists? filename)
+ (let [file (File. filename)]
+ (if (.isDirectory file)
+ (FileUtils/deleteDirectory file)
+ (.delete file))
+ (out/safe-println (format "Deleted %s." filename)))
+ (out/safe-println (format "Don't need to delete %s, file does not exist." filename)))
+ (assert (not (file-exists? filename)))))
+
+ ([file & more]
+ (dorun (map delete-file! (cons file more)))))
+
+(defn copy-file!
+ "Copy a `source` file (or directory, recursively) to `dest`."
+ [^String source ^String dest]
+ (let [source-file (File. (assert-file-exists source))
+ dest-file (File. dest)]
+ ;; Use native `cp` rather than FileUtils or the like because codesigning is broken when you use those because they
+ ;; don't preserve symlinks or something like that.
+ (if (.isDirectory source-file)
+ (steps/step (format "Copying directory %s -> %s" source dest)
+ (sh/sh "cp" "-R" source dest))
+ (steps/step (format "Copying file %s -> %s" source dest)
+ (sh/sh "cp" source dest))))
+ (assert-file-exists dest))
+
+(defn- ->URI ^java.net.URI [filename]
+ (java.net.URI. (str "file://" filename)))
+
+(defn- ->Path ^Path [filename]
+ (Paths/get (->URI filename)))
+
+(defn find-files
+ "Pure Java version of `find`. Recursively find files in `dir-path` that satisfy `pred`, which has the signature
+
+ (pred filename-string) -> Boolean"
+ [^String dir-path pred]
+ (->> (Files/find (->Path dir-path)
+ Integer/MAX_VALUE
+ (reify BiPredicate
+ (test [_ path _]
+ (boolean (pred (str path)))))
+ ^FileVisitOption (make-array FileVisitOption 0))
+ .toArray
+ (map str)
+ sort))
diff --git a/bin/metabuild_common/output.clj b/bin/metabuild_common/output.clj
new file mode 100644
index 00000000000..2f004a2774b
--- /dev/null
+++ b/bin/metabuild_common/output.clj
@@ -0,0 +1,34 @@
+(ns metabuild-common.output
+ (:require [clojure.string :as str]
+ [colorize.core :as colorize]))
+
+(def ^:dynamic *steps* [])
+
+(def ^:private step-indent (str/join (repeat 2 \space)))
+
+(defn- steps-indent []
+ (str/join (repeat (count *steps*) step-indent)))
+
+(defn safe-println
+ "Thread-safe version of `println` that also indents output based on the current step build step."
+ [& args]
+ (locking println
+ (print (steps-indent))
+ (apply println args)))
+
+(defn announce
+ "Like `safe-println` + `format`, but outputs text in magenta. Use this for printing messages such as when starting
+ build steps."
+ ([s]
+ (safe-println (colorize/magenta s)))
+
+ ([format-string & args]
+ (announce (apply format (str format-string) args))))
+
+(defn error
+ "Life `safe-println` + `format`, but outputs text in red. Use this for printing error messages or Exceptions."
+ ([s]
+ (safe-println (colorize/red s)))
+
+ ([format-string & args]
+ (error (apply format format-string args))))
diff --git a/bin/metabuild_common/shell.clj b/bin/metabuild_common/shell.clj
new file mode 100644
index 00000000000..da3951c54d8
--- /dev/null
+++ b/bin/metabuild_common/shell.clj
@@ -0,0 +1,76 @@
+(ns metabuild-common.shell
+ (:require [clojure.string :as str]
+ [colorize.core :as colorize]
+ [metabuild-common
+ [output :as out]
+ [steps :as steps]])
+ (:import [java.io BufferedReader File InputStreamReader]))
+
+(defn- read-lines [^java.io.BufferedReader reader {:keys [quiet? err?]}]
+ (loop [lines []]
+ (if-let [line (.readLine reader)]
+ (do
+ (when-not quiet?
+ (out/safe-println (if err? (colorize/red line) line)))
+ (recur (conj lines line)))
+ lines)))
+
+(defn- deref-with-timeout [dereffable timeout-ms]
+ (let [result (deref dereffable timeout-ms ::timed-out)]
+ (when (= result ::timed-out)
+ (throw (ex-info (format "Timed out after %d ms." timeout-ms) {})))
+ result))
+
+(def ^:private command-timeout-ms (* 15 60 1000)) ; 15 minutes
+
+(defn sh*
+ "Run a shell command. Like `clojure.java.shell/sh`, but prints output to stdout/stderr and returns results as a vector
+ of lines.
+
+ Options:
+
+ * `env` -- environment variables (as a map) to use when running `cmd`. If `:env` is `nil`, the default parent
+ environment (i.e., the environment in which this Clojure code itself is ran) will be used; if `:env` IS passed, it
+ completely replaces the parent environment in which this script is ran -- make sure you pass anything that might be
+ needed such as `JAVA_HOME` if you do this
+
+ * `dir` -- current directory to use when running the shell command. If not specified, command is run in the same
+ current directory as the Clojure scripts, `bin/build-drivers`
+
+ * `quiet?` -- whether to suppress output from this shell command."
+ {:arglists '([cmd & args] [{:keys [env dir quiet?]} cmd & args])}
+ [& args]
+ (steps/step (colorize/blue (str "$ " (str/join " " (map (comp pr-str str) (if (map? (first args))
+ (rest args)
+ args)))))
+ (let [[opts & args] (if (map? (first args))
+ args
+ (cons nil args))
+ {:keys [env dir]} opts
+ cmd-array (into-array (map str args))
+ env-array (when env
+ (assert (map? env))
+ (into-array String (for [[k v] env]
+ (format "%s=%s" (name k) (str v)))))
+ proc (.exec (Runtime/getRuntime)
+ ^"[Ljava.lang.String;" cmd-array
+ ^"[Ljava.lang.String;" env-array
+ ^File (when dir (File. ^String dir)))]
+ (with-open [out-reader (BufferedReader. (InputStreamReader. (.getInputStream proc)))
+ err-reader (BufferedReader. (InputStreamReader. (.getErrorStream proc)))]
+ (let [exit-code (future (.waitFor proc))
+ out (future (read-lines out-reader opts))
+ err (future (read-lines err-reader (assoc opts :err? true)))]
+ {:exit (deref-with-timeout exit-code command-timeout-ms)
+ :out (deref-with-timeout out command-timeout-ms)
+ :err (deref-with-timeout err command-timeout-ms)})))))
+
+(defn sh
+ "Like `sh*`, but throws an Exception if the command exits with a non-zero status. Options are the same as `sh*` -- see
+ its documentation for more information."
+ {:arglists '([cmd & args] [{:keys [env dir quiet?]} cmd & args])}
+ [& args]
+ (let [{:keys [exit out err], :as response} (apply sh* args)]
+ (if (zero? exit)
+ (concat out err)
+ (throw (ex-info (str/join "\n" (concat out err)) response)))))
diff --git a/bin/metabuild_common/steps.clj b/bin/metabuild_common/steps.clj
new file mode 100644
index 00000000000..62b6a81a1eb
--- /dev/null
+++ b/bin/metabuild_common/steps.clj
@@ -0,0 +1,30 @@
+(ns metabuild-common.steps
+ (:require [colorize.core :as colorize]
+ [metabuild-common.output :as out]))
+
+(defn do-step
+ "Impl for `step` macro."
+ [step thunk]
+ (out/safe-println (colorize/green (str step)))
+ (binding [out/*steps* (conj (vec out/*steps*) step)]
+ (try
+ (thunk)
+ (catch Throwable e
+ (throw (ex-info (str step) {} e))))))
+
+(defmacro step
+ "Start a new build step, which:
+
+ 1. Logs the `step` message
+ 2. Indents all output inside `body` by one level
+ 3. Catches any exceptions inside `body` and rethrows with additional context including `step` message
+
+ These are meant to be nested, e.g.
+
+ (step \"Build driver\"
+ (step \"Build dependencies\")
+ (step \"Build driver JAR\")
+ (step \"Verify driver\"))"
+ {:style/indent 1}
+ [step & body]
+ `(do-step ~step (fn [] ~@body)))
diff --git a/bin/metabuild_common/util.clj b/bin/metabuild_common/util.clj
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/bin/verify-driver b/bin/verify-driver
index 77a080f202b..4e463736085 100755
--- a/bin/verify-driver
+++ b/bin/verify-driver
@@ -1,50 +1,16 @@
#! /usr/bin/env bash
-set -euo pipefail
+set -eo pipefail
driver="$1"
if [ ! "$driver" ]; then
- echo 'Usage: ./bin/verify-driver [driver]'
+ echo "Usage: ./bin/verify-driver [driver]"
exit -1
fi
-echo "Verifying $driver driver..."
+source "./bin/check-clojure-cli.sh"
+check_clojure_cli
-driver_file="resources/modules/$driver.metabase-driver.jar"
-
-echo "Checking whether $driver_file exists...."
-if [ ! -f "$driver_file" ]; then
- echo 'File does not exist. Driver verification failed.'
- exit -2
-fi
-
-echo 'File exists.'
-
-# This assumes that the driver's main namespace is {driver}.clj, which is not necessarily required. Namespace is
-# determined by `load-namespace` in the plugin manifest
-#
-# TODO - this won't work for drivers that have slashes in the name, because of namespace munging
-munged_driver=`echo "$driver" | sed 's/-/_/g'`
-driver_main_class="metabase/driver/${munged_driver}__init.class"
-
-echo "Checking whether driver contains main class file $driver_main_class..."
-
-if [ `jar -tf "$driver_file" | grep "$driver_main_class"` ]; then
- echo 'Main class file found.'
-else
- echo 'Main class file missing. Driver verification failed.'
- exit -3
-fi
-
-echo "Checking whether driver contains plugin manifest..."
-
-if [ `jar -tf "$driver_file" | grep 'metabase-plugin.yaml'` ]; then
- echo 'Plugin manifest found.'
-else
- echo 'Plugin manifest missing. Driver verification failed.'
- exit -4
-fi
-
-echo 'Driver verification successful.'
-exit 0
+cd bin/build-drivers
+clojure -M -m verify-driver "$driver"
diff --git a/modules/drivers/bigquery/parents b/modules/drivers/bigquery/parents
deleted file mode 100644
index cb429113e0f..00000000000
--- a/modules/drivers/bigquery/parents
+++ /dev/null
@@ -1 +0,0 @@
-google
diff --git a/modules/drivers/googleanalytics/parents b/modules/drivers/googleanalytics/parents
deleted file mode 100644
index cb429113e0f..00000000000
--- a/modules/drivers/googleanalytics/parents
+++ /dev/null
@@ -1 +0,0 @@
-google
--
GitLab