Skip to content
Snippets Groups Projects
Commit 4315dbed authored by Cam Saul's avatar Cam Saul Committed by Cam Saul
Browse files

New driver build scripts (#13383) 

* New driver build scripts and fix Google driver build

* Install Clojure CLI in Circle CI

* Fix driver build script

* Build script fix :wrench:

* More dox

* Enforce minimum build Clojure CLI version and address feedback

* New metabuild-common directory for shared stuff. Address some PR feedback
parent ce3f06b4
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 836 additions and 272 deletions
.circleci/config.yml
+ 6
− 1
View file @ 4315dbed
......@@ -564,6 +564,12 @@ jobs:
- restore_cache:
keys:
- frontend-{{ checksum "./frontend-checksums.txt" }}
- run:
name: Install Clojure CLI
command: >
curl -O https://download.clojure.org/install/linux-install-1.10.1.708.sh &&
chmod +x linux-install-1.10.1.708.sh &&
sudo ./linux-install-1.10.1.708.sh
- run:
name: Build frontend if needed
command: >
......@@ -943,7 +949,6 @@ workflows:
requires:
- build-uberjar
- fe-deps
- fe-tests-cypress:
name: fe-tests-cypress-1
requires:
......
.gitignore
+ 1
− 0
View file @ 4315dbed
......@@ -8,6 +8,7 @@
*.xcworkspacedata
.DS_Store
.\#*
.cpcache/
.eastwood
.idea/
.nrepl-port
......
bin/build-driver.sh
+ 5
− 230
View file @ 4315dbed
#! /usr/bin/env bash
set -euo pipefail
project_root=`pwd`
set -eo pipefail
driver="$1"
......@@ -11,231 +9,8 @@ if [ ! "$driver" ]; then
exit -1
fi
driver_project_dir="$project_root/modules/drivers/$driver"
driver_jar="$driver.metabase-driver.jar"
dest_location="$project_root/resources/modules/$driver_jar"
metabase_uberjar="$project_root/target/uberjar/metabase.jar"
target_jar="$driver_project_dir/target/uberjar/$driver_jar"
parents=''
checksum_file="$driver_project_dir/target/checksum.txt"
################################ DELETING OLD INCORRECTLY BUILT DRIVERS ###############################
verify_existing_build() {
verification_failed=''
./bin/verify-driver "$driver" || verification_failed=true
if [ "$verification_failed" ]; then
echo 'No existing build, or existing build is invalid. (Re)building driver.'
# By removing the checksum it will force rebuilding the driver
rm -f "$checksum_file"
fi
}
######################################## CALCULATING CHECKSUMS ########################################
md5_command=''
if [ `command -v md5` ]; then
md5_command=md5
elif [ `command -v md5sum` ]; then
md5_command=md5sum
else
echo "Don't know what command to use to calculate md5sums."
exit -2
fi
# Calculate a checksum of all the driver source files. If we've already built the driver and the checksum is the same
# there's no need to build the driver a second time
calculate_checksum() {
find "$driver_project_dir" -name '*.clj' -or -name '*.yaml' | sort | xargs cat | $md5_command
}
# Check whether the saved checksum for the driver sources from the last build is the same as the current one. If so,
# we don't need to build again.
checksum_is_same() {
if [ -f "$checksum_file" ]; then
old_checksum=`cat "$checksum_file"`
current_checksum=`calculate_checksum`
echo "Checksum of source files for previous build: $old_checksum"
echo "Current checksum of source files: $current_checksum"
if [ "$current_checksum" == "$old_checksum" ]; then
# Make sure the target driver JAR actually exists as well!
if [ -f "$target_jar" ]; then
echo "$driver driver source unchanged since last build. Skipping re-build."
return 0
fi
fi
fi
return 1
}
######################################## BUILDING THE DRIVER ########################################
# Delete existing saved copies of the driver in the plugins and resources directories
delete_old_drivers() {
echo "Deleting old versions of $driver driver..."
rm -f plugins/"$driver_jar"
rm -f "$dest_location"
}
# Check if Metabase is installed locally for building drivers; install it if not
install_metabase_core() {
if [ ! "$(find ~/.m2/repository/metabase-core/metabase-core -name '*.jar')" ]; then
echo "Building Metabase and installing locally..."
lein clean
lein install-for-building-drivers
else
echo "metabase-core already installed to local Maven repo."
fi
}
# Build Metabase uberjar if needed, we'll need this for stripping duplicate classes
build_metabase_uberjar() {
if [ ! -f "$metabase_uberjar" ]; then
echo 'Building Metabase uberjar...'
lein uberjar
else
echo "Metabase uberjar already built."
fi
}
# Take a look at the `parents` file listing the parents to build, if applicable
build_parents() {
echo "Building parent drivers (if needed)..."
parents_list="$driver_project_dir"/parents
if [ -f "$parents_list" ]; then
parents=`cat "$parents_list"`
echo "Found driver parents: $parents"
fi
# Check and see if we need to recursively build or install any of our parents before proceeding
for parent in $parents; do
if [ ! -f resources/modules/"$parent.metabase-driver.jar" ]; then
echo "Building $parent..."
./bin/build-driver.sh "$parent"
fi
# Check whether built parent driver *JAR* exists in local Maven repo
parent_install_dir="~/.m2/repository/metabase/$parent-driver"
parent_installed_jar=''
if [ -f "$parent_install_dir" ]; then
parent_installed_jar=`find "$parent_install_dir" -name '*.jar'`
fi
source "./bin/check-clojure-cli.sh"
check_clojure_cli
if [ ! "$parent_installed_jar" ]; then
parent_project_dir="$project_root/modules/drivers/$parent"
echo "Installing $parent locally..."
cd "$parent_project_dir"
lein clean
lein install-for-building-drivers
cd "$project_root"
else
echo "$parent already installed to local Maven repo"
fi
done
}
# Build the driver uberjar itself
build_driver_uberjar() {
echo "Building $driver driver..."
cd "$driver_project_dir"
rm -rf target
lein clean
DEBUG=1 LEIN_SNAPSHOTS_IN_RELEASE=true lein uberjar
cd "$project_root"
if [ ! -f "$target_jar" ]; then
echo "Error: could not find $target_jar. Build failed."
return -3
fi
}
# Strip out any classes in driver JAR found in core Metabase uberjar or parent JARs; recompress with higher compression ratio
strip_and_compress() {
# ok, first things first, strip out any classes also found in the core Metabase uberjar
lein strip-and-compress "$target_jar"
# next, remove any classes also found in any of the parent JARs
for parent in $parents; do
echo "Removing duplicate classes with $parent uberjar..."
lein strip-and-compress "$target_jar" "resources/modules/$parent.metabase-driver.jar"
done
}
# copy finished JAR to the resources dir
copy_target_to_dest() {
echo "Copying $target_jar -> $dest_location"
cp "$target_jar" "$dest_location"
}
# check that JAR in resources dir looks correct
verify_build () {
verification_failed=''
./bin/verify-driver "$driver" || verification_failed=true
if [ "$verification_failed" ]; then
echo "./bin/build-driver.sh $driver FAILED."
rm -f "$checksum_file"
rm -f "$target_jar"
rm -f "$dest_location"
return -4
fi
}
# Save the checksum for the newly built JAR
save_checksum() {
echo "Saving checksum for source files to $checksum_file"
checksum=`calculate_checksum`
echo "$checksum" > "$checksum_file"
}
# Runs all the steps needed to build the driver.
build_driver() {
verify_existing_build &&
delete_old_drivers &&
install_metabase_core &&
build_metabase_uberjar &&
build_parents &&
build_driver_uberjar &&
strip_and_compress &&
copy_target_to_dest &&
verify_build &&
save_checksum
}
######################################## PUTTING IT ALL TOGETHER ########################################
clean_local_repo() {
echo "Deleting existing installed metabase-core and driver dependencies..."
rm -rf ~/.m2/repository/metabase-core
rm -rf ~/.m2/repository/metabase/*-driver
}
retry_clean_build() {
echo "Building without cleaning failed. Retrying clean build..."
clean_local_repo
build_driver
}
mkdir -p resources/modules
# run only a specific step with ./bin/build-driver.sh <driver> <step>
if [ $# -eq 2 ]; then
$2
# Build driver if checksum has changed
elif ! checksum_is_same; then
echo "Checksum has changed."
build_driver || retry_clean_build
# Either way, always copy the target uberjar to the dest location
else
echo "Checksum is unchanged."
(copy_target_to_dest && verify_build) || retry_clean_build
fi
cd bin/build-drivers
clojure -M -m build-driver "$driver"
bin/build-drivers.sh
+ 5
− 41
View file @ 4315dbed
#! /usr/bin/env bash
set -eo pipefail
set -euo pipefail
# If ran as `./bin/build-drivers.sh clean` then uninstall metabase-core from the local Maven repo and delete
if [ "$1" == clean ]; then
echo "Deleting existing installed metabase-core and driver dependencies..."
rm -rf ~/.m2/repository/metabase-core
rm -rf ~/.m2/repository/metabase/*-driver
source "./bin/check-clojure-cli.sh"
check_clojure_cli
echo "Deleting built drivers in resources/modules..."
rm -rf resources/modules
echo "Deleting build Metabase uberjar..."
rm -rf target
for target in `find modules -name 'target' -type d`; do
echo "Deleting $target..."
rm -rf "$target"
done
fi
# strip trailing slashes if `ls` is set to include them
drivers=`ls modules/drivers/ | sed 's|/$||'`
for driver in $drivers; do
echo "Build: $driver"
build_failed=''
./bin/build-driver.sh "$driver" || build_failed=true
if [ "$build_failed" ]; then
echo "Failed to build driver $driver."
exit -1
fi
done
# Double-check that all drivers were built successfully
for driver in $drivers; do
verification_failed=''
./bin/verify-driver "$driver" || verification_failed=true
if [ "$verification_failed" ]; then
exit -2
fi
done
cd bin/build-drivers
clojure -M -m build-drivers
bin/build-drivers/README.md 0 → 100644
+ 45
− 0
View file @ 4315dbed
# Build-drivers scripts
Scripts for building Metabase driver plugins. You must install the [Clojure CLI
tools](https://www.clojure.org/guides/getting_started) to use these.
There are three main entrypoints. Shell script wrappers are provided for convenience and compatibility.
### `build-drivers`
Builds *all* drivers as needed. If drivers were recently built and no relevant source code changed, skips rebuild.
```
cd bin/build-drivers
clojure -M -m build-drivers
# or
./bin/build-drivers.sh
```
### `build-driver`
Build a single driver as needed. Builds parent drivers if needed first.
```
cd bin/build-driver redshift
clojure -M -m build-driver redshift
# or
./bin/build-driver.sh redshift
```
### `verify-driver`
Verify that a built driver looks correctly built.
```
cd bin/verify-driver redshift
clojure -M -m verify-driver redshift
# or
./bin/verify-driver redshift
```
bin/build-drivers/build_driver.clj 0 → 100644
+ 10
− 0
View file @ 4315dbed
(ns build-driver
"Entrypoint for `bin/build-driver.sh`. Builds a single driver, if needed."
(:require [build-drivers.build-driver :as build-driver]
[metabuild-common.core :as u]))
(defn -main [& [driver]]
(u/exit-when-finished-nonzero-on-exception
(when-not (seq driver)
(throw (ex-info "Usage: clojure -m build-driver <driver>" {})))
(build-driver/build-driver! (keyword driver))))
bin/build-drivers/build_drivers.clj 0 → 100644
+ 20
− 0
View file @ 4315dbed
(ns build-drivers
"Entrypoint for `bin/build-drivers.sh`. Builds all drivers, if needed."
(:require [build-drivers
[build-driver :as build-driver]
[common :as c]]
[clojure.java.io :as io]
[metabuild-common.core :as u]))
(defn- all-drivers []
(map keyword (.list (io/file (c/filename c/project-root-directory "modules" "drivers")))))
(defn- build-drivers! []
(u/step "Building all drivers"
(doseq [driver (all-drivers)]
(build-driver/build-driver! driver))
(u/announce "Successfully built all drivers.")))
(defn -main []
(u/exit-when-finished-nonzero-on-exception
(build-drivers!)))
bin/build-drivers/build_drivers/build_driver.clj 0 → 100644
+ 168
− 0
View file @ 4315dbed
(ns build-drivers.build-driver
"Logic for building a single driver."
(:require [build-drivers
[checksum :as checksum]
[common :as c]
[install-driver-locally :as install-locally]
[metabase :as metabase]
[plugin-manifest :as manifest]
[verify :as verify]]
[colorize.core :as colorize]
[environ.core :as env]
[metabuild-common.core :as u]))
(defn- copy-driver!
"Copy the driver JAR from its `target/` directory to `resources/modules`/."
[driver]
(u/step (format "Copy %s driver uberjar from %s -> %s"
driver
(u/assert-file-exists (c/driver-jar-build-path driver))
(c/driver-jar-destination-path driver))
(u/delete-file! (c/driver-jar-destination-path driver))
(u/create-directory-unless-exists! c/driver-jar-destination-directory)
(u/copy-file! (c/driver-jar-build-path driver)
(c/driver-jar-destination-path driver))))
(defn- clean-driver-artifacts!
"Delete built JARs of `driver`."
[driver]
(u/step (format "Delete %s driver artifacts" driver)
(u/delete-file! (c/driver-target-directory driver))
(u/delete-file! (c/driver-jar-destination-path driver))))
(defn- clean-parents!
"Delete built JARs and local Maven installations of the parent drivers of `driver`."
[driver]
(u/step (format "Clean %s parent driver artifacts" driver)
(doseq [parent (manifest/parent-drivers driver)]
(clean-driver-artifacts! parent)
(install-locally/clean! parent)
(clean-parents! parent))))
(defn- clean-all!
"Delete all artifacts relating to building `driver`, including the driver JAR itself and installed
`metabase-core`/Metabase uberjar and any parent driver artifacts."
[driver]
(u/step "Clean all"
(clean-driver-artifacts! driver)
(clean-parents! driver)
(metabase/clean-metabase!)))
(declare build-driver!)
(defn- build-parents!
"Build and install to the local Maven repo any parent drivers of `driver` (e.g. `:google` is a parent of `:bigquery`).
The driver must be built as an uberjar so we can remove duplicate classes during the `strip-and-compress` stage; it
must be installed as a library so we can use it as a `:provided` dependency when building the child driver."
[driver]
(u/step (format "Build %s parent drivers" driver)
(doseq [parent (manifest/parent-drivers driver)]
(build-parents! parent)
(install-locally/install-locally! parent)
(build-driver! parent))
(u/announce "%s parents built successfully." driver)))
(defn- strip-and-compress-uberjar!
"Remove any classes in compiled `driver` that are also present in the Metabase uberjar or parent drivers. The classes
will be available at runtime, and we don't want to make things unpredictable by including them more than once in
different drivers.
This is only needed because `lein uberjar` does not seem to reliably exclude classes from `:provided` Clojure
dependencies like `metabase-core` and the parent drivers."
([driver]
(u/step (str (format "Strip out any classes in %s driver JAR found in core Metabase uberjar or parent JARs" driver)
" and recompress with higher compression ratio")
(let [uberjar (u/assert-file-exists (c/driver-jar-build-path driver))]
(u/step "strip out any classes also found in the core Metabase uberjar"
(strip-and-compress-uberjar! uberjar (u/assert-file-exists c/metabase-uberjar-path)))
(u/step "remove any classes also found in any of the parent JARs"
(doseq [parent (manifest/parent-drivers driver)]
(strip-and-compress-uberjar! uberjar (u/assert-file-exists (c/driver-jar-build-path parent))))))))
([target source]
(u/step (format "Remove classes from %s that are present in %s and recompress" target source)
(u/sh {:dir c/project-root-directory}
"lein"
"strip-and-compress"
(u/assert-file-exists target)
(u/assert-file-exists source)))))
(defn- build-uberjar! [driver]
(u/step (format "Build %s uberjar" driver)
(u/delete-file! (c/driver-target-directory driver))
(u/sh {:dir (c/driver-project-dir driver)} "lein" "clean")
(u/sh {:dir (c/driver-project-dir driver)
:env {"LEIN_SNAPSHOTS_IN_RELEASE" "true"
#_"DEBUG" #_"1"
"JAVA_HOME" (env/env :java-home)
"HOME" (env/env :user-home)}}
"lein" "uberjar")
(strip-and-compress-uberjar! driver)
(u/announce "%s uberjar build successfully." driver)))
(defn- build-and-verify!
"Build `driver` and verify the built JAR. This function ignores any existing artifacts and will always rebuild."
[driver]
(u/step (str (colorize/green "Build ") (colorize/yellow driver) (colorize/green " driver"))
(clean-driver-artifacts! driver)
(metabase/build-metabase!)
(build-parents! driver)
(build-uberjar! driver)
(copy-driver! driver)
(verify/verify-driver driver)
(u/step (format "Save checksum for %s driver to %s" driver (c/driver-checksum-filename driver))
(spit (c/driver-checksum-filename driver) (checksum/driver-checksum driver)))))
(defn- driver-checksum-matches?
"Check whether the saved checksum for the driver from the last build is the same as the current one. If so, we don't
need to build again. This checksum is based on driver sources as well as the checksums for Metabase sources and
parent drivers."
[driver]
(u/step (format "Determine whether %s driver source files have changed since last build" driver)
(let [existing-checksum (checksum/existing-driver-checksum driver)
current-checksum (checksum/driver-checksum driver)
same? (= existing-checksum current-checksum)]
(u/announce (if same?
"Checksum is the same. Do not need to rebuild driver."
"Checksum is different. Need to rebuild driver."))
same?)))
(defn build-driver!
"Build `driver`, if needed."
[driver]
{:pre [(keyword? driver)]}
(u/step (str (colorize/green "Build ") (colorize/yellow driver) (colorize/green " driver if needed"))
;; When we build a driver, we save a checksum of driver source code + metabase source code + parent drivers
;; alongside the built driver JAR. The next time this script is called, we recalculate that checksum -- if the
;; current checksum matches the saved one associated with the built driver JAR, we do not need to rebuild the
;; driver. If anything relevant has changed, we have to rebuild the driver.
(if (driver-checksum-matches? driver)
;; even if we're not rebuilding the driver, copy the artifact from `modules/drivers/<driver>/target/uberjar/`
;; to `resources/modules` so we can be sure we have the most up-to-date version there.
(try
(verify/verify-driver driver)
(copy-driver! driver)
;; if verification fails, delete all the existing artifacts and just rebuild the driver from scratch.
(catch Throwable e
(u/error "Error verifying existing driver:\n%s" (pr-str e))
(u/announce "Deleting existing driver artifacts and rebuilding.")
(clean-driver-artifacts! driver)
(build-driver! driver)))
;; if checksum does not match, build and verify the driver
(try
(build-and-verify! driver)
;; if building fails, clean everything, including metabase-core, the metabase uberjar, and parent
;; dependencies, *then* retry.
(catch Throwable e
(u/announce "Cleaning ALL and retrying...")
(clean-all! driver)
(try
(build-and-verify! driver)
;; if building the driver failed again, even after cleaning, delete anything that was built and then
;; give up.
(catch Throwable e
(u/safe-println (colorize/red (format "Failed to build %s driver." driver)))
(clean-driver-artifacts! driver)
(throw e))))))
;; if we make it this far, we've built the driver successfully.
(u/announce "Success.")))
bin/build-drivers/build_drivers/checksum.clj 0 → 100644
+ 76
− 0
View file @ 4315dbed
(ns build-drivers.checksum
"Shared code for calculating and reading hex-encoded MD5 checksums for relevant files."
(:require [build-drivers
[common :as c]
[plugin-manifest :as manifest]]
[clojure.java.io :as io]
[clojure.string :as str]
[metabuild-common.core :as u])
(:import org.apache.commons.codec.digest.DigestUtils))
(defn checksum-from-file
"Read a saved MD5 hash checksum from a file."
[filename]
(u/step (format "Read saved checksum from %s" filename)
(let [file (io/file filename)]
(if-not (.exists file)
(u/announce "%s does not exist" filename)
(when-let [[checksum-line] (not-empty (str/split-lines (slurp file)))]
(when-let [[_ checksum-hex] (re-matches #"(^[0-9a-f]{32}).*$" checksum-line)]
(u/safe-println (format "Saved checksum is %s" checksum-hex))
checksum-hex))))))
;;; -------------------------------------------- Metabase source checksum --------------------------------------------
(defn- metabase-source-paths []
(sort
(cons
(c/filename c/project-root-directory "project.clj")
(mapcat (fn [dir]
(try
(u/find-files dir #(str/ends-with? % ".clj"))
(catch Throwable _
[])))
[(c/filename c/project-root-directory "src")
(c/filename c/project-root-directory "enterprise" "backend" "src")
(c/filename c/project-root-directory "backend" "mbql")]))))
(defn metabase-source-checksum
"Checksum of Metabase backend source files and `project.clj`."
^String []
(let [paths (metabase-source-paths)]
(u/step (format "Calculate checksum for %d Metabase source files" (count paths))
(let [checksum (DigestUtils/md5Hex (str/join (map slurp paths)))]
(u/safe-println (format "Current checksum is %s" checksum))
checksum))))
;;; ---------------------------------------------- Driver source files -----------------------------------------------
(defn existing-driver-checksum
"Checksum from the relevant sources from last time we built `driver`."
[driver]
(checksum-from-file (c/driver-checksum-filename driver)))
(defn- driver-source-paths
"Returns sequence of the source filenames for `driver`."
[driver]
(u/find-files (c/driver-project-dir driver)
(fn [path]
(or (and (str/ends-with? path ".clj")
(not (str/starts-with? path (c/filename (c/driver-project-dir driver) "test"))))
(str/ends-with? path ".yaml")))))
(defn driver-checksum
"The driver checksum is based on a checksum of all the driver source files (`.clj` files and the plugin manifest YAML
file) combined with the checksums for `metabase-core` *and* the parent drivers. After building a driver, we save
this checksum. Next time the script is ran, we recalculate the checksum to determine whether anything relevant has
changed -- if it has, and the current checksum doesn't match the saved one, we need to rebuild the driver."
^String [driver]
(let [source-paths (driver-source-paths driver)]
(u/step (format "Calculate checksum for %d files: %s ..." (count source-paths) (first source-paths))
(let [checksum (DigestUtils/md5Hex (str/join (concat [(metabase-source-checksum)]
(map driver-checksum (manifest/parent-drivers driver))
(map slurp (driver-source-paths driver)))))]
(u/safe-println (format "Current checksum is %s" checksum))
checksum))))
bin/build-drivers/build_drivers/common.clj 0 → 100644
+ 75
− 0
View file @ 4315dbed
(ns build-drivers.common
"Shared constants and functions related to source and artifact paths used throughout this code."
(:require [clojure.string :as str]
[environ.core :as env])
(:import java.io.File))
;; since this file is used pretty much everywhere, this seemed like a good place to put this.
(set! *warn-on-reflection* true)
(when-not (str/ends-with? (env/env :user-dir) "/build-drivers")
(throw (ex-info "Please run build-driver scripts from the `bin/build-drivers` directory e.g. `cd bin/build-drivers; clojure -m build-driver`"
{:user-dir (env/env :user-dir)})))
(defn env-or-throw
"Fetch an env var value or throw an Exception if it is unset."
[k]
(or (get env/env k)
(throw (Exception. (format "%s is unset. Please set it and try again." (str/upper-case (str/replace (name k) #"-" "_")))))))
(defn filename [& path-components]
(str/join File/separatorChar path-components))
(def ^String project-root-directory
"e.g. /Users/cam/metabase"
(.. (File. ^String (env/env :user-dir)) getParentFile getParentFile getAbsolutePath))
(def ^String maven-repository-path
(filename (env/env :user-home) ".m2" "repository"))
;;; -------------------------------------------------- Driver Paths --------------------------------------------------
(defn driver-project-dir
"e.g. \"/home/cam/metabase/modules/drivers/redshift\""
[driver]
(filename project-root-directory "modules" "drivers" (name driver)))
(defn driver-jar-name
"e.g. \"redshift.metabase-driver.jar\""
[driver]
(format "%s.metabase-driver.jar" (name driver)))
(defn driver-target-directory
[driver]
(filename (driver-project-dir driver) "target"))
(defn driver-jar-build-path
"e.g. \"/home/cam/metabase/modules/drivers/redshift/target/uberjar/redshift.metabase-driver.jar\""
[driver]
(filename (driver-target-directory driver) "uberjar" (driver-jar-name driver)))
(def ^String driver-jar-destination-directory
(filename project-root-directory "resources" "modules"))
(defn driver-jar-destination-path
"e.g. \"/home/cam/metabase/resources/modules/redshift.metabase-driver.jar\""
[driver]
(filename driver-jar-destination-directory (driver-jar-name driver)))
(defn driver-checksum-filename
"e.g. \"/home/cam/metabase/modules/drivers/redshift/target/checksum.txt\""
[driver]
(filename (driver-project-dir driver) "target" "checksum.txt")) ; TODO - rename to checksum.md5
(defn driver-plugin-manifest-filename
"e.g. \"/home/cam/metabase/modules/drivers/bigquery/resources/plugin-manifest.yaml\""
[driver]
(filename (driver-project-dir driver) "resources" "metabase-plugin.yaml"))
;;; ------------------------------------------ Metabase Local Install Paths ------------------------------------------
(def ^String metabase-uberjar-path
"e.g. \"home/cam/metabase/target/uberjar/metabase.jar\""
(filename project-root-directory "target" "uberjar" "metabase.jar"))
bin/build-drivers/build_drivers/install_driver_locally.clj 0 → 100644
+ 50
− 0
View file @ 4315dbed
(ns build-drivers.install-driver-locally
"Logic related to installing a driver as a library in the local Maven repository so it can be used as a dependency
when building descandant drivers. Right now this is only used for `:google`, which is used by `:bigquery` and
`:googleanalytics`."
(:require [build-drivers
[checksum :as checksum]
[common :as c]]
[colorize.core :as colorize]
[metabuild-common.core :as u]))
(defn- driver-local-install-path [driver]
(c/filename c/maven-repository-path "metabase" (format "%s-driver" (name driver))))
(defn- driver-local-install-checksum-filename [driver]
(c/filename (driver-local-install-path driver) "checksum.md5"))
(defn clean!
"Delete local Maven installation of the library version of `driver`."
[driver]
(u/step (format "Deleting existing Maven installation of %s driver" driver)
(u/delete-file! (driver-local-install-path driver))))
(defn- local-install-checksum-matches?
"After installing the library version of `driver`, we save a checksum based on its sources; next time we call
`install-locally!`, we can recalculate the checksum; if the saved one matches the current one, we do not need to
reinstall."
[driver]
(u/step "Determine whether %s driver source files have changed since last local install"
(let [existing-checksum (checksum/checksum-from-file (driver-local-install-checksum-filename driver))
current-checksum (checksum/driver-checksum driver)
same? (= existing-checksum current-checksum)]
(u/announce (if same?
"Checksum is the same. Do not need to rebuild driver."
"Checksum is different. Need to rebuild driver."))
same?)))
(defn install-locally!
"Install `driver` as a library in the local Maven repository IF NEEDED so descendant drivers can use it as a
`:provided` dependency when building. E.g. before building `:bigquery` we need to install `:google` as a library
locally."
[driver]
{:pre [(keyword? driver)]}
(u/step (str (colorize/green "Install ") (colorize/yellow driver) (colorize/green " driver to local Maven repo if needed"))
(if (local-install-checksum-matches? driver)
(u/announce "Already installed locally.")
(u/step (str (colorize/green "Install ") (colorize/yellow driver) (colorize/green " driver to local Maven repo"))
(u/sh {:dir (c/driver-project-dir driver)} "lein" "clean")
(u/sh {:dir (c/driver-project-dir driver)} "lein" "install-for-building-drivers")
(u/step (format "Save checksum to %s" driver (driver-local-install-checksum-filename driver))
(spit (driver-local-install-checksum-filename driver) (checksum/driver-checksum driver)))))))
bin/build-drivers/build_drivers/metabase.clj 0 → 100644
+ 88
− 0
View file @ 4315dbed
(ns build-drivers.metabase
"Code for installing the main Metabase project as a library (`metabase-core`) in the local Maven repository, and for
building a Metabase uberjar. Both are needed when building drivers."
(:require [build-drivers
[checksum :as checksum]
[common :as c]]
[metabuild-common.core :as u]))
(def ^String ^:private uberjar-checksum-path
(str c/metabase-uberjar-path ".md5"))
(def ^String ^:private metabase-core-install-path
(c/filename c/maven-repository-path "metabase-core"))
(def ^String ^:private metabase-core-checksum-path
(c/filename metabase-core-install-path "checksum.md5"))
(defn metabase-core-checksum-matches? []
(u/step "Determine whether Metabase source files checksum has changed since last install of metabase-core"
(let [existing-checksum (checksum/checksum-from-file metabase-core-checksum-path)
current-checksum (checksum/metabase-source-checksum)
same? (= existing-checksum current-checksum)]
(u/announce (if same?
"Checksum is the same. Do not need to reinstall metabase-core locally."
"Checksum is different. Need to reinstall metabase-core locally."))
same?)))
(defn- delete-metabase-core-install! []
(u/step "Delete local installation of metabase-core"
(u/delete-file! metabase-core-install-path)))
(defn- install-metabase-core! []
(u/step "Install metabase-core locally if needed"
(if (metabase-core-checksum-matches?)
(u/announce "Up-to-date metabase-core already installed to local Maven repo")
(do
(delete-metabase-core-install!)
(u/sh {:dir c/project-root-directory} "lein" "clean")
(u/sh {:dir c/project-root-directory} "lein" "install-for-building-drivers")
(u/step "Save checksum for local installation of metabase-core"
(spit metabase-core-checksum-path (checksum/metabase-source-checksum)))
(u/announce "metabase-core dep installed to local Maven repo successfully.")))))
(defn uberjar-checksum-matches?
"After installing/building Metabase we save a MD5 hex checksum of Metabase backend source files (including
`project.clj`). The next time we run `build-metabase!`, if the checksums have changed we know we need to
rebuild/reinstall."
[]
(u/step "Determine whether Metabase source files checksum has changed since last build of uberjar"
(let [existing-checksum (checksum/checksum-from-file uberjar-checksum-path)
current-checksum (checksum/metabase-source-checksum)
same? (= existing-checksum current-checksum)]
(u/announce (if same?
"Checksum is the same. Do not need to rebuild Metabase uberjar."
"Checksum is different. Need to rebuild Metabase uberjar."))
same?)))
(defn- delete-metabase-uberjar! []
(u/step "Delete exist metabase uberjar"
(u/delete-file! (c/filename c/project-root-directory "target"))))
(defn- build-metabase-uberjar! []
(u/step "Build Metabase uberjar if needed"
(if (uberjar-checksum-matches?)
(u/announce "Update-to-date uberjar already built")
(do
(delete-metabase-uberjar!)
(u/sh {:dir c/project-root-directory} "lein" "clean")
(u/sh {:dir c/project-root-directory} "lein" "uberjar")
(u/step "Save checksum for Metabase uberar"
(spit uberjar-checksum-path (checksum/metabase-source-checksum)))
(u/announce "Metabase uberjar built successfully")))))
(defn clean-metabase!
"Delete local Maven repository installation of the `metabase-core` library and delete the built Metabase uberjar."
[]
(u/step "Clean local Metabase deps"
(delete-metabase-core-install!)
(delete-metabase-uberjar!)))
(defn build-metabase!
"Install `metabase-core` as a library in the local Maven repo, and build the Metabase uberjar IF NEEDED. We need to do
both because `metabase-core` is used as a dependency for drivers, and the Metabase uberjar is checked to make sure
we don't ship duplicate classes in the driver JAR (as part of the `strip-and-compress` stage.)"
[]
(u/step "Build metabase-core and install locally"
(install-metabase-core!)
(build-metabase-uberjar!)))
bin/build-drivers/build_drivers/plugin_manifest.clj 0 → 100644
+ 55
− 0
View file @ 4315dbed
(ns build-drivers.plugin-manifest
"Code for reading the YAML plugin manifest for a driver. "
(:require [build-drivers.common :as c]
[metabuild-common.core :as u]
[yaml.core :as yaml]))
(defn- plugin-manifest
"Read `driver` plugin manifest and return a map."
[driver]
{:post [(map? %)]}
(yaml/from-file (u/assert-file-exists (c/driver-plugin-manifest-filename driver))))
(defn- driver-declarations [manifest]
;; driver plugin manifest can have a single `:driver`, or multiple drivers, e.g. Spark SQL which also has the
;; `:hive-like` abstract driver
(let [{driver-declaration :driver} manifest]
(if (map? driver-declaration)
[driver-declaration]
driver-declaration)))
(defn- declared-drivers
"Sequence of all drivers declared in a plugin `manifest`. Usually only one driver, except for Spark SQL which declares
both `:hive-like` and `:sparksql`."
[manifest]
(map (comp keyword :name) (driver-declarations manifest)))
(def ^:private metabase-core-drivers
"Drivers that ship as part of the core Metabase project (as opposed to a plugin) and thus do not need to be built."
#{:sql
:sql-jdbc
:mysql
:h2
:postgres})
(defn parent-drivers
"Get the parent drivers of a driver for purposes of building a driver. Excludes drivers that ship as part of
`metabase-core`, since we don't need to worry about building those.
e.g.
(parent-drivers :googleanalytics) ;-> (:google)"
[driver]
(let [manifest (plugin-manifest driver)
declared (declared-drivers manifest)]
(or (not-empty
(for [{parent-declaration :parent} (driver-declarations manifest)
:let [parents (if (string? parent-declaration)
[parent-declaration]
parent-declaration)]
parent parents
:let [parent (keyword parent)]
:when (and (not (contains? (set declared) parent))
(not (contains? metabase-core-drivers parent)))]
parent))
(u/announce "%s does not have any parents" driver))))
bin/build-drivers/build_drivers/verify.clj 0 → 100644
+ 39
− 0
View file @ 4315dbed
(ns build-drivers.verify
(:require [build-drivers.common :as c]
[colorize.core :as colorize]
[metabuild-common.core :as u]))
(defn- verify-exists [driver]
(let [filename (c/driver-jar-destination-path driver)]
(u/step (format "Check %s exists" filename)
(if (u/file-exists? filename)
(u/announce "File exists.")
(throw (ex-info (format "Driver verification failed: %s does not exist" filename) {}))))))
(defn- verify-has-init-class [driver]
(let [filename (c/driver-jar-destination-path driver)
driver-init-class (format "metabase/driver/%s__init.class" (munge (name driver)))]
(u/step (format "Check %s contains init class file %s" filename driver-init-class)
(if (some (partial = driver-init-class)
(u/sh {:quiet? true} "jar" "-tf" filename))
(u/announce "Driver init class file found.")
(throw (ex-info (format "Driver verification failed: init class file %s not found" driver-init-class) {}))))))
(defn- verify-has-plugin-manifest [driver]
(let [filename (c/driver-jar-destination-path driver)
driver-init-class (format "metabase/driver/%s__init.class" (munge (name driver)))]
(u/step (format "Check %s contains metabase-plugin.yaml" filename)
(if (some (partial = "metabase-plugin.yaml")
(u/sh {:quiet? true} "jar" "-tf" filename))
(u/announce "Plugin manifest found.")
(throw (ex-info (format "Driver verification failed: plugin manifest missing" driver-init-class) {}))))))
(defn verify-driver
"Run a series of checks to make sure `driver` was built correctly. Throws exception if any checks fail."
[driver]
{:pre [(keyword? driver)]}
(u/step (str (colorize/green "Verify ") (colorize/yellow driver) (colorize/green " driver"))
(verify-exists driver)
(verify-has-init-class driver)
(verify-has-plugin-manifest driver)
(u/announce (format "%s driver verification successful." driver))))
bin/build-drivers/deps.edn 0 → 100644
+ 13
− 0
View file @ 4315dbed
{:paths ["./" "../"]
:deps
{cheshire/cheshire {:mvn/version "5.8.1"}
commons-codec/commons-codec {:mvn/version "1.14"}
commons-io/commons-io {:mvn/version "2.6"}
colorize/colorize {:mvn/version "0.1.1"}
environ/environ {:mvn/version "1.1.0"}
hiccup/hiccup {:mvn/version "1.0.5"}
io.forward/yaml {:mvn/version "1.0.9"}
org.flatland/ordered {:mvn/version "1.5.7"}
potemkin/potemkin {:mvn/version "0.4.5"}
stencil/stencil {:mvn/version "0.5.0"}}}
bin/build-drivers/verify_driver.clj 0 → 100644
+ 10
− 0
View file @ 4315dbed
(ns verify-driver
"Entrypoint for `bin/verify-driver`. Verify that a driver JAR looks correct."
(:require [build-drivers.verify :as verify]
[metabuild-common.core :as u]))
(defn -main [& [driver]]
(u/exit-when-finished-nonzero-on-exception
(when-not (seq driver)
(throw (ex-info "Usage: clojure -m verify-driver <driver>" {})))
(verify/verify-driver (keyword driver))))
bin/check-clojure-cli.sh 0 → 100755
+ 33
− 0
View file @ 4315dbed
#! /usr/bin/env bash
set -eou pipefail
you_need_to_upgrade() {
echo "Clojure CLI must be at least version 1.10.1.708. Your version is $version."
echo "See https://www.clojure.org/guides/getting_started for upgrade instructions."
exit -3
}
check_clojure_cli() {
if [ ! `which clojure` ]; then
echo "Please install the Clojure command line tools. See https://www.clojure.org/guides/getting_started for instructions."
exit -2
fi
version=`clojure --help | grep Version`
minor_version=`echo "$version" | cut -d '.' -f 2`
patch_version=`echo "$version" | cut -d '.' -f 3`
build_version=`echo "$version" | cut -d '.' -f 4`
if [ "$minor_version" -lt "10" ]; then
you_need_to_upgrade
elif [ "$minor_version" -eq "10" ]; then
if [ "$patch_version" -lt "1" ]; then
you_need_to_upgrade
elif [ "$patch_version" -eq "1" ]; then
if [ "$build_version" -lt "708" ]; then
you_need_to_upgrade
fi
fi
fi
}
bin/metabuild_common/core.clj 0 → 100644
+ 38
− 0
View file @ 4315dbed
(ns metabuild-common.core
(:require [metabuild-common
[entrypoint :as entrypoint]
[files :as files]
[output :as output]
[shell :as shell]
[steps :as steps]]
[potemkin :as p]))
(comment entrypoint/keep-me
files/keep-me
output/keep-me
shell/keep-me
steps/keep-me)
(p/import-vars
[entrypoint
exit-when-finished-nonzero-on-exception]
[files
assert-file-exists
copy-file!
create-directory-unless-exists!
delete-file!
file-exists?
find-files]
[output
announce
error
safe-println]
[shell
sh
sh*]
[steps
step])
bin/metabuild_common/entrypoint.clj 0 → 100644
+ 20
− 0
View file @ 4315dbed
(ns metabuild-common.entrypoint
(:require [clojure.pprint :as pprint]
[colorize.core :as colorize]))
(defn do-exit-when-finished-nonzero-on-exception [thunk]
(try
(thunk)
(System/exit 0)
(catch Throwable e
(let [e-map (Throwable->map e)]
(println (colorize/red (str "Command failed: " (:cause e-map))))
(binding [pprint/*print-right-margin* 120]
(pprint/pprint e-map))))))
(defmacro exit-when-finished-nonzero-on-exception
"Execute `body` and catch exceptions. If an Exception is thrown, exit with status code 0; if an exception was
thrown, print it and exit with a non-zero status code. Intended for use in `-main` functions."
{:style/indent 0}
[& body]
`(do-exit-when-finished-nonzero-on-exception (fn [] ~@body)))
bin/metabuild_common/files.clj 0 → 100644
+ 79
− 0
View file @ 4315dbed
(ns metabuild-common.files
(:require [metabuild-common
[output :as out]
[shell :as sh]
[steps :as steps]])
(:import java.io.File
[java.nio.file Files FileVisitOption Path Paths]
java.util.function.BiPredicate
org.apache.commons.io.FileUtils))
(defn file-exists?
"Does a file or directory with `filename` exist?"
[^String filename]
(when filename
(.exists (File. filename))))
(defn assert-file-exists
"If file with `filename` exists, return `filename` as is; otherwise, throw Exception."
^String [filename & [message]]
(when-not (file-exists? filename)
(throw (ex-info (format "File %s does not exist. %s" (pr-str filename) (or message "")) {:filename filename})))
(str filename))
(defn create-directory-unless-exists! [^String dir]
(when-not (file-exists? dir)
(steps/step (format "Creating directory %s..." dir)
(.mkdirs (File. dir))))
dir)
(defn delete-file!
"Delete a file or directory (recursively) if it exists."
([^String filename]
(steps/step (format "Deleting %s..." filename)
(if (file-exists? filename)
(let [file (File. filename)]
(if (.isDirectory file)
(FileUtils/deleteDirectory file)
(.delete file))
(out/safe-println (format "Deleted %s." filename)))
(out/safe-println (format "Don't need to delete %s, file does not exist." filename)))
(assert (not (file-exists? filename)))))
([file & more]
(dorun (map delete-file! (cons file more)))))
(defn copy-file!
"Copy a `source` file (or directory, recursively) to `dest`."
[^String source ^String dest]
(let [source-file (File. (assert-file-exists source))
dest-file (File. dest)]
;; Use native `cp` rather than FileUtils or the like because codesigning is broken when you use those because they
;; don't preserve symlinks or something like that.
(if (.isDirectory source-file)
(steps/step (format "Copying directory %s -> %s" source dest)
(sh/sh "cp" "-R" source dest))
(steps/step (format "Copying file %s -> %s" source dest)
(sh/sh "cp" source dest))))
(assert-file-exists dest))
(defn- ->URI ^java.net.URI [filename]
(java.net.URI. (str "file://" filename)))
(defn- ->Path ^Path [filename]
(Paths/get (->URI filename)))
(defn find-files
"Pure Java version of `find`. Recursively find files in `dir-path` that satisfy `pred`, which has the signature
(pred filename-string) -> Boolean"
[^String dir-path pred]
(->> (Files/find (->Path dir-path)
Integer/MAX_VALUE
(reify BiPredicate
(test [_ path _]
(boolean (pred (str path)))))
^FileVisitOption (make-array FileVisitOption 0))
.toArray
(map str)
sort))
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment