From da3a3f5824452b5692957726fe238339b6efe4f9 Mon Sep 17 00:00:00 2001
From: Cam Saul <1455846+camsaul@users.noreply.github.com>
Date: Thu, 26 Aug 2021 10:11:57 -0700
Subject: [PATCH] Support building and hacking on third-party drivers with our
new Clojure CLI setup (#17606)
* Determine MB project root directory relative to source directory of build scripts
* Support calling build-driver! as a -X fn, and support dirs outside of modules/drivers
* Support loading 3rd-party driver manifest files at (dev) launch
* Fix log4j not working on latest version of CIDER (unrelated)
* Fix dissoc
* project-root-directory is supposed to be a string
---
.../src/build_drivers/build_driver.clj | 31 +++++++++++++------
.../src/build_drivers/common.clj | 14 ++++++---
.../src/build_drivers/create_uberjar.clj | 1 -
bin/common/src/metabuild_common/files.clj | 28 +++++++----------
resources/log4j2.xml | 2 +-
src/metabase/plugins.clj | 17 ++++++++--
6 files changed, 58 insertions(+), 35 deletions(-)
diff --git a/bin/build-drivers/src/build_drivers/build_driver.clj b/bin/build-drivers/src/build_drivers/build_driver.clj
index 5bbb617668c..f05b81b5706 100644
--- a/bin/build-drivers/src/build_drivers/build_driver.clj
+++ b/bin/build-drivers/src/build_drivers/build_driver.clj
@@ -12,13 +12,24 @@
(u/delete-file-if-exists! (c/driver-jar-destination-path driver))))
(defn build-driver!
- [driver edition]
- (let [edition (or edition :oss)
- start-time-ms (System/currentTimeMillis)]
- (u/step (format "Build driver %s (edition = %s)" driver edition)
- (clean! driver)
- (copy-source-files/copy-source-files! driver edition)
- (compile-source-files/compile-clojure-source-files! driver edition)
- (create-uberjar/create-uberjar! driver edition)
- (u/announce "Built %s driver in %d ms." driver (- (System/currentTimeMillis) start-time-ms))
- (verify/verify-driver driver))))
+ ;; 1-arity that takes just a map is mean for use directly with clojure -X
+ ([{:keys [driver edition], :as options}]
+ (build-driver! driver edition (dissoc options :driver :edition)))
+
+ ([driver edition]
+ (build-driver! driver edition nil))
+
+ ([driver edition {:keys [project-dir target-dir], :as options}]
+ (let [edition (or edition :oss)
+ start-time-ms (System/currentTimeMillis)]
+ (binding [c/*driver-project-dir* (or project-dir
+ c/*driver-project-dir*)
+ c/*target-directory* (or target-dir
+ c/*target-directory*)]
+ (u/step (format "Build driver %s (edition = %s, options = %s)" driver edition (pr-str options))
+ (clean! driver)
+ (copy-source-files/copy-source-files! driver edition)
+ (compile-source-files/compile-clojure-source-files! driver edition)
+ (create-uberjar/create-uberjar! driver edition)
+ (u/announce "Built %s driver in %d ms." driver (- (System/currentTimeMillis) start-time-ms))
+ (verify/verify-driver driver))))))
diff --git a/bin/build-drivers/src/build_drivers/common.clj b/bin/build-drivers/src/build_drivers/common.clj
index 86f099024b9..07cc5932aa3 100644
--- a/bin/build-drivers/src/build_drivers/common.clj
+++ b/bin/build-drivers/src/build_drivers/common.clj
@@ -2,23 +2,29 @@
(:require [clojure.tools.deps.alpha :as deps]
[metabuild-common.core :as u]))
+(def ^:dynamic *driver-project-dir* nil)
+
+(def ^:dynamic *target-directory* nil)
+
(defn driver-project-dir
"e.g. \"/home/cam/metabase/modules/drivers/redshift\""
^String [driver]
- (u/filename u/project-root-directory "modules" "drivers" (name driver)))
+ (or *driver-project-dir*
+ (u/filename u/project-root-directory "modules" "drivers" (name driver))))
(defn driver-jar-name
"e.g. \"redshift.metabase-driver.jar\""
^String [driver]
(format "%s.metabase-driver.jar" (name driver)))
-(def ^String driver-jar-destination-directory
- (u/filename u/project-root-directory "resources" "modules"))
+(defn driver-jar-destination-directory ^String []
+ (or *target-directory*
+ (u/filename u/project-root-directory "resources" "modules")))
(defn driver-jar-destination-path
"e.g. \"/home/cam/metabase/resources/modules/redshift.metabase-driver.jar\""
^String [driver]
- (u/filename driver-jar-destination-directory (driver-jar-name driver)))
+ (u/filename (driver-jar-destination-directory) (driver-jar-name driver)))
(defn compiled-source-target-dir [driver]
(u/filename (driver-project-dir driver) "target" "jar"))
diff --git a/bin/build-drivers/src/build_drivers/create_uberjar.clj b/bin/build-drivers/src/build_drivers/create_uberjar.clj
index c4f08ca203a..6fa58070bf8 100644
--- a/bin/build-drivers/src/build_drivers/create_uberjar.clj
+++ b/bin/build-drivers/src/build_drivers/create_uberjar.clj
@@ -1,7 +1,6 @@
(ns build-drivers.create-uberjar
(:require [build-drivers.common :as c]
[clojure.java.io :as io]
- [clojure.tools.build.api :as build]
[clojure.tools.deps.alpha :as deps]
[clojure.tools.deps.alpha.util.dir :as deps.dir]
[colorize.core :as colorize]
diff --git a/bin/common/src/metabuild_common/files.clj b/bin/common/src/metabuild_common/files.clj
index 4a1d9e34565..d44bb449b90 100644
--- a/bin/common/src/metabuild_common/files.clj
+++ b/bin/common/src/metabuild_common/files.clj
@@ -1,6 +1,6 @@
(ns metabuild-common.files
- (:require [clojure.string :as str]
- [environ.core :as env]
+ (:require [clojure.java.io :as io]
+ [clojure.string :as str]
[metabuild-common.misc :as misc]
[metabuild-common.output :as out]
[metabuild-common.shell :as sh]
@@ -97,20 +97,16 @@
(str/join File/separatorChar path-components))
(def ^String project-root-directory
- "Root directory of the Metabase repo, e.g. `/users/cam/metabase`. Determined by finding the directory that has
- `.git` in it."
- (loop [^File dir (File. ^String (env/env :user-dir))]
- (cond
- (file-exists? (filename (.getAbsolutePath dir) "package.json"))
- (.getAbsolutePath dir)
-
- (.getParentFile dir)
- (recur (.getParentFile dir))
-
- :else
- (throw (ex-info (format "Can't find project root directory: no parent directory of %s has a .git directory"
- (env/env :user-dir))
- {:dir (env/env :user-dir)})))))
+ "Root directory of the Metabase repo, e.g. `/users/cam/metabase`. Determined based on its location relative to this
+ source file."
+ (.. (Paths/get (.toURI (io/resource "metabuild_common/files.clj")))
+ toFile
+ getParentFile ; /home/cam/metabase/bin/common/src/metabuild_common
+ getParentFile ; /home/cam/metabase/bin/common/src/
+ getParentFile ; /home/cam/metabase/bin/common/
+ getParentFile ; /home/cam/metabase/bin/
+ getParentFile ; /home/cam/metabase/
+ getCanonicalPath))
(defn download-file!
"Download a file from `url` to `dest-path` using `wget`."
diff --git a/resources/log4j2.xml b/resources/log4j2.xml
index e5e4e04063d..9a97c01d8f8 100644
--- a/resources/log4j2.xml
+++ b/resources/log4j2.xml
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration>
<Appenders>
- <Console name="STDOUT" target="SYSTEM_OUT">
+ <Console name="STDOUT" target="SYSTEM_OUT" follow="true">
<PatternLayout pattern="%date %level %logger{2} :: %message%n%throwable">
<replace regex=":basic-auth \\[.*\\]" replacement=":basic-auth [redacted]"/>
</PatternLayout>
diff --git a/src/metabase/plugins.clj b/src/metabase/plugins.clj
index e6965e9092e..8263e915ec7 100644
--- a/src/metabase/plugins.clj
+++ b/src/metabase/plugins.clj
@@ -107,15 +107,26 @@
(defn- load-local-plugin-manifest! [^Path path]
(some-> (slurp (str path)) yaml.core/parse-string initialize/init-plugin-with-info!))
+ (defn- driver-manifest-paths []
+ (filter
+ files/exists?
+ (concat
+ (for [path (files/files-seq (files/get-path "modules/drivers/"))]
+ (files/get-path (str path) "/resources/metabase-plugin.yaml"))
+ ;; for hacking on 3rd-party drivers locally: set
+ ;; `-Dmb.dev.additional.driver.manifest.paths=/path/to/whatever/metabase-plugin.yaml` or
+ ;; `MB_DEV_ADDITIONAL_DRIVER_MANIFEST_PATHS=...` to have that plugin manifest get loaded during startup. Specify
+ ;; multiple plugin manifests by comma-separating them.
+ (when-let [additional-paths (env/env :mb-dev-additional-driver-manifest-paths)]
+ (map files/get-path (str/split additional-paths #","))))))
+
(defn- load-local-plugin-manifests!
"Load local plugin manifest files when running in dev or test mode, to simulate what would happen when loading those
same plugins from the uberjar. This is needed because some plugin manifests define driver methods and the like that
aren't defined elsewhere."
[]
;; TODO - this should probably do an actual search in case we ever add any additional directories
- (doseq [path (files/files-seq (files/get-path "modules/drivers/"))
- :let [manifest-path (files/get-path (str path) "/resources/metabase-plugin.yaml")]
- :when (files/exists? manifest-path)]
+ (doseq [manifest-path (driver-manifest-paths)]
(log/info (trs "Loading local plugin manifest at {0}" (str manifest-path)))
(load-local-plugin-manifest! manifest-path))))
--
GitLab