Support building and hacking on third-party drivers with our new Clojure CLI setup (#17606)

* Determine MB project root directory relative to source directory of build scripts

* Support calling build-driver! as a -X fn, and support dirs outside of modules/drivers

* Support loading 3rd-party driver manifest files at (dev) launch

* Fix log4j not working on latest version of CIDER (unrelated)

* Fix dissoc

* project-root-directory is supposed to be a string

bin/build-drivers/src/build_drivers/build_driver.clj

@@ -12,13 +12,24 @@
     (u/delete-file-if-exists! (c/driver-jar-destination-path driver))))
 
 (defn build-driver!
-  [driver edition]
-  (let [edition (or edition :oss)
-        start-time-ms (System/currentTimeMillis)]
-    (u/step (format "Build driver %s (edition = %s)" driver edition)
-      (clean! driver)
-      (copy-source-files/copy-source-files! driver edition)
-      (compile-source-files/compile-clojure-source-files! driver edition)
-      (create-uberjar/create-uberjar! driver edition)
-      (u/announce "Built %s driver in %d ms." driver (- (System/currentTimeMillis) start-time-ms))
-      (verify/verify-driver driver))))
+  ;; 1-arity that takes just a map is mean for use directly with clojure -X
+  ([{:keys [driver edition], :as options}]
+   (build-driver! driver edition (dissoc options :driver :edition)))
+
+  ([driver edition]
+   (build-driver! driver edition nil))
+
+  ([driver edition {:keys [project-dir target-dir], :as options}]
+   (let [edition       (or edition :oss)
+         start-time-ms (System/currentTimeMillis)]
+     (binding [c/*driver-project-dir* (or project-dir
+                                          c/*driver-project-dir*)
+               c/*target-directory*   (or target-dir
+                                          c/*target-directory*)]
+       (u/step (format "Build driver %s (edition = %s, options = %s)" driver edition (pr-str options))
+         (clean! driver)
+         (copy-source-files/copy-source-files! driver edition)
+         (compile-source-files/compile-clojure-source-files! driver edition)
+         (create-uberjar/create-uberjar! driver edition)
+         (u/announce "Built %s driver in %d ms." driver (- (System/currentTimeMillis) start-time-ms))
+         (verify/verify-driver driver))))))

bin/build-drivers/src/build_drivers/common.clj

@@ -2,23 +2,29 @@
   (:require [clojure.tools.deps.alpha :as deps]
             [metabuild-common.core :as u]))
 
+(def ^:dynamic *driver-project-dir* nil)
+
+(def ^:dynamic *target-directory* nil)
+
 (defn driver-project-dir
   "e.g. \"/home/cam/metabase/modules/drivers/redshift\""
   ^String [driver]
-  (u/filename u/project-root-directory "modules" "drivers" (name driver)))
+  (or *driver-project-dir*
+      (u/filename u/project-root-directory "modules" "drivers" (name driver))))
 
 (defn driver-jar-name
   "e.g. \"redshift.metabase-driver.jar\""
   ^String [driver]
   (format "%s.metabase-driver.jar" (name driver)))
 
-(def ^String driver-jar-destination-directory
-  (u/filename u/project-root-directory "resources" "modules"))
+(defn driver-jar-destination-directory ^String []
+  (or *target-directory*
+      (u/filename u/project-root-directory "resources" "modules")))
 
 (defn driver-jar-destination-path
   "e.g. \"/home/cam/metabase/resources/modules/redshift.metabase-driver.jar\""
   ^String [driver]
-  (u/filename driver-jar-destination-directory (driver-jar-name driver)))
+  (u/filename (driver-jar-destination-directory) (driver-jar-name driver)))
 
 (defn compiled-source-target-dir [driver]
   (u/filename (driver-project-dir driver) "target" "jar"))

bin/build-drivers/src/build_drivers/create_uberjar.clj

 (ns build-drivers.create-uberjar
   (:require [build-drivers.common :as c]
             [clojure.java.io :as io]
-            [clojure.tools.build.api :as build]
             [clojure.tools.deps.alpha :as deps]
             [clojure.tools.deps.alpha.util.dir :as deps.dir]
             [colorize.core :as colorize]

bin/common/src/metabuild_common/files.clj

 (ns metabuild-common.files
-  (:require [clojure.string :as str]
-            [environ.core :as env]
+  (:require [clojure.java.io :as io]
+            [clojure.string :as str]
             [metabuild-common.misc :as misc]
             [metabuild-common.output :as out]
             [metabuild-common.shell :as sh]
@@ -97,20 +97,16 @@
   (str/join File/separatorChar path-components))
 
 (def ^String project-root-directory
-  "Root directory of the Metabase repo, e.g. `/users/cam/metabase`. Determined by finding the directory that has
-  `.git` in it."
-  (loop [^File dir (File. ^String (env/env :user-dir))]
-    (cond
-      (file-exists? (filename (.getAbsolutePath dir) "package.json"))
-      (.getAbsolutePath dir)
-
-      (.getParentFile dir)
-      (recur (.getParentFile dir))
-
-      :else
-      (throw (ex-info (format "Can't find project root directory: no parent directory of %s has a .git directory"
-                              (env/env :user-dir))
-                      {:dir (env/env :user-dir)})))))
+  "Root directory of the Metabase repo, e.g. `/users/cam/metabase`. Determined based on its location relative to this
+  source file."
+  (.. (Paths/get (.toURI (io/resource "metabuild_common/files.clj")))
+      toFile
+      getParentFile   ; /home/cam/metabase/bin/common/src/metabuild_common
+      getParentFile   ; /home/cam/metabase/bin/common/src/
+      getParentFile   ; /home/cam/metabase/bin/common/
+      getParentFile   ; /home/cam/metabase/bin/
+      getParentFile   ; /home/cam/metabase/
+      getCanonicalPath))
 
 (defn download-file!
   "Download a file from `url` to `dest-path` using `wget`."

resources/log4j2.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <Configuration>
   <Appenders>
-    <Console name="STDOUT" target="SYSTEM_OUT">
+    <Console name="STDOUT" target="SYSTEM_OUT" follow="true">
       <PatternLayout pattern="%date %level %logger{2} :: %message%n%throwable">
         <replace regex=":basic-auth \\[.*\\]" replacement=":basic-auth [redacted]"/>
       </PatternLayout>

src/metabase/plugins.clj

@@ -107,15 +107,26 @@
   (defn- load-local-plugin-manifest! [^Path path]
     (some-> (slurp (str path)) yaml.core/parse-string initialize/init-plugin-with-info!))
 
+  (defn- driver-manifest-paths []
+    (filter
+     files/exists?
+     (concat
+      (for [path (files/files-seq (files/get-path "modules/drivers/"))]
+        (files/get-path (str path) "/resources/metabase-plugin.yaml"))
+      ;; for hacking on 3rd-party drivers locally: set
+      ;; `-Dmb.dev.additional.driver.manifest.paths=/path/to/whatever/metabase-plugin.yaml` or
+      ;; `MB_DEV_ADDITIONAL_DRIVER_MANIFEST_PATHS=...` to have that plugin manifest get loaded during startup. Specify
+      ;; multiple plugin manifests by comma-separating them.
+      (when-let [additional-paths (env/env :mb-dev-additional-driver-manifest-paths)]
+        (map files/get-path (str/split additional-paths #","))))))
+
   (defn- load-local-plugin-manifests!
     "Load local plugin manifest files when running in dev or test mode, to simulate what would happen when loading those
   same plugins from the uberjar. This is needed because some plugin manifests define driver methods and the like that
   aren't defined elsewhere."
     []
     ;; TODO - this should probably do an actual search in case we ever add any additional directories
-    (doseq [path  (files/files-seq (files/get-path "modules/drivers/"))
-            :let  [manifest-path (files/get-path (str path) "/resources/metabase-plugin.yaml")]
-            :when (files/exists? manifest-path)]
+    (doseq [manifest-path (driver-manifest-paths)]
       (log/info (trs "Loading local plugin manifest at {0}" (str manifest-path)))
       (load-local-plugin-manifest! manifest-path))))