Integrate driver JAR strip-and-compress logic into build-driver script (#14535)

* Integrate driver JAR strip-and-compress logic into build-driver script

* Fix missing import

* Minor improvements

bin/build-drivers/src/build_drivers/build_driver.clj

 (ns build-drivers.build-driver
   "Logic for building a single driver."
-  (:require [build-drivers
-             [checksum :as checksum]
-             [common :as c]
-             [install-driver-locally :as install-locally]
-             [metabase :as metabase]
-             [plugin-manifest :as manifest]
-             [verify :as verify]]
+  (:require [build-drivers.checksum :as checksum]
+            [build-drivers.common :as c]
+            [build-drivers.install-driver-locally :as install-locally]
+            [build-drivers.metabase :as metabase]
+            [build-drivers.plugin-manifest :as manifest]
+            [build-drivers.strip-and-compress :as strip-and-compress]
+            [build-drivers.verify :as verify]
             [colorize.core :as colorize]
             [environ.core :as env]
             [metabuild-common.core :as u]))
@@ -18,7 +18,7 @@
                   driver
                   (u/assert-file-exists (c/driver-jar-build-path driver))
                   (c/driver-jar-destination-path driver))
-    (u/delete-file! (c/driver-jar-destination-path driver))
+    (u/delete-file-if-exists! (c/driver-jar-destination-path driver))
     (u/create-directory-unless-exists! c/driver-jar-destination-directory)
     (u/copy-file! (c/driver-jar-build-path driver)
                   (c/driver-jar-destination-path driver))))
@@ -27,8 +27,8 @@
   "Delete built JARs of `driver`."
   [driver]
   (u/step (format "Delete %s driver artifacts" driver)
-    (u/delete-file! (c/driver-target-directory driver))
-    (u/delete-file! (c/driver-jar-destination-path driver))))
+    (u/delete-file-if-exists! (c/driver-target-directory driver))
+    (u/delete-file-if-exists! (c/driver-jar-destination-path driver))))
 
 (defn- clean-parents!
   "Delete built JARs and local Maven installations of the parent drivers of `driver`."
@@ -62,34 +62,11 @@
       (build-driver! parent))
     (u/announce "%s parents built successfully." driver)))
 
-(defn- strip-and-compress-uberjar!
-  "Remove any classes in compiled `driver` that are also present in the Metabase uberjar or parent drivers. The classes
-  will be available at runtime, and we don't want to make things unpredictable by including them more than once in
-  different drivers.
 
-  This is only needed because `lein uberjar` does not seem to reliably exclude classes from `:provided` Clojure
-  dependencies like `metabase-core` and the parent drivers."
-  ([driver]
-   (u/step (str (format "Strip out any classes in %s driver JAR found in core Metabase uberjar or parent JARs" driver)
-                " and recompress with higher compression ratio")
-     (let [uberjar (u/assert-file-exists (c/driver-jar-build-path driver))]
-       (u/step "strip out any classes also found in the core Metabase uberjar"
-         (strip-and-compress-uberjar! uberjar (u/assert-file-exists c/metabase-uberjar-path)))
-       (u/step "remove any classes also found in any of the parent JARs"
-         (doseq [parent (manifest/parent-drivers driver)]
-           (strip-and-compress-uberjar! uberjar (u/assert-file-exists (c/driver-jar-build-path parent))))))))
-
-  ([target source]
-   (u/step (format "Remove classes from %s that are present in %s and recompress" target source)
-     (u/sh {:dir u/project-root-directory}
-           "lein"
-           "strip-and-compress"
-           (u/assert-file-exists target)
-           (u/assert-file-exists source)))))
 
 (defn- build-uberjar! [driver]
   (u/step (format "Build %s uberjar" driver)
-    (u/delete-file! (c/driver-target-directory driver))
+    (u/delete-file-if-exists! (c/driver-target-directory driver))
     (u/sh {:dir (c/driver-project-dir driver)} "lein" "clean")
     (u/sh {:dir (c/driver-project-dir driver)
            :env {"LEIN_SNAPSHOTS_IN_RELEASE" "true"
@@ -97,7 +74,7 @@
                  "PATH"                      (env/env :path)
                  "JAVA_HOME"                 (env/env :java-home)}}
           "lein" "uberjar")
-    (strip-and-compress-uberjar! driver)
+    (strip-and-compress/strip-and-compress-uberjar! driver)
     (u/announce "%s uberjar build successfully." driver)))
 
 (defn- build-and-verify!

bin/build-drivers/src/build_drivers/strip_and_compress.clj

+(ns build-drivers.strip-and-compress
+  (:require [build-drivers.common :as c]
+            [build-drivers.plugin-manifest :as manifest]
+            [metabuild-common.core :as u])
+  (:import java.io.FileOutputStream
+           [java.util.zip ZipEntry ZipFile ZipOutputStream]
+           org.apache.commons.io.IOUtils))
+
+(def ^:private files-to-always-include
+  "Files to always include regardless of whether they are present in blacklist JAR."
+  #{"metabase-plugin.yaml"})
+
+(defn- jar-contents
+  "Get a set of all files in a JAR that we should strip out from the driver JAR -- either the Metabase uberjar itself or
+  a parent driver JAR."
+  [^String jar-path]
+  (with-open [zip-file (ZipFile. jar-path)]
+    (set
+     (for [^ZipEntry zip-entry (enumeration-seq (.entries zip-file))
+           :let                [filename (str zip-entry)]
+           :when               (not (files-to-always-include filename))]
+       filename))))
+
+(defn- strip-classes! [^String driver-jar-path ^String blacklist-jar-path]
+  (u/step (format "Remove classes from %s that are present in %s and recompress" driver-jar-path blacklist-jar-path)
+    (let [jar-contents (jar-contents blacklist-jar-path)
+          temp-driver-jar-path  "/tmp/driver.jar"
+          wrote           (atom 0)
+          skipped         (atom 0)]
+      (u/delete-file-if-exists! temp-driver-jar-path)
+      (with-open [source-zip (ZipFile. (u/assert-file-exists driver-jar-path))
+                  os         (doto (ZipOutputStream. (FileOutputStream. temp-driver-jar-path))
+                               (.setMethod ZipOutputStream/DEFLATED)
+                               (.setLevel 9))]
+        (doseq [^ZipEntry entry (enumeration-seq (.entries source-zip))]
+          (if (jar-contents (str entry))
+            (swap! skipped inc)
+            (with-open [is (.getInputStream source-zip entry)]
+              (.putNextEntry os (ZipEntry. (.getName entry)))
+              (IOUtils/copy is os)
+              (.closeEntry os)
+              (swap! wrote inc)))))
+      (u/announce (format "Done. wrote: %d skipped: %d" @wrote @skipped))
+      (u/safe-println (format "Original size: %s" (u/format-bytes (u/file-size driver-jar-path))))
+      (u/safe-println (format "Stripped/extra-compressed size: %s" (u/format-bytes (u/file-size temp-driver-jar-path))))
+      (u/step "replace the original source JAR with the stripped one"
+        (u/delete-file-if-exists! driver-jar-path)
+        (u/copy-file! temp-driver-jar-path driver-jar-path)))))
+
+(defn strip-and-compress-uberjar!
+  "Remove any classes in compiled `driver` that are also present in the Metabase uberjar or parent drivers. The classes
+  will be available at runtime, and we don't want to make things unpredictable by including them more than once in
+  different drivers.
+
+  This is only needed because `lein uberjar` does not seem to reliably exclude classes from `:provided` Clojure
+  dependencies like `metabase-core` and the parent drivers."
+  [driver]
+  (u/step (str (format "Strip out any classes in %s driver JAR found in core Metabase uberjar or parent JARs" driver)
+               " and recompress with higher compression ratio")
+    (let [driver-jar-path (u/assert-file-exists (c/driver-jar-build-path driver))]
+      (u/step "strip out any classes also found in the core Metabase uberjar"
+        (strip-classes! driver-jar-path (u/assert-file-exists c/metabase-uberjar-path)))
+      (u/step "remove any classes also found in any of the parent JARs"
+        (doseq [parent (manifest/parent-drivers driver)]
+          (strip-classes! driver-jar-path (u/assert-file-exists (c/driver-jar-build-path parent))))))))

bin/common/src/metabuild_common/core.clj

 (ns metabuild-common.core
-  (:require [metabuild-common
-             [aws :as aws]
-             [entrypoint :as entrypoint]
-             [env :as build.env]
-             [files :as files]
-             [input :as input]
-             [output :as output]
-             [shell :as shell]
-             [steps :as steps]]
+  (:require [metabuild-common.aws :as aws]
+            [metabuild-common.entrypoint :as entrypoint]
+            [metabuild-common.env :as build.env]
+            [metabuild-common.files :as files]
+            [metabuild-common.input :as input]
+            [metabuild-common.misc :as misc]
+            [metabuild-common.output :as output]
+            [metabuild-common.shell :as shell]
+            [metabuild-common.steps :as steps]
             [potemkin :as p]))
 
 ;; since this file is used pretty much everywhere, this seemed like a good place to put this.
@@ -18,6 +18,7 @@
          build.env/env
          files/keep-me
          input/keep-me
+         misc/keep-me
          output/keep-me
          shell/keep-me
          steps/keep-me)
@@ -41,8 +42,10 @@
   delete-file-if-exists!
   download-file!
   file-exists?
+  file-size
   filename
   find-files
+  nio-path
   project-root-directory]
 
  [input
@@ -51,9 +54,13 @@
   read-line-with-prompt
   yes-or-no-prompt]
 
+ [misc
+  varargs]
+
  [output
   announce
   error
+  format-bytes
   pretty-print-exception
   safe-println]
 

bin/common/src/metabuild_common/files.clj

 (ns metabuild-common.files
   (:require [clojure.string :as str]
             [environ.core :as env]
-            [metabuild-common
-             [output :as out]
-             [shell :as sh]
-             [steps :as steps]])
+            [metabuild-common.misc :as misc]
+            [metabuild-common.output :as out]
+            [metabuild-common.shell :as sh]
+            [metabuild-common.steps :as steps])
   (:import java.io.File
-           [java.nio.file Files FileVisitOption Path Paths]
+           [java.nio.file Files FileSystems FileVisitOption Path Paths]
            java.util.function.BiPredicate
            org.apache.commons.io.FileUtils))
 
@@ -120,3 +120,13 @@
     (delete-file-if-exists! dest-path)
     (sh/sh {:quiet? true} "wget" "--quiet" "--no-cache" "--output-document" dest-path url)
     (assert-file-exists dest-path)))
+
+(defn nio-path
+  "Convert a String `path` to a `java.nio.file.Path`, for use with NIO methods."
+  ^Path [^String path]
+  (.getPath (FileSystems/getDefault) path (misc/varargs String)))
+
+(defn file-size
+  "Get the size, in bytes, of the file at `path`."
+  ^Long [^String path]
+  (Files/size (nio-path path)))

bin/common/src/metabuild_common/misc.clj

+(ns metabuild-common.misc)
+
+(defmacro varargs
+  "Utility macro for passing varargs of a certain `klass` to a Java method.
+
+    (Files/createTempFile \"driver\" \".jar\" (varargs FileAttribute))"
+  {:style/indent 1, :arglists '([klass] [klass xs])}
+  [klass & [objects]]
+  (vary-meta `(into-array ~klass ~objects)
+             assoc :tag (format "[L%s;" (.getCanonicalName ^Class (ns-resolve *ns* klass)))))

bin/common/src/metabuild_common/output.clj

@@ -40,3 +40,12 @@
     (println (colorize/red (str "Step failed: " (.getMessage e))))
     (binding [pprint/*print-right-margin* 120]
       (pprint/pprint e-map))))
+
+(defn format-bytes
+  "Nicely format `num-bytes` in a human-readable way (e.g. KB/MB/etc.)"
+  [num-bytes]
+  (loop [n num-bytes [suffix & more] ["B" "KB" "MB" "GB"]]
+    (if (and (seq more)
+             (>= n 1024))
+      (recur (/ n 1024.0) more)
+      (format "%.1f %s" n suffix))))

lein-commands/strip-and-compress/metabase/strip_and_compress_module.clj

-(ns metabase.strip-and-compress-module
-  (:import [java.nio.file CopyOption Files FileSystems OpenOption]
-           java.nio.file.attribute.FileAttribute
-           [java.util.zip ZipEntry ZipFile ZipOutputStream]
-           org.apache.commons.io.IOUtils))
-
-(defmacro ^:private varargs
-  {:style/indent 1, :arglists '([klass] [klass xs])}
-  [klass & [objects]]
-  (vary-meta `(into-array ~klass ~objects)
-             assoc :tag (format "[L%s;" (.getCanonicalName ^Class (ns-resolve *ns* klass)))))
-
-(defn- format-bytes [num-bytes]
-  (loop [n num-bytes [suffix & more] ["B" "KB" "MB" "GB"]]
-    (if (and (seq more)
-             (>= n 1024))
-      (recur (/ n 1024.0) more)
-      (format "%.1f %s" n suffix))))
-
-(def ^:private files-to-always-include
-  "Files to always include regardless of whether they are present in blacklist JAR."
-  #{"metabase-plugin.yaml"})
-
-(defn- files-blacklist
-  "Get a set of all files in the Metabase uberjar."
-  [^String blacklist-jar]
-  (with-open [zip-file (ZipFile. blacklist-jar)]
-    (set
-     (for [^ZipEntry zip-entry (enumeration-seq (.entries zip-file))
-           :let                [filename (str zip-entry)]
-           :when               (not (files-to-always-include filename))]
-       filename))))
-
-(defn -main
-  "Remove any classes from a module JAR that are also found in the Metabase uberjar. Compress the module JAR using the
-  maximum compression level, shrinking the size to an amazingly small level."
-  ([source]
-   (-main source "target/uberjar/metabase.jar"))
-
-  ([source blacklist-jar]
-   (println (format "Stripping duplicate classes and extra-compressing %s..." source))
-   (let [files-blacklist (files-blacklist blacklist-jar)
-         temp-file-path  (Files/createTempFile "driver" ".jar" (varargs FileAttribute))
-         wrote           (atom 0)
-         skipped         (atom 0)]
-     (with-open [source-zip (ZipFile. source)]
-       (with-open [os (doto (ZipOutputStream. (Files/newOutputStream temp-file-path (varargs OpenOption)))
-                        (.setMethod ZipOutputStream/DEFLATED)
-                        (.setLevel 9))]
-         (doseq [^ZipEntry entry (enumeration-seq (.entries source-zip))]
-           (if (files-blacklist (str entry))
-             (swap! skipped inc)
-             (with-open [is (.getInputStream source-zip entry)]
-               (.putNextEntry os (ZipEntry. (.getName entry)))
-               (IOUtils/copy is os)
-               (.closeEntry os)
-               (swap! wrote inc))))))
-     (println (format "Done. wrote: %d skipped: %d" @wrote @skipped))
-     (let [source-path (.getPath (FileSystems/getDefault) source (varargs String))]
-       (println "Original size:" (format-bytes (Files/size source-path)))
-       (println "Stripped/extra-compressed size:" (format-bytes (Files/size temp-file-path)))
-       (println (format "Copying to %s..." source-path))
-       ;; Now replace the original source JAR with the stripped one
-       (Files/delete source-path)
-       (Files/move temp-file-path source-path (varargs CopyOption))))
-   (System/exit 0)))

project.clj

@@ -36,7 +36,6 @@
    "lint"                              ["do" ["eastwood"] ["bikeshed"] ["check-namespace-decls"] ["docstring-checker"] ["cloverage"]]
    "repl"                              ["with-profile" "+repl" "repl"]
    "repl-ee"                           ["with-profile" "+repl,+ee" "repl"]
-   "strip-and-compress"                ["with-profile" "+strip-and-compress,-user,-dev" "run"]
    "compare-h2-dbs"                    ["with-profile" "+compare-h2-dbs" "run"]
    "uberjar"                           ["uberjar"]
    "uberjar-ee"                        ["with-profile" "+ee" "uberjar"]}
@@ -413,15 +412,6 @@
    {:auto-clean true
     :aot        :all}
 
-   ;; lein strip-and-compress my-plugin.jar [path/to/metabase.jar]
-   ;; strips classes from my-plugin.jar that already exist in other JAR and recompresses with higher compression ratio.
-   ;; Second arg (other JAR) is optional; defaults to target/uberjar/metabase.jar
-   :strip-and-compress
-   {:aliases      ^:replace {"run" ["run"]}
-    :source-paths ^:replace ["lein-commands/strip-and-compress"]
-    :test-paths   ^:replace []
-    :main         ^:skip-aot metabase.strip-and-compress-module}
-
    ;; Profile Metabase start time with `lein profile`
    :profile
    {:jvm-opts ["-XX:+CITime"                                          ; print time spent in JIT compiler