Easy checking for missing library licenses (#16989)

* Easy checking for missing library licenses

* include bin/build-mb readme

bin/build-mb/README.md

+## Build Metabase Tooling
+
+This project is to build the Metabase jar. It can be called standalone and is also called from the release project when creating releases.
+
+## License Information
+
+We create license information for all of our dependencies, both frontend and backend, and package them in our jar.
+
+Tests will run in CI that we have license information for all dependencies. If you see these failing, an easy way to get a report of dependencies without license information can be obtained by running
+
+```shell
+build-mb % clojure -X build/list-without-license
+$ "lein" "with-profile" "-dev,+ee,+include-all-drivers" "classpath"
+All dependencies have licenses
+```
+
+If there are dependencies with missing license information you will see output like
+
+```shell
+build-mb % clojure -X build/list-lacking-license
+$ "lein" "with-profile" "-dev,+ee,+include-all-drivers" "classpath"
+Missing License: /Users/dan/.m2/repository/org/eclipse/jetty/jetty-webapp/9.3.19.v20170502/jetty-webapp-9.3.19.v20170502.jar
+Missing License: /Users/dan/.m2/repository/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar
+Missing License: /Users/dan/.m2/repository/org/opensaml/opensaml-security-impl/3.4.5/opensaml-security-impl-3.4.5.jar
+Missing License: /Users/dan/.m2/repository/colorize/colorize/0.1.1/colorize-0.1.1.jar
+```
+
+You can check the overrides file (resources/overrides.edn) and add the license information there, or perhaps improve the license discovery mechanism in the code.

bin/build-mb/src/build.clj

@@ -133,3 +133,23 @@
     (build! (merge {:edition (edition-from-env-var)}
                    (when-let [steps (not-empty steps)]
                      {:steps steps})))))
+
+(defn list-without-license [{:keys []}]
+  (let [classpath-and-logs        (u/sh {:dir    u/project-root-directory
+                                         :quiet? true}
+                                        "lein"
+                                        "with-profile"
+                                        "-dev,+ee,+include-all-drivers"
+                                        "classpath")
+        classpath                 (last classpath-and-logs)
+        classpath-entries (license/jar-entries classpath)
+        {:keys [without-license]} (license/process*
+                                   {:classpath-entries classpath-entries
+                                    :backfill        (edn/read-string
+                                                        (slurp (io/resource "overrides.edn")))})]
+    (if (seq without-license)
+      (run! (comp (partial u/error "Missing License: %s") first)
+            without-license)
+      (u/announce "All dependencies have licenses"))
+    (shutdown-agents)
+    (System/exit (if (seq without-license) 1 0))))

bin/build-mb/src/build/licenses.clj

@@ -185,6 +185,12 @@
     {:with-license    (categorized true)
      :without-license (categorized false)}))
 
+(defn jar-entries
+  "Returns a seq of jar entries on the classpath"
+  [classpath]
+  (->> (str/split classpath (re-pattern classpath-separator))
+       (filter jar-file?)))
+
 (defn generate
   "Process a classpath, creating a file of all license information, writing to `:output-filename`. Backfill is a clojure
   data structure or a filename of an edn file of a clojure datastructure providing for backfilling license information
@@ -212,8 +218,7 @@
   (let [backfill (if (string? backfill)
                    (edn/read-string (slurp backfill))
                    (or backfill {}))
-        entries  (->> (str/split classpath (re-pattern classpath-separator))
-                      (filter jar-file?))]
+        entries  (jar-entries classpath)]
     (let [{:keys [with-license without-license] :as license-info}
           (process* {:classpath-entries     entries
                      :backfill              backfill})]

bin/build-mb/test/build/licenses_test.clj

@@ -221,7 +221,9 @@
         (let [results (lic/process* {:classpath-entries classpath-entries
                                      :backfill  (edn/read-string
                                                  (slurp (io/resource "overrides.edn")))})]
-          (is (nil? (:without-license results)) "Some deps don't have identifiable licenses")
+          (is (nil? (:without-license results))
+              (str "Deps without license information:\n"
+                   (str/join "\n" (map first (:without-license results)))))
           (is (= (set classpath-entries)
                  (into #{} (->> results :with-license (map first))))))
         (is (some? (:without-license