From f1319da6a6a3eca86102711fd3894f598915e749 Mon Sep 17 00:00:00 2001
From: Luis Paolini <paoliniluis@gmail.com>
Date: Thu, 25 Jul 2024 07:56:58 -0300
Subject: [PATCH] Make drivers build in parallel rather than sequentially
(#45897)
* Make drivers build in parallel rather than sequentially
* Make the whitespace linter happy
* Add duration to steps
---
bin/build/src/build.clj | 6 +++--
bin/build/src/build_drivers.clj | 8 +++----
.../src/build_drivers/copy_source_files.clj | 22 ++++++++++++++-----
3 files changed, 25 insertions(+), 11 deletions(-)
diff --git a/bin/build/src/build.clj b/bin/build/src/build.clj
index 744616876d2..6fb4e2bcb95 100644
--- a/bin/build/src/build.clj
+++ b/bin/build/src/build.clj
@@ -106,7 +106,8 @@
:or {edition (edition-from-env-var)
steps (keys all-steps)}}]
(let [version (or version
- (version-properties/current-snapshot-version edition))]
+ (version-properties/current-snapshot-version edition))
+ start-time-ms (System/currentTimeMillis)]
(u/step (format "Running build steps for %s version %s: %s"
(case edition
:oss "Community (OSS) Edition"
@@ -118,7 +119,8 @@
(throw (ex-info (format "Invalid step: %s" step-name)
{:step step-name
:valid-steps (keys all-steps)})))]]
- (step-fn {:version version, :edition edition}))
+ (step-fn {:version version, :edition edition})
+ (u/announce "Did %s in %d ms." step-name (- (System/currentTimeMillis) start-time-ms)))
(u/announce "All build steps finished.")))))
(defn build-cli
diff --git a/bin/build/src/build_drivers.clj b/bin/build/src/build_drivers.clj
index 74389c01d03..751a649913d 100644
--- a/bin/build/src/build_drivers.clj
+++ b/bin/build/src/build_drivers.clj
@@ -22,13 +22,13 @@
(map (comp keyword #(.getName ^File %)))))
(defn build-drivers!
- "Build `edition`(`:ee` or `:oss`) versions of *all* the drivers in `modules/drivers`."
+ "Build `edition`(`:ee` or `:oss`) versions of *all* the drivers in `modules/drivers` in parallel."
[edition]
(let [edition (or edition :oss)]
(assert (#{:oss :ee} edition))
- (u/step (format "Building all drivers (%s edition)" (pr-str edition))
- (doseq [driver (all-drivers)]
- (build-driver/build-driver! driver edition))
+ (u/step (format "Building all drivers in parallel (%s edition)" (pr-str edition))
+ (doall ; Force evaluation of pmap
+ (pmap #(build-driver/build-driver! % edition) (all-drivers)))
(u/announce "Successfully built all drivers."))))
(defn build-drivers
diff --git a/bin/build/src/build_drivers/copy_source_files.clj b/bin/build/src/build_drivers/copy_source_files.clj
index 3e22cc74b43..12da97f02c1 100644
--- a/bin/build/src/build_drivers/copy_source_files.clj
+++ b/bin/build/src/build_drivers/copy_source_files.clj
@@ -1,11 +1,25 @@
(ns build-drivers.copy-source-files
(:require
[build-drivers.common :as c]
- [clojure.tools.build.api :as b]
- [metabuild-common.core :as u]))
+ [clojure.java.io :as io]
+ [metabuild-common.core :as u])
+ (:import (java.nio.file Files)))
(set! *warn-on-reflection* true)
+(defn- copy-files [src-dirs target-dir]
+ (doseq [src-dir src-dirs]
+ (let [src (io/file src-dir)
+ target (io/file target-dir)]
+ (when (.exists src)
+ (u/announce "Copying files from %s to %s" src target)
+ (doseq [file (file-seq src)]
+ (when (.isFile file)
+ (let [relative-path (.relativize (.toPath src) (.toPath file))
+ target-file (io/file target (str relative-path))]
+ (.mkdirs (.getParentFile target-file))
+ (Files/copy (.toPath file) (.toPath target-file) (into-array java.nio.file.CopyOption [])))))))))
+
(defn copy-source-files!
"Copy source files into the build driver JAR."
[driver edition]
@@ -15,9 +29,7 @@
(assert (every? u/absolute? dirs)
(format "All dirs should be absolute, got: %s" (pr-str dirs)))
(u/announce "Copying files in %s" (pr-str dirs))
- (b/copy-dir
- {:src-dirs dirs
- :target-dir (c/compiled-source-target-dir driver)})
+ (copy-files dirs (c/compiled-source-target-dir driver))
(u/announce "Copied files in %d directories in %d ms."
(count dirs)
(- (System/currentTimeMillis) start-time-ms)))))
--
GitLab