Merge pull request #9218 from metabase/driver-building-checksums

Don't rebuild drivers unless needed

.gitignore

@@ -58,3 +58,4 @@ bin/node_modules/
 *.trace.db
 /backend-checksums.txt
 .vscode
+target/checksum.txt

bin/build-driver.sh

@@ -11,93 +11,164 @@ if [ ! "$driver" ]; then
     exit -1
 fi
 
-# First, remove any existing drivers
+driver_project_dir="$project_root/modules/drivers/$driver"
 driver_jar="$driver.metabase-driver.jar"
+dest_location="$project_root/resources/modules/$driver_jar"
+metabase_uberjar="$project_root/target/uberjar/metabase.jar"
+target_jar="$driver_project_dir/target/uberjar/$driver_jar"
+parents=''
+checksum_file="$driver_project_dir/target/checksum.txt"
+
+######################################## CALCULATING CHECKSUMS ########################################
+
+# Calculate a checksum of all the driver source files. If we've already built the driver and the checksum is the same
+# there's no need to build the driver a second time
+calculate_checksum() {
+    find "$driver_project_dir" -name '*.clj' -or -name '*.yaml' | sort | cat | md5sum
+}
+
+# Check whether the saved checksum for the driver sources from the last build is the same as the current one. If so,
+# we don't need to build again.
+checksum_is_same() {
+    result=""
+    if [ -f "$checksum_file" ]; then
+        old_checksum=`cat "$checksum_file"`
+        if [ "$(calculate_checksum)" == "$old_checksum" ]; then
+            # Make sure the target driver JAR actually exists as well!
+            if [ -f "$target_jar" ]; then
+                result="$driver driver source unchanged since last build. Skipping re-build."
+            fi
+        fi
+    fi
+    echo "$result"
+}
 
-if [ -f resources/modules/"$driver.metabase-driver.jar" ]; then
-    echo "$driver is already built."
-    echo "To rebuild the driver, delete resources/modules/$driver.metabase-driver.jar and run this script again."
-    exit 0
-fi
-
-mkdir -p resources/modules
+######################################## BUILDING THE DRIVER ########################################
 
-echo "Deleting old versions of $driver driver..."
-rm -f plugins/"$driver_jar"
-
-driver_project_dir="$project_root/modules/drivers/$driver"
+# Delete existing saved copies of the driver in the plugins and resources directories
+delete_old_drivers() {
+    echo "Deleting old versions of $driver driver..."
+    rm -f plugins/"$driver_jar"
+    rm -f "$dest_location"
+}
 
 # Check if Metabase is installed locally for building drivers; install it if not
-if [ ! `find ~/.m2/repository/metabase-core/metabase-core -name '*.jar'` ]; then
-    echo "Building Metabase and installing locally..."
-    lein clean
-    lein install-for-building-drivers
-fi
+install_metabase_core() {
+    if [ ! "$(find ~/.m2/repository/metabase-core/metabase-core -name '*.jar')" ]; then
+        echo "Building Metabase and installing locally..."
+        lein clean
+        lein install-for-building-drivers
+    else
+        echo "metabase-core already installed to local Maven repo."
+    fi
+}
 
-# Build Metabase uberjar if needed, we'll need this for stripping duplicate classes
-metabase_uberjar="$project_root/target/uberjar/metabase.jar"
 
-if [ ! -f "$metabase_uberjar" ]; then
-    echo 'Building Metabase uberjar...'
-    lein uberjar
-fi
+# Build Metabase uberjar if needed, we'll need this for stripping duplicate classes
+build_metabase_uberjar() {
+    if [ ! -f "$metabase_uberjar" ]; then
+        echo 'Building Metabase uberjar...'
+        lein uberjar
+    else
+        echo "Metabase uberjar already built."
+    fi
+}
 
 # Take a look at the `parents` file listing the parents to build, if applicable
-parents_list="$driver_project_dir"/parents
-parents=''
+build_parents() {
+    echo "Building parent drivers (if needed)..."
 
-if [ -f "$parents_list" ]; then
-    parents=`cat "$parents_list"`
-    echo "Found driver parents: $parents"
-fi
+    parents_list="$driver_project_dir"/parents
 
-# Check and see if we need to recursively build or install any of our parents before proceeding
-for parent in $parents; do
-    if [ ! -f resources/modules/"$parent.metabase-driver.jar" ]; then
-        echo "Building $parent..."
-        ./bin/build-driver.sh "$parent"
+    if [ -f "$parents_list" ]; then
+        parents=`cat "$parents_list"`
+        echo "Found driver parents: $parents"
     fi
 
-    if [ ! `find "~/.m2/repository/metabase/$parent-driver/" -name '*.jar'` ]; then
-        parent_project_dir="$project_root/modules/drivers/$parent"
-        echo "Installing $parent locally..."
-        cd "$parent_project_dir"
-        lein clean
-        lein install-for-building-drivers
-        cd "$project_root"
-    fi
-done
-
-# ok, now we can build the driver! wow
-echo "Building $driver driver..."
-
-cd "$driver_project_dir"
+    # Check and see if we need to recursively build or install any of our parents before proceeding
+    for parent in $parents; do
+        if [ ! -f resources/modules/"$parent.metabase-driver.jar" ]; then
+            echo "Building $parent..."
+            ./bin/build-driver.sh "$parent"
+        fi
+
+        # Check whether built parent driver *JAR* exists in local Maven repo
+        parent_install_dir="~/.m2/repository/metabase/$parent-driver"
+        parent_installed_jar=''
+        if [ -f "$parent_install_dir" ]; then
+            parent_installed_jar=`find "$parent_install_dir" -name '*.jar'`
+        fi
+
+        if [ ! "$parent_installed_jar" ]; then
+            parent_project_dir="$project_root/modules/drivers/$parent"
+            echo "Installing $parent locally..."
+            cd "$parent_project_dir"
+            lein clean
+            lein install-for-building-drivers
+            cd "$project_root"
+        else
+            echo "$parent already installed to local Maven repo"
+        fi
+    done
+}
+
+# Build the driver uberjar itself
+build_driver_uberjar() {
+    echo "Building $driver driver..."
+
+    cd "$driver_project_dir"
+
+    rm -rf target
 
-rm -rf target
+    lein clean
+    DEBUG=1 LEIN_SNAPSHOTS_IN_RELEASE=true lein uberjar
 
-lein clean
-DEBUG=1 LEIN_SNAPSHOTS_IN_RELEASE=true lein uberjar
+    cd "$project_root"
 
-cd "$project_root"
+    if [ ! -f "$target_jar" ]; then
+        echo "Error: could not find $target_jar. Build failed."
+        exit -1
+    fi
+}
+
+# Strip out any classes in driver JAR found in core Metabase uberjar or parent JARs; recompress with higher compression ratio
+strip_and_compress() {
+    # ok, first things first, strip out any classes also found in the core Metabase uberjar
+    lein strip-and-compress "$target_jar"
+
+    # next, remove any classes also found in any of the parent JARs
+    for parent in $parents; do
+        echo "Removing duplicate classes with $parent uberjar..."
+        lein strip-and-compress "$target_jar" "resources/modules/$parent.metabase-driver.jar"
+    done
+}
+
+# Save the checksum for the newly built JAR
+save_checksum() {
+    echo "Saving checksum for source files to $checksum_file"
+    checksum=`calculate_checksum`
+    echo "$checksum" > "$checksum_file"
+}
+
+# Runs all the steps needed to build the driver.
+build_driver() {
+    delete_old_drivers
+    install_metabase_core
+    build_metabase_uberjar
+    build_parents
+    build_driver_uberjar
+    strip_and_compress
+    save_checksum
+}
+
+######################################## PUTTING IT ALL TOGETHER ########################################
 
-target_jar="$driver_project_dir/target/uberjar/$driver_jar"
+mkdir -p resources/modules
 
-if [ ! -f "$target_jar" ]; then
-    echo "Error: could not find $target_jar. Build failed."
-    exit -1
+if [ ! "$(checksum_is_same)" ]; then
+    build_driver
 fi
 
-# ok, first things first, strip out any classes also found in the core Metabase uberjar
-lein strip-and-compress "$target_jar"
-
-# next, remove any classes also found in any of the parent JARs
-for parent in $parents; do
-    echo "Removing duplicate classes with $parent uberjar..."
-    lein strip-and-compress "$target_jar" "resources/modules/$parent.metabase-driver.jar"
-done
-
 # ok, finally, copy finished JAR to the resources dir
-dest_location="$project_root/resources/modules/$driver_jar"
-
 echo "Copying $target_jar -> $dest_location"
 cp "$target_jar" "$dest_location"

bin/build-drivers.sh

@@ -2,18 +2,24 @@
 
 set -e
 
+# If ran as `./bin/build-drivers.sh clean` then uninstall metabase-core from the local Maven repo and delete
 if [ "$1" == clean ]; then
+    echo "Deleting existing installed metabase-core and driver dependencies..."
     rm -rf ~/.m2/repository/metabase-core
     rm -rf ~/.m2/repository/metabase/*-driver
 
+    echo "Deleting built drivers in resources/modules..."
     rm -rf resources/modules
+    echo "Deleting build Metabase uberjar..."
     rm -rf target
 
-    for target in `find modules -name target -type d`; do
+    for target in `find modules -name 'target' -type d`; do
+        echo "Deleting $target..."
         rm -rf "$target"
     done
 fi
 
-for driver in `ls modules/drivers/ | perl -pe 's|/$||'`; do # strip trailing slashed if ls is set to include them
+for driver in `ls modules/drivers/ | perl -pe 's|/$||'`; do # strip trailing slashes if `ls` is set to include them
+    echo "Build: $driver"
     ./bin/build-driver.sh "$driver"
 done