From 8456a5cb1c84972056fd14bf9722251103f55323 Mon Sep 17 00:00:00 2001
From: Cam Saul <1455846+camsaul@users.noreply.github.com>
Date: Fri, 13 Nov 2020 15:23:46 -0800
Subject: [PATCH] =?UTF-8?q?Mega=20release=20script=20improvements=20?=
=?UTF-8?q?=F0=9F=A4=AF=20(#13795)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* Mega build script improvements 2
* Fix Circle
---
.circleci/config.yml | 36 +++++++++++++++
OSX/release.sh | 5 ++-
bin/build-drivers/deps.edn | 10 ++++-
bin/build-drivers/{ => src}/build_driver.clj | 0
bin/build-drivers/{ => src}/build_drivers.clj | 0
.../{ => src}/build_drivers/build_driver.clj | 0
.../{ => src}/build_drivers/checksum.clj | 0
.../{ => src}/build_drivers/common.clj | 0
.../build_drivers/install_driver_locally.clj | 0
.../{ => src}/build_drivers/metabase.clj | 0
.../build_drivers/plugin_manifest.clj | 0
.../{ => src}/build_drivers/verify.clj | 0
bin/build-drivers/{ => src}/verify_driver.clj | 0
bin/build-mb/deps.edn | 10 ++++-
bin/build-mb/{ => src}/build.clj | 0
bin/build-mb/{ => src}/build/version_info.clj | 34 +++++++-------
bin/build-mb/test/build/version_info_test.clj | 27 +++++++++++
bin/common/deps.edn | 10 ++++-
bin/common/{ => src}/metabuild_common/aws.clj | 0
.../{ => src}/metabuild_common/core.clj | 0
.../{ => src}/metabuild_common/entrypoint.clj | 0
bin/common/{ => src}/metabuild_common/env.clj | 0
.../{ => src}/metabuild_common/files.clj | 0
.../{ => src}/metabuild_common/input.clj | 0
.../{ => src}/metabuild_common/java.clj | 0
.../{ => src}/metabuild_common/output.clj | 0
.../{ => src}/metabuild_common/shell.clj | 0
.../{ => src}/metabuild_common/steps.clj | 0
bin/release/deps.edn | 10 ++++-
bin/release/{ => src}/release.clj | 2 +
.../{ => src}/release/check_prereqs.clj | 0
bin/release/src/release/checkout_latest.clj | 7 +++
bin/release/{ => src}/release/common.clj | 45 +++++++++++++++++++
bin/release/{ => src}/release/common/git.clj | 0
.../{ => src}/release/common/github.clj | 16 ++++---
bin/release/{ => src}/release/common/hash.clj | 0
bin/release/{ => src}/release/common/http.clj | 0
.../{ => src}/release/common/slack.clj | 0
.../{ => src}/release/common/upload.clj | 0
bin/release/{ => src}/release/docker.clj | 19 ++++++--
.../{ => src}/release/draft_release.clj | 5 ++-
.../release/draft_release/release-template.md | 0
.../{ => src}/release/elastic_beanstalk.clj | 17 ++++---
.../.ebextensions/01_metabase.config | 0
.../metabase_config/metabase-setup.sh | 0
.../metabase_config/nginx/log_x_real_ip.conf | 0
.../metabase_config/papertrail/log_files.yml | 0
.../metabase_config/papertrail/remote_syslog | 0
.../launch-aws-eb.html.template | 0
bin/release/{ => src}/release/git_tags.clj | 0
bin/release/{ => src}/release/heroku.clj | 3 ++
.../{ => src}/release/set_build_options.clj | 0
bin/release/{ => src}/release/uberjar.clj | 20 +++++++--
.../{ => src}/release/update_website.clj | 3 ++
.../{ => src}/release/version_info.clj | 5 ++-
bin/release/test/release/common_test.clj | 26 +++++++++++
56 files changed, 264 insertions(+), 46 deletions(-)
rename bin/build-drivers/{ => src}/build_driver.clj (100%)
rename bin/build-drivers/{ => src}/build_drivers.clj (100%)
rename bin/build-drivers/{ => src}/build_drivers/build_driver.clj (100%)
rename bin/build-drivers/{ => src}/build_drivers/checksum.clj (100%)
rename bin/build-drivers/{ => src}/build_drivers/common.clj (100%)
rename bin/build-drivers/{ => src}/build_drivers/install_driver_locally.clj (100%)
rename bin/build-drivers/{ => src}/build_drivers/metabase.clj (100%)
rename bin/build-drivers/{ => src}/build_drivers/plugin_manifest.clj (100%)
rename bin/build-drivers/{ => src}/build_drivers/verify.clj (100%)
rename bin/build-drivers/{ => src}/verify_driver.clj (100%)
rename bin/build-mb/{ => src}/build.clj (100%)
rename bin/build-mb/{ => src}/build/version_info.clj (76%)
create mode 100644 bin/build-mb/test/build/version_info_test.clj
rename bin/common/{ => src}/metabuild_common/aws.clj (100%)
rename bin/common/{ => src}/metabuild_common/core.clj (100%)
rename bin/common/{ => src}/metabuild_common/entrypoint.clj (100%)
rename bin/common/{ => src}/metabuild_common/env.clj (100%)
rename bin/common/{ => src}/metabuild_common/files.clj (100%)
rename bin/common/{ => src}/metabuild_common/input.clj (100%)
rename bin/common/{ => src}/metabuild_common/java.clj (100%)
rename bin/common/{ => src}/metabuild_common/output.clj (100%)
rename bin/common/{ => src}/metabuild_common/shell.clj (100%)
rename bin/common/{ => src}/metabuild_common/steps.clj (100%)
rename bin/release/{ => src}/release.clj (95%)
rename bin/release/{ => src}/release/check_prereqs.clj (100%)
create mode 100644 bin/release/src/release/checkout_latest.clj
rename bin/release/{ => src}/release/common.clj (69%)
rename bin/release/{ => src}/release/common/git.clj (100%)
rename bin/release/{ => src}/release/common/github.clj (83%)
rename bin/release/{ => src}/release/common/hash.clj (100%)
rename bin/release/{ => src}/release/common/http.clj (100%)
rename bin/release/{ => src}/release/common/slack.clj (100%)
rename bin/release/{ => src}/release/common/upload.clj (100%)
rename bin/release/{ => src}/release/docker.clj (58%)
rename bin/release/{ => src}/release/draft_release.clj (94%)
rename bin/release/{ => src}/release/draft_release/release-template.md (100%)
rename bin/release/{ => src}/release/elastic_beanstalk.clj (88%)
rename bin/release/{ => src}/release/elastic_beanstalk/.ebextensions/01_metabase.config (100%)
rename bin/release/{ => src}/release/elastic_beanstalk/.ebextensions/metabase_config/metabase-setup.sh (100%)
rename bin/release/{ => src}/release/elastic_beanstalk/.ebextensions/metabase_config/nginx/log_x_real_ip.conf (100%)
rename bin/release/{ => src}/release/elastic_beanstalk/.ebextensions/metabase_config/papertrail/log_files.yml (100%)
rename bin/release/{ => src}/release/elastic_beanstalk/.ebextensions/metabase_config/papertrail/remote_syslog (100%)
rename bin/release/{ => src}/release/elastic_beanstalk/launch-aws-eb.html.template (100%)
rename bin/release/{ => src}/release/git_tags.clj (100%)
rename bin/release/{ => src}/release/heroku.clj (94%)
rename bin/release/{ => src}/release/set_build_options.clj (100%)
rename bin/release/{ => src}/release/uberjar.clj (71%)
rename bin/release/{ => src}/release/update_website.clj (97%)
rename bin/release/{ => src}/release/version_info.clj (95%)
create mode 100644 bin/release/test/release/common_test.clj
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3c992330c8a..291fcc74408 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -474,6 +474,38 @@ jobs:
./bin/test-load-and-dump.sh
no_output_timeout: 5m
+ test-build-scripts:
+ executor: clojure-and-node
+ steps:
+ - attach-workspace
+ - restore-be-deps-cache
+ - run:
+ name: Install Clojure CLI
+ command: >
+ curl -O https://download.clojure.org/install/linux-install-1.10.1.708.sh &&
+ chmod +x linux-install-1.10.1.708.sh &&
+ sudo ./linux-install-1.10.1.708.sh
+ - run:
+ name: Run metabuild-common build script tests
+ command: >
+ cd /home/circleci/metabase/metabase/bin/common && clojure -M:test
+ no_output_timeout: 5m
+ - run:
+ name: Run build-drivers build script tests
+ command: >
+ cd /home/circleci/metabase/metabase/bin/build-drivers && clojure -M:test
+ no_output_timeout: 5m
+ - run:
+ name: Run build-mb build script tests
+ command: >
+ cd /home/circleci/metabase/metabase/bin/build-mb && clojure -M:test
+ no_output_timeout: 5m
+ - run:
+ name: Run release script tests
+ command: >
+ cd /home/circleci/metabase/metabase/bin/release && clojure -M:test
+ no_output_timeout: 5m
+
########################################################################################################################
# FRONTEND #
@@ -960,6 +992,10 @@ workflows:
db-type: mysql
<<: *Matrix
+ - test-build-scripts:
+ requires:
+ - be-deps
+
- fe-deps:
requires:
- checkout
diff --git a/OSX/release.sh b/OSX/release.sh
index f44195faa4f..78b679a8e7f 100755
--- a/OSX/release.sh
+++ b/OSX/release.sh
@@ -2,7 +2,10 @@
set -euo pipefail
-source "./bin/check-clojure-cli.sh"
+script_directory=`dirname "${BASH_SOURCE[0]}"`
+cd "$script_directory"
+
+source "../bin/check-clojure-cli.sh"
check_clojure_cli
cd OSX/macos_release
diff --git a/bin/build-drivers/deps.edn b/bin/build-drivers/deps.edn
index bb291868c5a..ce487fa0905 100644
--- a/bin/build-drivers/deps.edn
+++ b/bin/build-drivers/deps.edn
@@ -1,4 +1,4 @@
-{:paths ["./"]
+{:paths ["src"]
:deps
{common/common {:local/root "../common"}
@@ -6,4 +6,10 @@
commons-codec/commons-codec {:mvn/version "1.14"}
hiccup/hiccup {:mvn/version "1.0.5"}
io.forward/yaml {:mvn/version "1.0.9"}
- stencil/stencil {:mvn/version "0.5.0"}}}
+ stencil/stencil {:mvn/version "0.5.0"}}
+
+ :aliases
+ {:test {:extra-paths ["test"]
+ :extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner.git"
+ :sha "209b64504cb3bd3b99ecfec7937b358a879f55c1"}}
+ :main-opts ["-m" "cognitect.test-runner"]}}}
diff --git a/bin/build-drivers/build_driver.clj b/bin/build-drivers/src/build_driver.clj
similarity index 100%
rename from bin/build-drivers/build_driver.clj
rename to bin/build-drivers/src/build_driver.clj
diff --git a/bin/build-drivers/build_drivers.clj b/bin/build-drivers/src/build_drivers.clj
similarity index 100%
rename from bin/build-drivers/build_drivers.clj
rename to bin/build-drivers/src/build_drivers.clj
diff --git a/bin/build-drivers/build_drivers/build_driver.clj b/bin/build-drivers/src/build_drivers/build_driver.clj
similarity index 100%
rename from bin/build-drivers/build_drivers/build_driver.clj
rename to bin/build-drivers/src/build_drivers/build_driver.clj
diff --git a/bin/build-drivers/build_drivers/checksum.clj b/bin/build-drivers/src/build_drivers/checksum.clj
similarity index 100%
rename from bin/build-drivers/build_drivers/checksum.clj
rename to bin/build-drivers/src/build_drivers/checksum.clj
diff --git a/bin/build-drivers/build_drivers/common.clj b/bin/build-drivers/src/build_drivers/common.clj
similarity index 100%
rename from bin/build-drivers/build_drivers/common.clj
rename to bin/build-drivers/src/build_drivers/common.clj
diff --git a/bin/build-drivers/build_drivers/install_driver_locally.clj b/bin/build-drivers/src/build_drivers/install_driver_locally.clj
similarity index 100%
rename from bin/build-drivers/build_drivers/install_driver_locally.clj
rename to bin/build-drivers/src/build_drivers/install_driver_locally.clj
diff --git a/bin/build-drivers/build_drivers/metabase.clj b/bin/build-drivers/src/build_drivers/metabase.clj
similarity index 100%
rename from bin/build-drivers/build_drivers/metabase.clj
rename to bin/build-drivers/src/build_drivers/metabase.clj
diff --git a/bin/build-drivers/build_drivers/plugin_manifest.clj b/bin/build-drivers/src/build_drivers/plugin_manifest.clj
similarity index 100%
rename from bin/build-drivers/build_drivers/plugin_manifest.clj
rename to bin/build-drivers/src/build_drivers/plugin_manifest.clj
diff --git a/bin/build-drivers/build_drivers/verify.clj b/bin/build-drivers/src/build_drivers/verify.clj
similarity index 100%
rename from bin/build-drivers/build_drivers/verify.clj
rename to bin/build-drivers/src/build_drivers/verify.clj
diff --git a/bin/build-drivers/verify_driver.clj b/bin/build-drivers/src/verify_driver.clj
similarity index 100%
rename from bin/build-drivers/verify_driver.clj
rename to bin/build-drivers/src/verify_driver.clj
diff --git a/bin/build-mb/deps.edn b/bin/build-mb/deps.edn
index 434f9aeabe2..746faa697ca 100644
--- a/bin/build-mb/deps.edn
+++ b/bin/build-mb/deps.edn
@@ -1,6 +1,12 @@
-{:paths ["./"]
+{:paths ["src"]
:deps
{common/common {:local/root "../common"}
build-drivers/build-drivers {:local/root "../build-drivers"}
- org.flatland/ordered {:mvn/version "1.5.7"}}}
+ org.flatland/ordered {:mvn/version "1.5.7"}}
+
+ :aliases
+ {:test {:extra-paths ["test"]
+ :extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner.git"
+ :sha "209b64504cb3bd3b99ecfec7937b358a879f55c1"}}
+ :main-opts ["-m" "cognitect.test-runner"]}}}
diff --git a/bin/build-mb/build.clj b/bin/build-mb/src/build.clj
similarity index 100%
rename from bin/build-mb/build.clj
rename to bin/build-mb/src/build.clj
diff --git a/bin/build-mb/build/version_info.clj b/bin/build-mb/src/build/version_info.clj
similarity index 76%
rename from bin/build-mb/build/version_info.clj
rename to bin/build-mb/src/build/version_info.clj
index 011b3109325..e490dff4b5c 100644
--- a/bin/build-mb/build/version_info.clj
+++ b/bin/build-mb/src/build/version_info.clj
@@ -38,12 +38,13 @@
(defn most-recent-tag []
(shell-output-when-nonzero "git" "describe" "--abbrev=0" "--tags"))
-(defn most-recent-tag-parts []
- (when-let [tag (most-recent-tag)]
- (for [part (str/split tag #"\.")
- :let [numeric-part (re-find #"\d+" part)]
- :when (seq numeric-part)]
- (Integer/parseInt numeric-part))))
+(defn- tag-parts [tag]
+ (when tag
+ (not-empty
+ (for [part (str/split tag #"\.")
+ :let [numeric-part (re-find #"\d+" part)]
+ :when (seq numeric-part)]
+ (Integer/parseInt numeric-part)))))
(defn current-snapshot-version
"Attempt to come up with a snapshot version for builds that aren't given explicit version info based on the most
@@ -54,15 +55,18 @@
For builds from `master`, increment the minor version instead e.g.
v0.37.1 -> v0.38.0-SNAPSHOT"
- []
- (if-let [tag-parts (most-recent-tag-parts)]
- (let [[major minor patch] tag-parts
- major (or major 0)
- [minor patch] (if (= (git-branch) "master")
- [(inc (or minor 0)) 0]
- [(or minor 0) (inc (or patch 0))])]
- (format "v%d.%d.%d-SNAPSHOT" major minor patch))
- "UNKNOWN"))
+ ([]
+ (current-snapshot-version (git-branch) (most-recent-tag)))
+
+ ([branch tag]
+ (if-let [tag-parts (not-empty (tag-parts tag))]
+ (let [[major minor patch] tag-parts
+ major (or major 0)
+ [minor patch] (if (= branch "master")
+ [(inc (or minor 0)) 0]
+ [(or minor 0) (inc (or patch 0))])]
+ (format "v%d.%d.%d-SNAPSHOT" major minor patch))
+ "UNKNOWN")))
(defn generate-version-info-file!
"Generate version.properties file"
diff --git a/bin/build-mb/test/build/version_info_test.clj b/bin/build-mb/test/build/version_info_test.clj
new file mode 100644
index 00000000000..f4e64185b01
--- /dev/null
+++ b/bin/build-mb/test/build/version_info_test.clj
@@ -0,0 +1,27 @@
+(ns build.version-info-test
+ (:require [build.version-info :as version-info]
+ [clojure.test :refer :all]))
+
+(deftest tag-parts-test
+ (doseq [[tag expected] {nil nil
+ "0.37.0" [0 37 0]
+ "0.37.0.1" [0 37 0 1]
+ "0.37.1-rc2" [0 37 1]}
+ tag [tag (str \v tag)]]
+ (testing (str (pr-str (list 'tag-parts tag)) " => " (pr-str expected))
+ (is (= expected
+ (#'version-info/tag-parts tag))))))
+
+(deftest current-snapshot-version-test
+ (doseq [[branch tag->expected] {"release-x.37.x" {nil "UNKNOWN"
+ "0.37.0" "v0.37.1-SNAPSHOT"
+ "0.37.0.1" "v0.37.1-SNAPSHOT"
+ "0.37.1-rc2" "v0.37.2-SNAPSHOT"}
+ "master" {nil "UNKNOWN"
+ "0.37.0" "v0.38.0-SNAPSHOT"
+ "0.37.0.1" "v0.38.0-SNAPSHOT"
+ "0.37.1-rc2" "v0.38.0-SNAPSHOT"}}
+ [tag expected] tag->expected]
+ (testing (str (pr-str (list 'current-snapshot-version branch tag)) " => " (pr-str expected))
+ (is (= expected
+ (version-info/current-snapshot-version branch tag))))))
diff --git a/bin/common/deps.edn b/bin/common/deps.edn
index 31ea40aa439..d70bd6b86d1 100644
--- a/bin/common/deps.edn
+++ b/bin/common/deps.edn
@@ -1,7 +1,13 @@
-{:paths ["./"]
+{:paths ["src"]
:deps
{commons-io/commons-io {:mvn/version "2.6"}
colorize/colorize {:mvn/version "0.1.1"}
environ/environ {:mvn/version "1.1.0"}
- potemkin/potemkin {:mvn/version "0.4.5"}}}
+ potemkin/potemkin {:mvn/version "0.4.5"}}
+
+ :aliases
+ {:test {:extra-paths ["test"]
+ :extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner.git"
+ :sha "209b64504cb3bd3b99ecfec7937b358a879f55c1"}}
+ :main-opts ["-m" "cognitect.test-runner"]}}}
diff --git a/bin/common/metabuild_common/aws.clj b/bin/common/src/metabuild_common/aws.clj
similarity index 100%
rename from bin/common/metabuild_common/aws.clj
rename to bin/common/src/metabuild_common/aws.clj
diff --git a/bin/common/metabuild_common/core.clj b/bin/common/src/metabuild_common/core.clj
similarity index 100%
rename from bin/common/metabuild_common/core.clj
rename to bin/common/src/metabuild_common/core.clj
diff --git a/bin/common/metabuild_common/entrypoint.clj b/bin/common/src/metabuild_common/entrypoint.clj
similarity index 100%
rename from bin/common/metabuild_common/entrypoint.clj
rename to bin/common/src/metabuild_common/entrypoint.clj
diff --git a/bin/common/metabuild_common/env.clj b/bin/common/src/metabuild_common/env.clj
similarity index 100%
rename from bin/common/metabuild_common/env.clj
rename to bin/common/src/metabuild_common/env.clj
diff --git a/bin/common/metabuild_common/files.clj b/bin/common/src/metabuild_common/files.clj
similarity index 100%
rename from bin/common/metabuild_common/files.clj
rename to bin/common/src/metabuild_common/files.clj
diff --git a/bin/common/metabuild_common/input.clj b/bin/common/src/metabuild_common/input.clj
similarity index 100%
rename from bin/common/metabuild_common/input.clj
rename to bin/common/src/metabuild_common/input.clj
diff --git a/bin/common/metabuild_common/java.clj b/bin/common/src/metabuild_common/java.clj
similarity index 100%
rename from bin/common/metabuild_common/java.clj
rename to bin/common/src/metabuild_common/java.clj
diff --git a/bin/common/metabuild_common/output.clj b/bin/common/src/metabuild_common/output.clj
similarity index 100%
rename from bin/common/metabuild_common/output.clj
rename to bin/common/src/metabuild_common/output.clj
diff --git a/bin/common/metabuild_common/shell.clj b/bin/common/src/metabuild_common/shell.clj
similarity index 100%
rename from bin/common/metabuild_common/shell.clj
rename to bin/common/src/metabuild_common/shell.clj
diff --git a/bin/common/metabuild_common/steps.clj b/bin/common/src/metabuild_common/steps.clj
similarity index 100%
rename from bin/common/metabuild_common/steps.clj
rename to bin/common/src/metabuild_common/steps.clj
diff --git a/bin/release/deps.edn b/bin/release/deps.edn
index e4e3bb9f579..4f49b98e30a 100644
--- a/bin/release/deps.edn
+++ b/bin/release/deps.edn
@@ -1,4 +1,4 @@
-{:paths ["./"]
+{:paths ["src"]
:deps
{common/common {:local/root "../common"}
@@ -8,4 +8,10 @@
enlive/enlive {:mvn/version "1.1.6"}
hiccup/hiccup {:mvn/version "1.0.5"}
org.flatland/ordered {:mvn/version "1.5.7"}
- stencil/stencil {:mvn/version "0.5.0"}}}
+ stencil/stencil {:mvn/version "0.5.0"}}
+
+ :aliases
+ {:test {:extra-paths ["test"]
+ :extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner.git"
+ :sha "209b64504cb3bd3b99ecfec7937b358a879f55c1"}}
+ :main-opts ["-m" "cognitect.test-runner"]}}}
diff --git a/bin/release/release.clj b/bin/release/src/release.clj
similarity index 95%
rename from bin/release/release.clj
rename to bin/release/src/release.clj
index 940cc8dc085..96ea1327a4f 100644
--- a/bin/release/release.clj
+++ b/bin/release/src/release.clj
@@ -5,6 +5,7 @@
[metabuild-common.core :as u]
[release
[check-prereqs :as check-prereqs]
+ [checkout-latest :as checkout-latest]
[common :as c]
[docker :as docker]
[draft-release :as draft-release]
@@ -21,6 +22,7 @@
(def ^:private steps*
(ordered-map/ordered-map
+ :checkout-latest checkout-latest/checkout-latest!
:build-uberjar uberjar/build-uberjar!
:build-docker docker/build-docker-image!
:push-git-tags git-tags/push-tags!
diff --git a/bin/release/release/check_prereqs.clj b/bin/release/src/release/check_prereqs.clj
similarity index 100%
rename from bin/release/release/check_prereqs.clj
rename to bin/release/src/release/check_prereqs.clj
diff --git a/bin/release/src/release/checkout_latest.clj b/bin/release/src/release/checkout_latest.clj
new file mode 100644
index 00000000000..50798517dca
--- /dev/null
+++ b/bin/release/src/release/checkout_latest.clj
@@ -0,0 +1,7 @@
+(ns release.checkout-latest
+ (:require [metabuild-common.core :as u]
+ [release.common :as c]))
+
+(defn checkout-latest! []
+ (u/step (format "Checkout latest code for %s" (c/branch))
+ (u/sh {:dir u/project-root-directory} "git" "pull")))
diff --git a/bin/release/release/common.clj b/bin/release/src/release/common.clj
similarity index 69%
rename from bin/release/release/common.clj
rename to bin/release/src/release/common.clj
index 62bdd257eab..f47e07e344b 100644
--- a/bin/release/release/common.clj
+++ b/bin/release/src/release/common.clj
@@ -137,3 +137,48 @@
([s3-bucket version filename]
(format "s3://%s%s" s3-bucket (s3-artifact-path version filename))))
+
+(defn version-greater-than
+ "Is version string `x` greater than version `y`?"
+ [x y]
+ {:pre [(string? x) (string? y)]}
+ (letfn [(parts [s]
+ (for [part (-> s (str/replace #"^v" "") (str/split #"\."))]
+ (Integer/parseUnsignedInt part)))]
+ (loop [[x & x-more] (parts x) [y & y-more] (parts y)]
+ (cond
+ ((fnil > 0 0) x y) true
+ ((fnil < 0 0) x y) false
+ ((fnil = 0 0) x y) (if (or (seq x-more) (seq y-more))
+ (recur x-more y-more)
+ false)))))
+
+(defn- recent-tags-from-github
+ "Recent tags for the current edition from GitHub."
+ []
+ (->> ((requiring-resolve 'release.common.github/recent-tags))
+ (filter (case (edition)
+ :ee (partial re-matches #"v1(?:\.\d+){2,}$")
+ :ce (partial re-matches #"v0(?:\.\d+){2,}$")))))
+
+(defn- most-recent-tag
+ "Given a set of release `tags`, return the most recent one."
+ [tags]
+ (->> tags
+ (sort-by identity (fn [x y]
+ (cond
+ (version-greater-than x y) -1
+ (version-greater-than y x) 1
+ :else 0)))
+ first))
+
+(def ^{:arglists '([])} latest-version?
+ "Is the version we're building going to be the new latest version for this edition?"
+ (memoize
+ (fn []
+ (u/step (format "Check whether %s will be the latest version" (version))
+ (let [latest-gh-tag (most-recent-tag (recent-tags-from-github))]
+ (u/announce "Latest %s version from GitHub is %s" (edition) latest-gh-tag)
+ (let [latest? (version-greater-than (version) latest-gh-tag)]
+ (u/announce "%s %s be the new latest %s version." (version) (if latest? "will" "will NOT") (edition))
+ latest?))))))
diff --git a/bin/release/release/common/git.clj b/bin/release/src/release/common/git.clj
similarity index 100%
rename from bin/release/release/common/git.clj
rename to bin/release/src/release/common/git.clj
diff --git a/bin/release/release/common/github.clj b/bin/release/src/release/common/github.clj
similarity index 83%
rename from bin/release/release/common/github.clj
rename to bin/release/src/release/common/github.clj
index 3a95ea4c3b7..7d9a8600c2d 100644
--- a/bin/release/release/common/github.clj
+++ b/bin/release/src/release/common/github.clj
@@ -2,8 +2,8 @@
(:require [cheshire.core :as json]
[clj-http.client :as http]
[clojure.string :as str]
- [release.common :as c]
- [metabuild-common.core :as u]))
+ [metabuild-common.core :as u]
+ [release.common :as c]))
(defn github-api-base []
(str "https://api.github.com/repos/" (c/metabase-repo)))
@@ -12,12 +12,15 @@
{"Content-Type" "application/json"
"Authorization" (format "token %s" (u/env-or-throw :github-token))})
+(defn- GET [endpoint]
+ (-> (http/get (str (github-api-base) endpoint) {:headers (github-api-request-headers)})
+ :body
+ (json/parse-string true)))
+
(defn milestones
"Fetch open GitHub milestones for the current repo."
[]
- (-> (http/get (str (github-api-base) "/milestones") {:headers (github-api-request-headers)})
- :body
- (json/parse-string true)))
+ (GET "/milestones"))
(defn matching-milestone
"Return the GitHub milestone matching the version we're releasing, if any."
@@ -46,3 +49,6 @@
labels)
:bug
:enhancement))
+
+(defn recent-tags []
+ (map :tag_name (GET "/releases")))
diff --git a/bin/release/release/common/hash.clj b/bin/release/src/release/common/hash.clj
similarity index 100%
rename from bin/release/release/common/hash.clj
rename to bin/release/src/release/common/hash.clj
diff --git a/bin/release/release/common/http.clj b/bin/release/src/release/common/http.clj
similarity index 100%
rename from bin/release/release/common/http.clj
rename to bin/release/src/release/common/http.clj
diff --git a/bin/release/release/common/slack.clj b/bin/release/src/release/common/slack.clj
similarity index 100%
rename from bin/release/release/common/slack.clj
rename to bin/release/src/release/common/slack.clj
diff --git a/bin/release/release/common/upload.clj b/bin/release/src/release/common/upload.clj
similarity index 100%
rename from bin/release/release/common/upload.clj
rename to bin/release/src/release/common/upload.clj
diff --git a/bin/release/release/docker.clj b/bin/release/src/release/docker.clj
similarity index 58%
rename from bin/release/release/docker.clj
rename to bin/release/src/release/docker.clj
index d91a162a24e..e976409edbb 100644
--- a/bin/release/release/docker.clj
+++ b/bin/release/src/release/docker.clj
@@ -5,8 +5,12 @@
(defn build-docker-image! []
(u/step "Build Docker image"
- (u/copy-file! (u/assert-file-exists c/uberjar-path) (str c/root-directory "/bin/docker/metabase.jar"))
- (u/sh "docker" "build" "-t" (c/docker-tag) (str c/root-directory "/bin/docker"))))
+ (let [docker-dir (u/filename c/root-directory "bin" "docker")
+ uberjar-path (u/filename docker-dir "metabase.jar")]
+ (u/delete-file-if-exists! uberjar-path)
+ (u/copy-file! (u/assert-file-exists c/uberjar-path) uberjar-path)
+ (u/assert-file-exists uberjar-path)
+ (u/sh "docker" "build" "-t" (c/docker-tag) docker-dir))))
(defn- validate-docker-image []
(u/step "Validate Docker image"
@@ -22,8 +26,15 @@
(defn push-docker-image! []
(u/step "Push Docker image"
(u/sh "docker" "push" (c/docker-tag))
- (when-not (c/pre-release-version?)
- (let [latest-tag (str (c/docker-image-name) ":latest")]
+ (let [latest-tag (str (c/docker-image-name) ":latest")]
+ (cond
+ (c/pre-release-version?)
+ (u/announce "Pre release version -- not pushing %s" latest-tag)
+
+ (not (c/latest-version?))
+ (u/announce "Version is not latest -- not pushing %s" latest-tag)
+
+ :else
(u/step (format "Pushing tag %s" latest-tag)
(u/sh "docker" "tag" (c/docker-tag) latest-tag)
(u/sh "docker" "push" latest-tag)))))
diff --git a/bin/release/release/draft_release.clj b/bin/release/src/release/draft_release.clj
similarity index 94%
rename from bin/release/release/draft_release.clj
rename to bin/release/src/release/draft_release.clj
index 7d68badb59d..9bb8532aa38 100644
--- a/bin/release/release/draft_release.clj
+++ b/bin/release/src/release/draft_release.clj
@@ -13,11 +13,14 @@
;; Disable caching of our template files for easier REPL debugging, we're only rendering them once anyways
(stencil.loader/set-cache (cache/ttl-cache-factory {} :ttl 0))
+(def ^:private release-template-filename
+ (u/assert-file-exists (u/filename c/root-directory "bin" "release" "src" "release" "draft_release" "release-template.md")))
+
(defn- generate-draft-changelog []
(u/step "Generate draft changelog"
(let [pre-release? (c/pre-release-version?)
{bugs :bug, enhancements :enhancement} (group-by github/issue-type (github/milestone-issues))]
- (stencil/render-file (u/assert-file-exists "release/draft_release/release-template.md")
+ (stencil/render-file release-template-filename
{:enhancements enhancements
:bug-fixes bugs
:docker-tag (c/docker-tag)
diff --git a/bin/release/release/draft_release/release-template.md b/bin/release/src/release/draft_release/release-template.md
similarity index 100%
rename from bin/release/release/draft_release/release-template.md
rename to bin/release/src/release/draft_release/release-template.md
diff --git a/bin/release/release/elastic_beanstalk.clj b/bin/release/src/release/elastic_beanstalk.clj
similarity index 88%
rename from bin/release/release/elastic_beanstalk.clj
rename to bin/release/src/release/elastic_beanstalk.clj
index ae365299e25..cd449dd58c3 100644
--- a/bin/release/release/elastic_beanstalk.clj
+++ b/bin/release/src/release/elastic_beanstalk.clj
@@ -23,7 +23,7 @@
(def ^:private eb-extensions-source
"Source location of the .ebextensions directory"
- (u/assert-file-exists (u/filename c/root-directory "bin" "release" "release" "elastic_beanstalk" ".ebextensions")))
+ (u/assert-file-exists (u/filename c/root-directory "bin" "release" "src" "release" "elastic_beanstalk" ".ebextensions")))
(def ^:private archive-temp-dir
"Path where we'll put the contents of the ZIP file before we create it."
@@ -81,14 +81,17 @@
(u/sh {:dir archive-temp-dir} "zip" "--recurse-paths" archive-path ".")
(u/assert-file-exists archive-path))))
+(def ^:private launch-template-filename
+ (u/assert-file-exists (u/filename c/root-directory "bin" "release" "src" "release" "elastic_beanstalk" "launch-aws-eb.html.template")))
+
(defn- create-html-file! []
(u/step (format "Create launch-aws-eb.html for Docker image %s" (c/docker-tag))
- (u/delete-file-if-exists! html-file-path)
- (spit html-file-path
- (stencil/render-file (u/assert-file-exists "release/elastic_beanstalk/launch-aws-eb.html.template")
- {:url (java.net.URLEncoder/encode (c/artifact-download-url "metabase-aws-eb.zip")
- "UTF-8")}))
- (u/assert-file-exists html-file-path)))
+ (u/delete-file-if-exists! html-file-path)
+ (spit html-file-path
+ (stencil/render-file launch-template-filename
+ {:url (java.net.URLEncoder/encode (c/artifact-download-url "metabase-aws-eb.zip")
+ "UTF-8")}))
+ (u/assert-file-exists html-file-path)))
(defn- upload-artifacts! []
(u/step "Upload Elastic Beanstalk artifacts"
diff --git a/bin/release/release/elastic_beanstalk/.ebextensions/01_metabase.config b/bin/release/src/release/elastic_beanstalk/.ebextensions/01_metabase.config
similarity index 100%
rename from bin/release/release/elastic_beanstalk/.ebextensions/01_metabase.config
rename to bin/release/src/release/elastic_beanstalk/.ebextensions/01_metabase.config
diff --git a/bin/release/release/elastic_beanstalk/.ebextensions/metabase_config/metabase-setup.sh b/bin/release/src/release/elastic_beanstalk/.ebextensions/metabase_config/metabase-setup.sh
similarity index 100%
rename from bin/release/release/elastic_beanstalk/.ebextensions/metabase_config/metabase-setup.sh
rename to bin/release/src/release/elastic_beanstalk/.ebextensions/metabase_config/metabase-setup.sh
diff --git a/bin/release/release/elastic_beanstalk/.ebextensions/metabase_config/nginx/log_x_real_ip.conf b/bin/release/src/release/elastic_beanstalk/.ebextensions/metabase_config/nginx/log_x_real_ip.conf
similarity index 100%
rename from bin/release/release/elastic_beanstalk/.ebextensions/metabase_config/nginx/log_x_real_ip.conf
rename to bin/release/src/release/elastic_beanstalk/.ebextensions/metabase_config/nginx/log_x_real_ip.conf
diff --git a/bin/release/release/elastic_beanstalk/.ebextensions/metabase_config/papertrail/log_files.yml b/bin/release/src/release/elastic_beanstalk/.ebextensions/metabase_config/papertrail/log_files.yml
similarity index 100%
rename from bin/release/release/elastic_beanstalk/.ebextensions/metabase_config/papertrail/log_files.yml
rename to bin/release/src/release/elastic_beanstalk/.ebextensions/metabase_config/papertrail/log_files.yml
diff --git a/bin/release/release/elastic_beanstalk/.ebextensions/metabase_config/papertrail/remote_syslog b/bin/release/src/release/elastic_beanstalk/.ebextensions/metabase_config/papertrail/remote_syslog
similarity index 100%
rename from bin/release/release/elastic_beanstalk/.ebextensions/metabase_config/papertrail/remote_syslog
rename to bin/release/src/release/elastic_beanstalk/.ebextensions/metabase_config/papertrail/remote_syslog
diff --git a/bin/release/release/elastic_beanstalk/launch-aws-eb.html.template b/bin/release/src/release/elastic_beanstalk/launch-aws-eb.html.template
similarity index 100%
rename from bin/release/release/elastic_beanstalk/launch-aws-eb.html.template
rename to bin/release/src/release/elastic_beanstalk/launch-aws-eb.html.template
diff --git a/bin/release/release/git_tags.clj b/bin/release/src/release/git_tags.clj
similarity index 100%
rename from bin/release/release/git_tags.clj
rename to bin/release/src/release/git_tags.clj
diff --git a/bin/release/release/heroku.clj b/bin/release/src/release/heroku.clj
similarity index 94%
rename from bin/release/release/heroku.clj
rename to bin/release/src/release/heroku.clj
index 5f76a09fc62..b46b9d6094b 100644
--- a/bin/release/release/heroku.clj
+++ b/bin/release/src/release/heroku.clj
@@ -26,6 +26,9 @@
(= (c/edition) :ee)
(u/announce "EE build -- not updating Herkou buildpack")
+ (not (c/latest-version?))
+ (u/announce "Not the latest version -- not updating Heroku buildpack")
+
:else
(do
(u/step (format "Clone Herkou Buildpack repo %s to %s" heroku-repo dir)
diff --git a/bin/release/release/set_build_options.clj b/bin/release/src/release/set_build_options.clj
similarity index 100%
rename from bin/release/release/set_build_options.clj
rename to bin/release/src/release/set_build_options.clj
diff --git a/bin/release/release/uberjar.clj b/bin/release/src/release/uberjar.clj
similarity index 71%
rename from bin/release/release/uberjar.clj
rename to bin/release/src/release/uberjar.clj
index 48d0df7afe7..3d7addf0859 100644
--- a/bin/release/release/uberjar.clj
+++ b/bin/release/src/release/uberjar.clj
@@ -19,7 +19,8 @@
(defn- validate-uberjar []
(u/step "Validate uberjar(s) on downloads.metabase.com"
(doseq [url [(c/artifact-download-url "metabase.jar")
- (when (= (c/edition) :ee)
+ (when (and (= (c/edition) :ee)
+ (c/latest-version?))
(c/artifact-download-url "latest" "metabase.jar"))]
:when url]
(u/step (format "Validate %s" url)
@@ -38,7 +39,18 @@
(u/step "Upload uberjar and validate"
(u/step (format "Upload uberjar to %s" (c/artifact-download-url "metabase.jar"))
(upload/upload-artifact! c/uberjar-path "metabase.jar"))
- (when (= (c/edition) :ee)
- (u/step (format "Upload uberjar to %s" (c/artifact-download-url "latest" "metabase.jar"))
- (upload/upload-artifact! c/uberjar-path "latest" "metabase.jar")))
+ (let [latest-download-url (c/artifact-download-url "latest" "metabase.jar")]
+ (cond
+ (not= (c/edition) :ee)
+ (u/announce "Not EE build -- not uploading %s" latest-download-url)
+
+ (c/pre-release-version?)
+ (u/announce "Pre release version -- not uploading %s" latest-download-url)
+
+ (not (c/latest-version?))
+ (u/announce "Not latest version -- not uploading %s" latest-download-url)
+
+ :else
+ (u/step (format "Upload uberjar to %s" latest-download-url)
+ (upload/upload-artifact! c/uberjar-path "latest" "metabase.jar"))))
(validate-uberjar)))
diff --git a/bin/release/release/update_website.clj b/bin/release/src/release/update_website.clj
similarity index 97%
rename from bin/release/release/update_website.clj
rename to bin/release/src/release/update_website.clj
index c69fec81324..91e1bab2f0f 100644
--- a/bin/release/release/update_website.clj
+++ b/bin/release/src/release/update_website.clj
@@ -92,6 +92,9 @@
(= (c/edition) :ee)
(u/announce "EE build -- not updating Metabase docs")
+ (not (c/latest-version?))
+ (u/announce "Not the latest version -- not updating Metabase docs")
+
:else
(do
(update-website!*)
diff --git a/bin/release/release/version_info.clj b/bin/release/src/release/version_info.clj
similarity index 95%
rename from bin/release/release/version_info.clj
rename to bin/release/src/release/version_info.clj
index 8e87d1c6ed7..9a8600b7d24 100644
--- a/bin/release/release/version_info.clj
+++ b/bin/release/src/release/version_info.clj
@@ -33,7 +33,7 @@
(defn- generate-version-info! []
(u/step "Generate version-info.json"
(u/step (format "Generate %s" tmp-version-info-filename)
- (u/delete-file! tmp-version-info-filename)
+ (u/delete-file-if-exists! tmp-version-info-filename)
(let [{:keys [latest], :as info} (current-version-info)]
(spit tmp-version-info-filename (-> info
;; move the current `:latest` to the beginning of `:older`
@@ -80,5 +80,8 @@
(= (c/edition) :ee)
(u/announce "Enterprise Edition release, not updating version-info.json")
+ (not (c/latest-version?))
+ (u/announce "Not the latest version, not updating version-info.json")
+
:else
(update-version-info!*))))
diff --git a/bin/release/test/release/common_test.clj b/bin/release/test/release/common_test.clj
new file mode 100644
index 00000000000..5b303b89da5
--- /dev/null
+++ b/bin/release/test/release/common_test.clj
@@ -0,0 +1,26 @@
+(ns release.common-test
+ (:require [clojure.test :refer :all]
+ [release.common :as c]))
+
+(deftest version-greater-than-test
+ (doseq [[x y expected] [["1.37.0" "1.36.0" true]
+ ["1.36.1" "1.36.0" true]
+ ["1.36.0" "1.36.0" false]
+ ["1.37.0" "1.36.9" true]
+ ["1.37.0" "1.36.9" true]
+ ["1.37.0.1" "1.37.0" true]
+ ["1.37.1.1" "1.37.0.2" true]
+ ["1.37.10" "1.37.2" true]
+ ["1.37.0.0" "1.37.0" false]]]
+ (testing (str (pr-str (list 'version-greater-than x y)) " => " (pr-str expected))
+ (doseq [x [x (str "v" x)]
+ y [y (str "v" y)]]
+ (is (= expected
+ (c/version-greater-than x y)))
+ (when expected
+ (is (= (not expected)
+ (c/version-greater-than y x))))))))
+
+(deftest most-recent-tag-test
+ (is (= "v1.37.0.1"
+ (#'c/most-recent-tag (shuffle ["v1.37.0.0" "v1.37.0.1" "v1.36.0" "v1.36.8.9"])))))
--
GitLab