Merge pull request #12087 from metabase/merge-release-0.34.x-into-master

Merge release-0.34.x into master

.circleci/config.yml

@@ -266,7 +266,7 @@ jobs:
       - run:
           name: Generate checksums of all backend source files to use as Uberjar cache key
           command: >
-            for file in `find ./src -type f -name '*.clj' | sort`;
+            for file in `find ./src ./backend -type f -name '*.clj' | sort`;
               do echo `md5sum $file` >> backend-checksums.txt;
             done;
             echo `md5sum project.clj` >> backend-checksums.txt

.gitignore

@@ -57,6 +57,7 @@
 /stats.json
 /target
 /test-report-*
+OSX/.cpcache
 OSX/Metabase/jre
 OSX/Resources/metabase.jar
 OSX/build

OSX/Metabase/Metabase-Info.plist

@@ -17,11 +17,11 @@
 	<key>CFBundlePackageType</key>
 	<string>APPL</string>
 	<key>CFBundleShortVersionString</key>
-	<string>0.15.0.0</string>
+	<string>0.34.3</string>
 	<key>CFBundleSignature</key>
 	<string>????</string>
 	<key>CFBundleVersion</key>
-	<string>0.15.0.0</string>
+	<string>0.34.3</string>
 	<key>LSApplicationCategoryType</key>
 	<string>public.app-category.utilities</string>
 	<key>LSMinimumSystemVersion</key>

OSX/deps.edn

+{:paths ["./"]
+
+ :deps
+ {org.clojure/data.xml  {:mvn/version "0.0.8"}
+  cheshire              {:mvn/version "5.8.1"}
+  clj-http              {:mvn/version "3.9.1"}
+  clj-tagsoup           {:mvn/version "0.3.0"}
+  commons-io/commons-io {:mvn/version "2.6"}
+  colorize              {:mvn/version "0.1.1"}
+  environ               {:mvn/version "1.1.0"}
+  hiccup                {:mvn/version "1.0.5"}
+  org.flatland/ordered  {:mvn/version "1.5.7"}}}

OSX/macos_release.clj

+(ns macos-release
+  (:require [colorize.core :as colorize]
+            [flatland.ordered.map :as ordered-map]
+            [macos-release
+             [build :as build]
+             [codesign :as codesign]
+             [common :as c]
+             [create-dmg :as create-dmg]
+             [notarize :as notarize]
+             [sparkle-artifacts :as sparkle-artifacts]
+             [upload :as upload]]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private steps*
+  (ordered-map/ordered-map
+   :build                      build/build!
+   :codesign                   codesign/codesign!
+   :generate-sparkle-artifacts sparkle-artifacts/generate-sparkle-artifacts!
+   :create-dmg                 create-dmg/create-dmg!
+   :notarize                   notarize/notarize!
+   :upload                     upload/upload!))
+
+(defn- do-step! [step-name]
+  (let [thunk (or (get steps* (keyword step-name))
+                  (throw (ex-info (format "Invalid step name: %s" step-name)
+                           {:found (set (keys steps*))})))]
+    (println (colorize/magenta (format "Running step %s..." step-name)))
+    (thunk)))
+
+(defn- do-steps! [steps]
+  (c/announce "Running steps: %s" steps)
+  (doseq [step-name steps]
+    (do-step! step-name))
+  (c/announce "Success."))
+
+(defn -main [& steps]
+  (let [steps (or (seq steps)
+                  (keys steps*))]
+    (try
+      (do-steps! steps)
+      (catch Throwable e
+        (println (colorize/red (pr-str e)))
+        (System/exit -1))))
+  (System/exit 0))

OSX/macos_release/addShortcut.scpt

+tell application "Finder"
+  tell disk "Metabase"
+    open
+    set current view of container window to icon view
+    set toolbar visible of container window to false
+    set statusbar visible of container window to false
+    set the bounds of container window to {400, 100, 885, 430}
+    set theViewOptions to the icon view options of container window
+    set arrangement of theViewOptions to not arranged
+    set icon size of theViewOptions to 72
+    make new alias file at container window to POSIX file "/Applications" with properties {name:"Applications"}
+    set position of item "Metabase.app" of container window to {100, 100}
+    set position of item "Applications" of container window to {375, 100}
+    update without registering applications
+    delay 5
+    close
+  end tell
+end tell

OSX/macos_release/build.clj

+(ns macos-release.build
+  (:require [clojure.string :as str]
+            [macos-release.common :as c]))
+
+(def ^String info-plist-file
+  (c/assert-file-exists (str c/macos-source-dir "/Metabase/Metabase-Info.plist")))
+
+(def ^String export-options-plist-file
+  (c/assert-file-exists (str c/macos-source-dir "/exportOptions.plist")))
+
+(def ^String xcode-project-file
+  (c/assert-file-exists (str c/macos-source-dir "/Metabase.xcodeproj")))
+
+(defn- plist-buddy
+  "Run a `PlistBuddy` command."
+  [plist-file command]
+  (let [[out] (c/sh (c/assert-file-exists "/usr/libexec/PlistBuddy")
+                    "-c" (str command)
+                    (c/assert-file-exists plist-file))]
+    (some-> out str/trim)))
+
+(defn- plist-value
+  "Fetch value `k` from a Plist file.
+
+    (plist-value config/info-plist-file \"CFBundleVersion\") ; -> \"0.34.2.0\""
+  [plist-filename k]
+  (plist-buddy plist-filename (format "Print %s" (str k))))
+
+(defn- set-plist-value!
+  "Set value of `k` in a Plist file. Verifies version is set correctly."
+  [plist-filename k v]
+  (plist-buddy plist-filename (format "Set %s %s" (str k) (str v)))
+  (assert (= (plist-value plist-filename k) v))
+  v)
+
+(defn- xcode-build [& args]
+  (apply c/sh "xcodebuild" "-UseNewBuildSystem=NO" args))
+
+(defn- set-version! []
+  (c/step (format "Bump version from %s -> %s" (plist-value info-plist-file "CFBundleVersion") (c/version))
+    (set-plist-value! info-plist-file "CFBundleVersion" (c/version))
+    (set-plist-value! info-plist-file "CFBundleShortVersionString" (c/version))))
+
+(defn- clean! []
+  (c/step "Clean XCode build artifacts"
+    (xcode-build "-project" xcode-project-file "clean")
+    (c/delete-file! c/artifacts-directory)))
+
+(defn- build-xcarchive! []
+  (let [filename (c/artifact "Metabase.xcarchive")]
+    (c/delete-file! filename)
+    (c/step (format "Build %s" filename)
+      (xcode-build "-project"       xcode-project-file
+                   "-scheme"        "Metabase"
+                   "-configuration" "Release"
+                   "-archivePath"   filename
+                   "archive")
+      (c/assert-file-exists filename))))
+
+(defn- build-app! []
+  (let [filename (c/artifact "Metabase.app")]
+    (c/delete-file! filename)
+    (c/step (format "Create %s" filename)
+      (xcode-build "-exportArchive"
+                   "-exportOptionsPlist" export-options-plist-file
+                   "-archivePath"        (c/assert-file-exists (c/artifact "Metabase.xcarchive"))
+                   "-exportPath"         c/artifacts-directory)
+      (c/assert-file-exists filename))))
+
+(defn build! []
+  (c/step "Build artifacts"
+    (c/assert-file-exists (str c/macos-source-dir "/Metabase/jre/bin/java")
+                          "Make sure you copy the JRE it before building Mac App (see build instructions)")
+    (set-version!)
+    (clean!)
+    (build-xcarchive!)
+    (build-app!)
+    (c/announce "Metabase.app built sucessfully.")))

OSX/macos_release/codesign.clj

+(ns macos-release.codesign
+  (:require [clojure.string :as str]
+            [macos-release.common :as c]))
+
+(def ^:private codesigning-identity "Developer ID Application: Metabase, Inc")
+
+(defn- entitlements-file []
+  (c/assert-file-exists (str c/macos-source-dir "/Metabase/Metabase.entitlements")))
+
+(defn- codesign-file! [filename]
+  (c/step (format "Code sign %s" filename)
+    (c/sh "codesign" "--force" "--verify"
+          #_"-vvv"
+          "--sign" codesigning-identity
+          "-r=designated => anchor trusted"
+          "--timestamp"
+          "--options" "runtime"
+          "--entitlements" (entitlements-file)
+          "--deep"
+          (c/assert-file-exists filename))
+    (c/announce "Codesigned %s." filename)))
+
+(defn verify-codesign [filename]
+  (c/step (format "Verify code signature of %s" filename)
+    (c/sh "codesign" "--verify" "--deep"
+          "--display"
+          "--strict"
+          #_"--verbose=4"
+          (c/assert-file-exists filename))
+    ;; double
+    (c/step "Check codesigning status with the System Policy Security Tool"
+      (c/sh "spctl" "--assess"
+            #_"--verbose=4"
+            "--type" "exec"
+            filename))
+    (when (str/ends-with? filename "Metabase.app")
+      (doseq [file ["/Contents/MacOS/Metabase"
+                    "/Contents/Frameworks/Sparkle.framework/Versions/Current/Resources/Autoupdate.app"
+                    "/Contents/Frameworks/Sparkle.framework/Versions/Current/Resources/Autoupdate.app/Contents/MacOS/Autoupdate"
+                    "/Contents/Frameworks/Sparkle.framework/Versions/A/Resources/Autoupdate.app/Contents/MacOS/Autoupdate"]]
+        (verify-codesign (str filename file))))
+    (c/announce "Codesign for %s is valid." filename)))
+
+(defn codesign! []
+  (c/step "Codesign"
+    (let [app         (c/assert-file-exists (c/artifact "Metabase.app"))
+          sparkle-app (c/assert-file-exists (str app "/Contents/Frameworks/Sparkle.framework/Versions/A/Resources/AutoUpdate.app"))]
+      (doseq [file [sparkle-app app]]
+        (codesign-file! file)
+        (verify-codesign file)))))

OSX/macos_release/common.clj

+(ns macos-release.common
+  (:require [clojure.string :as str]
+            [colorize.core :as colorize]
+            [environ.core :as env])
+  (:import [java.io BufferedReader File InputStreamReader]
+           org.apache.commons.io.FileUtils))
+
+(def ^String macos-source-dir
+  "e.g. /Users/cam/metabase/OSX"
+  (env/env :user-dir))
+
+(assert (str/ends-with? macos-source-dir "/OSX")
+  "Please switch to the /OSX directory before running macos_release.clj")
+
+(def ^String root-directory
+  "e.g. /Users/cam/metabase"
+  (.getParent (File. macos-source-dir)))
+
+(def ^String artifacts-directory
+  "e.g. /Users/cam/metabase/osx-artifacts"
+  (str root-directory "/osx-artifacts"))
+
+;;; ---------------------------------------------------- Util Fns ----------------------------------------------------
+
+(def ^:dynamic *steps* [])
+
+(def ^:private step-indent (str/join (repeat 2 \space)))
+
+(defn- steps-indent []
+  (str/join (repeat (count *steps*) step-indent)))
+
+(defn safe-println [& args]
+  (locking println
+    (print (steps-indent))
+    (apply println args)))
+
+(defn announce
+  "Like `println` + `format`, but outputs text in green. Use this for printing messages such as when starting build
+  steps."
+  ([s]
+   (safe-println (colorize/magenta s)))
+
+  ([format-string & args]
+   (announce (apply format (str format-string) args))))
+
+(defn do-step [step thunk]
+  (safe-println (colorize/green (str step)))
+  (binding [*steps* (conj *steps* step)]
+    (try
+      (thunk)
+      (catch Throwable e
+        (throw (ex-info (str step) {} e))))))
+
+(defmacro step {:style/indent 1} [step & body]
+  `(do-step ~step (fn [] ~@body)))
+
+(defn exists? [^String filename]
+  (when filename
+    (.exists (File. filename))))
+
+(defn assert-file-exists
+  "If file with `filename` exists, return `filename` as is; otherwise, throw Exception."
+  ^String [filename & [message]]
+  (when-not (exists? filename)
+    (throw (ex-info (format "File %s does not exist. %s" (pr-str filename) (or message "")) {:filename filename})))
+  (str filename))
+
+(defn create-directory-unless-exists! [^String dir]
+  (when-not (exists? dir)
+    (step (format "Creating directory %s..." dir)
+      (.mkdirs (File. dir))))
+  dir)
+
+(defn artifact
+  "Return the full path of a file in the build artifacts directory."
+  ^String [filename]
+  (create-directory-unless-exists! artifacts-directory)
+  (str artifacts-directory "/" filename))
+
+(defn delete-file!
+  "Delete a file or directory if it exists."
+  ([^String filename]
+   (step (format "Deleting %s..." filename)
+     (if (exists? filename)
+       (let [file (File. filename)]
+         (if (.isDirectory file)
+           (FileUtils/deleteDirectory file)
+           (.delete file))
+         (safe-println (format "Deleted %s." filename)))
+       (safe-println (format "Don't need to delete %s, file does not exist." filename)))
+     (assert (not (exists? filename)))))
+
+  ([file & more]
+   (dorun (map delete-file! (cons file more)))))
+
+(declare sh)
+
+(defn copy-file! [^String source ^String dest]
+  (let [source-file (File. (assert-file-exists source))
+        dest-file   (File. dest)]
+    ;; Use native `cp` rather than FileUtils or the like because codesigning is broken when you use those because they
+    ;; don't preserve symlinks or something like that.
+    (if (.isDirectory source-file)
+      (step (format "Copying directory %s -> %s" source dest)
+        (sh "cp" "-R" source dest))
+      (step (format "Copying file %s -> %s" source dest)
+        (sh "cp" source dest))))
+  (assert-file-exists dest))
+
+(defn- read-lines [^java.io.BufferedReader reader {:keys [quiet? err?]}]
+  (loop [lines []]
+    (if-let [line (.readLine reader)]
+      (do
+        (when-not quiet?
+          (safe-println (if err? (colorize/red line) line)))
+        (recur (conj lines line)))
+      lines)))
+
+(defn- deref-with-timeout [dereffable timeout-ms]
+  (let [result (deref dereffable timeout-ms ::timed-out)]
+    (when (= result ::timed-out)
+      (throw (ex-info (format "Timed out after %d ms." timeout-ms) {})))
+    result))
+
+(def ^:private command-timeout-ms (* 5 60 1000)) ; 5 minutes
+
+(defn sh*
+  "Run a shell command. Like `clojure.java.shell/sh`, but prints output to stdout/stderr and returns results as a vector
+  of lines."
+  {:arglists '([cmd & args] [{:keys [dir quiet?]} cmd & args])}
+  [& args]
+  (step (colorize/blue (str "$ " (str/join " " (map (comp pr-str str) args))))
+    (let [[opts & args] (if (map? (first args))
+                          args
+                          (cons nil args))
+          {:keys [dir]} opts
+          cmd-array     (into-array (map str args))
+          proc          (.exec (Runtime/getRuntime) ^"[Ljava.lang.String;" cmd-array nil ^File (when dir (File. ^String dir)))]
+      (with-open [out-reader (BufferedReader. (InputStreamReader. (.getInputStream proc)))
+                  err-reader (BufferedReader. (InputStreamReader. (.getErrorStream proc)))]
+        (let [exit-code (future (.waitFor proc))
+              out       (future (read-lines out-reader opts))
+              err       (future (read-lines err-reader (assoc opts :err? true)))]
+          {:exit (deref-with-timeout exit-code command-timeout-ms)
+           :out  (deref-with-timeout out command-timeout-ms)
+           :err  (deref-with-timeout err command-timeout-ms)})))))
+
+(defn sh
+  "Run a shell command, returning its output if it returns zero or throwning an Exception if it returns non-zero."
+  {:arglists '([cmd & args] [{:keys [dir quiet?]} cmd & args])}
+  [& args]
+  (let [{:keys [exit out err], :as response} (apply sh* args)]
+    (if (zero? exit)
+      (concat out err)
+      (throw (ex-info (str/join "\n" (concat out err)) response)))))
+
+(defn- version* []
+  (let [[out]       (sh (assert-file-exists (str root-directory "/bin/version")))
+        [_ version] (re-find #"^v([\d.]+)" out)]
+    (when-not (seq version)
+      (throw (ex-info "Error parsing version." {:out out})))
+    version))
+
+(def ^{:arglists '([])} version
+  "Currently tagged Metabase version. e.g. `0.34.3`"
+  (partial deref (delay (version*))))
+
+(defn uploaded-artifact-url [artifact]
+  (format "https://downloads.metabase.com/v%s/%s" (version) artifact))

OSX/macos_release/create_dmg.clj

+(ns macos-release.create-dmg
+  (:require [macos-release
+             [codesign :as codesign]
+             [common :as c]]))
+
+(def ^:private dmg         (c/artifact "Metabase.dmg"))
+(def ^:private temp-dmg    "/tmp/Metabase.dmg")
+(def ^:private source-dir  "/tmp/Metabase.dmg.source")
+(def ^:private mounted-dmg "/Volumes/Metabase")
+
+(defn- copy-app-to-source-dir! []
+  (c/step "Copy app to source dir"
+    (c/delete-file! source-dir)
+    (c/create-directory-unless-exists! source-dir)
+    (let [source-app (c/assert-file-exists (c/artifact "Metabase.app"))
+          dest-app   (str source-dir "/Metabase.app")]
+      (c/copy-file! source-app dest-app)
+      (c/assert-file-exists dest-app)
+      (codesign/verify-codesign dest-app))))
+
+(defn- create-dmg-from-source-dir! []
+  (c/delete-file! temp-dmg)
+  (c/step (format "Create DMG %s from source dir %s" temp-dmg source-dir)
+    (c/sh "hdiutil"    "create"
+          "-srcfolder" (str (c/assert-file-exists source-dir) "/")
+          "-volname"  "Metabase"
+          "-fs"  "HFS+"
+          "-fsargs"  "-c c=64,a=16,e=16"
+          "-format"  "UDRW"
+          ;; has to be big enough to hold everything uncompressed, but doesn't matter if there's extra
+          ;; space -- compression slims it down
+          "-size" "512MB"
+          temp-dmg)
+    (c/announce "Created %s." temp-dmg)))
+
+(defn- mount-dmg! [dmg {:keys [readonly?]
+                        :or   {readonly? false}}]
+  (c/step (format "Mount %s -> %s" (c/assert-file-exists dmg) mounted-dmg)
+    (let [[out]      (c/sh "hdiutil" "attach"
+                           (if readonly? "-readonly" "-readwrite")
+                           "-noverify"
+                           "-noautoopen" dmg)
+          [_ device] (re-find #"(/dev/disk\d+)" out)]
+      device)))
+
+(defn- unmount-dmg! [device]
+  (c/step (format "Unmount device %s" device)
+    (letfn [(unmount! []
+              ;; force completion of any pending disk writes
+              (c/sh "sync")
+              (c/sh "sync")
+              (c/sh "hdiutil" "detach" device))]
+      (try
+        (unmount!)
+        (catch Throwable _
+          ;; if the unmount fails at first because the device is "busy" wait a few seconds and try again
+          (c/announce "Wait a bit for DMG to stop being 'busy'")
+          (Thread/sleep 5000)
+          (unmount!))))))
+
+(defn- do-with-mounted-dmg [dmg options f]
+  (c/step (format "Mount %s" dmg)
+    (let [device (mount-dmg! dmg options)]
+      (try
+        (f device)
+        (finally
+          (unmount-dmg! device))))))
+
+(defmacro ^:private with-mounted-dmg [[device-binding dmg options] & body]
+  `(do-with-mounted-dmg ~dmg ~options (fn [~device-binding] ~@body)))
+
+(defn- add-applications-shortcut! []
+  (c/assert-file-exists mounted-dmg)
+  (c/sh "osascript" (c/assert-file-exists (str c/macos-source-dir "/macos_release/addShortcut.scpt"))))
+
+(defn- delete-temporary-files-in-dmg!
+  "Delete any temporary files that might have creeped in."
+  []
+  (c/assert-file-exists mounted-dmg)
+  (c/delete-file! (str mounted-dmg "/.Trashes")
+                  (str mounted-dmg "/.fseventsd")))
+
+(defn- set-dmg-permissions! []
+  (c/sh "chmod" "-Rf" "go-w" (c/assert-file-exists mounted-dmg)))
+
+(defn- verify-dmg-codesign! []
+  (codesign/verify-codesign (str mounted-dmg "/Metabase.app")))
+
+(defn- compress-and-copy-dmg!
+  []
+  (c/delete-file! dmg)
+  (c/step (format "Compress DMG %s -> %s" (c/assert-file-exists temp-dmg) dmg)
+    (c/sh "hdiutil" "convert" temp-dmg
+          "-format" "UDZO"
+          "-imagekey" "zlib-level-9"
+          "-o" dmg)
+    (c/assert-file-exists dmg)))
+
+(defn- delete-temp-files! []
+  (c/step "Delete temp files"
+    (c/delete-file! temp-dmg source-dir)))
+
+(defn create-dmg! []
+  (c/step (format "Create %s" dmg)
+    (c/delete-file! dmg temp-dmg source-dir)
+    (copy-app-to-source-dir!)
+    (create-dmg-from-source-dir!)
+    (with-mounted-dmg [_ temp-dmg]
+      (add-applications-shortcut!)
+      (delete-temporary-files-in-dmg!)
+      (set-dmg-permissions!)
+      (verify-dmg-codesign!))
+    (compress-and-copy-dmg!)
+    (delete-temp-files!)
+    (with-mounted-dmg [_ dmg {:readonly? true}]
+      (verify-dmg-codesign!))
+    (c/announce "Successfully created %s." dmg)))

OSX/macos_release/notarize.clj

+(ns macos-release.notarize
+  (:require [cheshire.core :as json]
+            [clj-http.client :as http]
+            [clojure.string :as str]
+            [environ.core :as env]
+            [macos-release.common :as c]))
+
+(def ^:private asc-provider "BR27ZJK7WW")
+
+(defn- apple-id []
+  (or (env/env :metabase-mac-app-build-apple-id)
+      (throw (ex-info "Please set the METABASE_MAC_APP_BUILD_APPLE_ID env var." {}))))
+
+(def ^:private keychain-password "@keychain:METABASE_MAC_APP_BUILD_PASSWORD")
+
+(defn- notarize-file!
+  "Returns request UUID."
+  [filename]
+  (c/step (format "Notarize %s" (c/assert-file-exists filename))
+    (let [lines (c/sh "xcrun" "altool" "--notarize-app"
+                      "--primary-bundle-id" "com.metabase.Metabase"
+                      "--username" (apple-id)
+                      "--password" keychain-password
+                      "--asc-provider" asc-provider
+                      "--file" filename)]
+      (some (fn [line]
+              (when-let [[_ uuid] (re-matches #"RequestUUID = ([\w-]+)" line)]
+                uuid))
+            lines))))
+
+(defn- notarization-status
+  "Returns a map with string keys like `LogFileURL` and `Status`."
+  [uuid]
+  (reduce (fn [m line]
+            (if-let [[_ k v] (re-matches #"^(.+):\s+(.+)$" line)]
+              (assoc m (str/trim k) (str/trim v))
+              m))
+          {}
+          (c/sh "xcrun" "altool" "--notarization-info" uuid
+                "-u" (apple-id)
+                "-p" keychain-password
+                "-asc-provider" asc-provider)))
+
+(def ^:private notarization-timeout-ms (* 5 60 1000)) ; five minutes
+
+(defn notarization-log-info
+  "Comes back as a map."
+  [url]
+  (-> (http/get url)
+      :body
+      (json/parse-string true)))
+
+(defn- wait-for-notarization [uuid]
+  (c/step (format "Wait for notarization for %s" uuid)
+    (let [start-time (System/currentTimeMillis)]
+      (loop []
+        (let [duration (- (System/currentTimeMillis) start-time)]
+          (when (> duration notarization-timeout-ms)
+            (throw (ex-info "Notarization timed out." {}))))
+        (let [{status "Status", log-url "LogFileURL", :as status-map} (notarization-status uuid)]
+          (condp = status
+            "in progress"
+            (do (Thread/sleep 5000)
+                (recur))
+
+            "success"
+            (c/announce "Notarization successful.")
+
+            (let [error-info (try
+                               (some-> log-url notarization-log-info)
+                               (catch Throwable e
+                                 (locking println (println "Error fetching log info:" e))
+                                 nil))]
+              (try
+                (some->> log-url (c/sh "open"))
+                (catch Throwable _))
+              (throw (ex-info "Notarization error."
+                       {:uuid     uuid
+                        :status   status-map
+                        :log-info error-info})))))))))
+
+(defn- staple-notarization! [filename]
+  (c/step (format "Staple notarization to %s" (c/assert-file-exists filename))
+    (c/sh "xcrun" "stapler" "staple" "-v" filename)
+    (c/announce "Notarization stapled successfully.")))
+
+(defn- verify-notarization
+  "Verify that an app is Signed & Notarized correctly. See https://help.apple.com/xcode/mac/current/#/dev1cc22a95c"
+  [filename]
+  (c/step (format "Verify notarization for %s" (c/assert-file-exists filename))
+    (let [source (some (fn [line]
+                         (when-let [[_ source] (re-matches #"^source=(.+)$" line)]
+                           (str/trim source)))
+                       (c/sh "spctl" "-a" "-v" filename))]
+      (assert (= source "Notarized Developer ID") (format "Unexpected source: %s" (pr-str source))))
+    (c/announce "Verification successful.")))
+
+(defn notarize! []
+  (c/step "Notarize"
+    (let [dmg-request-uuid (notarize-file! (c/artifact "Metabase.dmg"))
+          zip-request-uuid (notarize-file! (c/artifact "Metabase.zip"))]
+      (wait-for-notarization dmg-request-uuid)
+      (wait-for-notarization zip-request-uuid))
+    (staple-notarization! (c/artifact "Metabase.dmg"))
+    (verify-notarization (c/artifact "Metabase.app"))))
+
+(defn- notarization-history
+  "Provided primarily for REPL usage."
+  []
+  (c/sh "xcrun" "altool"
+        "--notarization-history" "0"
+        "-u" (apple-id)
+        "-p" keychain-password
+        "--asc-provider" asc-provider))

OSX/macos_release/sparkle_artifacts.clj

+(ns macos-release.sparkle-artifacts
+  (:require [clojure.data.xml :as xml]
+            [clojure.java.io :as io]
+            [clojure.string :as str]
+            [hiccup.core :as h]
+            [macos-release
+             [codesign :as codesign]
+             [common :as c]]
+            [pl.danieljanus.tagsoup :as tagsoup])
+  (:import [java.io File FileOutputStream OutputStreamWriter]
+           java.nio.charset.StandardCharsets))
+
+(def ^:private ^String appcast-file       (c/artifact "appcast.xml"))
+(def ^:private ^String release-notes-file (c/artifact "release-notes.html"))
+(def ^:private ^String zip-file           (c/artifact "Metabase.zip"))
+
+(defn- verify-zip-codesign []
+  (c/step (format "Verify code signature of Metabase.app archived in %s" zip-file)
+    (let [temp-file "/tmp/Metabase.zip"]
+      (c/delete-file! temp-file)
+      (c/sh {:quiet? true}
+            "unzip" (c/assert-file-exists zip-file)
+            "-d"    temp-file)
+      (c/assert-file-exists temp-file)
+      (let [unzipped-app-file (c/assert-file-exists (str temp-file "/Metabase.app"))]
+        (codesign/verify-codesign unzipped-app-file)))))
+
+(defn- create-zip-archive! []
+  (c/delete-file! zip-file)
+  (c/step (format "Create ZIP file %s" zip-file)
+    (c/assert-file-exists (c/artifact "Metabase.app"))
+    ;; Use ditto instead of zip to preserve the codesigning -- see https://forums.developer.apple.com/thread/116831
+    (c/sh {:dir c/artifacts-directory}
+          "ditto" "-c" "-k" "--sequesterRsrc"
+          "--keepParent" "Metabase.app" "Metabase.zip")
+    (c/assert-file-exists zip-file)
+    (verify-zip-codesign)))
+
+(defn- generate-file-signature [filename]
+  (c/step (format "Generate signature for %s" filename)
+    (let [private-key (c/assert-file-exists (str c/macos-source-dir "/dsa_priv.pem"))
+          script      (c/assert-file-exists (str c/root-directory "/bin/lib/sign_update.rb"))
+          [out]       (c/sh script (c/assert-file-exists filename) private-key)
+          signature   (str/trim out)]
+      (assert (seq signature))
+      signature)))
+
+(defn- handle-namespaced-keyword [k]
+  (if (namespace k)
+    (str (namespace k) ":" (name k))
+    k))
+
+(defn- xml [form]
+  (if (and (sequential? form)
+           (keyword? (first form)))
+    (let [[element & more] form
+          [attrs & body]   (if (map? (first more))
+                             more
+                             (cons {} more))
+          attrs            (into {} (for [[k v] attrs]
+                                      [(handle-namespaced-keyword k) v]))
+          element          (handle-namespaced-keyword element)]
+      (apply xml/element element attrs (map xml body)))
+    form))
+
+(defn- appcast-xml
+  ([]
+   (appcast-xml (.length (File. (c/assert-file-exists zip-file)))
+                (generate-file-signature zip-file)))
+
+  ([length signature]
+   (xml
+    [:rss
+     {:version       "2.0"
+      :xmlns/sparkle "http://www.andymatuschak.org/xml-namespaces/sparkle"
+      :xmlns/dc      "http://purl.org/dc/elements/1.1/"}
+     [:channel
+      [:title "Metabase ChangeLog"]
+      [:link (c/uploaded-artifact-url "appcast.xml")]
+      [:language "en"]
+      [:item
+       [:title (format "Version %s" (c/version))]
+       [:sparkle/releaseNotesLink (c/uploaded-artifact-url "release-notes.html")]
+       [:enclosure
+        {:url                  (c/uploaded-artifact-url "Metabase.zip")
+         :sparkle/version      (c/version)
+         :length               length
+         :type                 "application/octet-stream"
+         :sparkle/dsaSignature signature}]]]])))
+
+(defn- generate-appcast! []
+  (c/delete-file! appcast-file)
+  (c/step (format "Generate appcast %s" appcast-file)
+    (with-open [os (FileOutputStream. (File. appcast-file))
+                w  (OutputStreamWriter. os StandardCharsets/UTF_8)]
+      (xml/indent (appcast-xml) w))
+    (c/assert-file-exists appcast-file)))
+
+(defn- release-notes-body []
+  (let [url (format "https://github.com/metabase/metabase/releases/tag/v%s" (c/version))]
+    (try
+      (letfn [(find-body [x]
+                (when (sequential? x)
+                  (let [[element {klass :class} & body] x]
+                    (if (and (= element :div)
+                             (= klass "markdown-body"))
+                      x
+                      (some find-body body)))))]
+        (->> (tagsoup/parse url)
+             find-body))
+      (catch Throwable e
+        (throw (ex-info (format "Error parsing release notes at %s: are you sure they exists?" url) {} e))))))
+
+(defn- release-notes []
+  [:html
+   [:head [:title "Metabase Release Notes"]]
+   [:body (release-notes-body)]])
+
+(defn- generate-release-notes! []
+  (c/delete-file! release-notes-file)
+  (c/step (format "Generate release notes %s" release-notes-file)
+    (let [notes (release-notes)]
+      (with-open [w (io/writer release-notes-file)]
+        (.write w (h/html notes))))))
+
+(defn generate-sparkle-artifacts! []
+  (c/step "Generate Sparkle artifacts"
+    (create-zip-archive!)
+    (generate-appcast!)
+    (generate-release-notes!)
+    (c/announce "Sparkle artifacts generated successfully.")))

OSX/macos_release/upload.clj

+(ns macos-release.upload
+  (:require [macos-release.common :as c]))
+
+(def ^:private aws-profile "metabase")
+(def ^:private s3-bucket "downloads.metabase.com")
+(def ^:private cloudfront-distribution-id "E35CJLWZIZVG7K")
+
+(def ^:private upload-dir (c/artifact "upload"))
+
+(defn- copy-files-to-upload-dir! []
+  (c/delete-file! upload-dir)
+  (c/step (format "Copy files to %s" upload-dir)
+    (c/step "Copy top-level files"
+      (c/create-directory-unless-exists! upload-dir)
+      (c/copy-file! (c/assert-file-exists (c/artifact "appcast.xml")) (str upload-dir "/appcast.xml")))
+    (let [version-upload-dir (str upload-dir "/v" (c/version))]
+      (c/step (format "Copy files to %s" version-upload-dir)
+        (c/create-directory-unless-exists! version-upload-dir)
+        (doseq [file ["Metabase.zip" "Metabase.dmg" "release-notes.html"]]
+          (c/copy-file! (c/assert-file-exists (c/artifact file)) (str version-upload-dir "/" file)))))))
+
+(defn- upload-artifacts! []
+  (c/step "Upload artifacts to https://downloads.metabase.com"
+    (c/sh "aws" "--recursive"
+          "--profile" aws-profile
+          "--region" "us-east-1"
+          "s3" "cp" upload-dir
+          (format "s3://%s" s3-bucket))
+    (c/announce "All files uploaded.")))
+
+(defn- create-cloudfront-invalidation! []
+  (c/step "Create CloudFront invalidation"
+    (c/step "Enable CloudFront CLI"
+      (c/sh "aws" "configure" "set" "preview.cloudfront" "true"))
+    (c/step "Invalidate /appcast.xml"
+      (c/sh "aws" "cloudfront" "create-invalidation"
+            "--profile" aws-profile
+            "--distribution-id" cloudfront-distribution-id
+            "--paths" "/appcast.xml"))
+    (c/announce "Invalidation created successfully.")))
+
+(defn upload! []
+  (c/step "Upload artifacts"
+    (copy-files-to-upload-dir!)
+    (upload-artifacts!)
+    (create-cloudfront-invalidation!)))

bin/Metabase/Util.pm

-use strict;
-use warnings;
-
-package Metabase::Util;
-
-use Cwd 'getcwd';
-use Exporter;
-use JSON;
-use Readonly;
-use Term::ANSIColor qw(:constants);
-
-our @ISA = qw(Exporter);
-our @EXPORT = qw(config
-                 config_or_die
-                 announce
-                 print_giant_success_banner
-                 get_file_or_die
-                 plist_buddy_exec
-                 OSX_ARTIFACTS_DIR
-                 artifact);
-
-Readonly my $config_file => getcwd() . '/bin/config.json';
-warn "Missing config file: $config_file\n" .
-     "Please copy $config_file.template, and edit it as needed.\n"
-     unless (-e $config_file);
-Readonly my $config => from_json(`cat $config_file`) if -e $config_file;
-
-sub config {
-    return $config ? $config->{ $_[0] } : '';
-}
-
-sub config_or_die {
-    my ($configKey) = @_;
-    return config($configKey) or die "Missing config.json property '$configKey'";
-}
-
-sub announce {
-    print "\n\n" . GREEN . $_[0] . RESET . "\n\n";
-}
-
-sub print_giant_success_banner {
-    print "\n\n". BLUE .
-        '+----------------------------------------------------------------------+' . "\n" .
-        '|                                                                      |' . "\n" .
-        '|   _______           _______  _______  _______  _______  _______  _   |' . "\n" .
-        '|  (  ____ \|\     /|(  ____ \(  ____ \(  ____ \(  ____ \(  ____ \( )  |' . "\n" .
-        '|  | (    \/| )   ( || (    \/| (    \/| (    \/| (    \/| (    \/| |  |' . "\n" .
-        '|  | (_____ | |   | || |      | |      | (__    | (_____ | (_____ | |  |' . "\n" .
-        '|  (_____  )| |   | || |      | |      |  __)   (_____  )(_____  )| |  |' . "\n" .
-        '|        ) || |   | || |      | |      | (            ) |      ) |(_)  |' . "\n" .
-        '|  /\____) || (___) || (____/\| (____/\| (____/\/\____) |/\____) | _   |' . "\n" .
-        '|  \_______)(_______)(_______/(_______/(_______/\_______)\_______)(_)  |' . "\n" .
-        '|                                                                      |' . "\n" .
-        '|                                                                      |' . "\n" .
-        '+----------------------------------------------------------------------+' . RESET . "\n\n";
-}
-
-# Check if a file exists, or die.
-# If file path is relative, qualify it by prepending the current dir.
-# Return the fully-qualified file path.
-sub get_file_or_die {
-    my ($filename) = @_;
-    $filename = (getcwd() . '/' . $filename) if $filename !~ /^\//;
-
-    die "Error: $filename does not exist.\n" unless -e $filename;
-
-    return $filename;
-}
-
-# Run a PlistBuddy command against Metabase-Info.plist
-sub plist_buddy_exec {
-    my $info_plist = get_file_or_die('OSX/Metabase/Metabase-Info.plist');
-    return `/usr/libexec/PlistBuddy -c '@_' "$info_plist"`;
-}
-
-use constant OSX_ARTIFACTS_DIR => getcwd() . '/osx-artifacts';
-sub artifact {
-    # Make the artifacts directory if needed
-    system('mkdir', OSX_ARTIFACTS_DIR) if ! -d OSX_ARTIFACTS_DIR;
-
-    return OSX_ARTIFACTS_DIR . "/$_[0]";
-}
-
-1;

bin/config.json

-{
-  "codesigningIdentity": "Developer ID Application: Metabase, Inc",
-  "appStoreConnectProviderShortName": "BR27ZJK7WW",
-  "awsProfile": "metabase",
-  "awsBucket": "downloads.metabase.com",
-  "cloudFrontDistributionID": "E35CJLWZIZVG7K"
-}

bin/osx-release

-#! /usr/bin/env perl -I./bin
-
-use strict;
-use warnings;
-
-use Cwd 'getcwd';
-use File::Copy 'copy';
-use File::Copy::Recursive 'rcopy';   # CPAN
-use File::Path 'remove_tree';
-use File::stat 'stat';
-use Readonly;                        # CPAN
-use String::Util 'trim';             # CPAN
-use Text::Caml;                      # CPAN
-
-use Metabase::Util;
-
-Readonly my $app             => artifact('Metabase.app');
-Readonly my $zipfile         => artifact('Metabase.zip');
-Readonly my $appcast         => artifact('appcast.xml');
-Readonly my $release_notes   => artifact('release-notes.html');
-Readonly my $dmg             => artifact('Metabase.dmg');
-
-Readonly my $xcode_project   => get_file_or_die('OSX/Metabase.xcodeproj');
-Readonly my $export_options  => get_file_or_die('OSX/exportOptions.plist');
-
-# Get the version saved in the CFBundle, e.g. '0.11.3.1'
-sub version {
-    return trim(plist_buddy_exec('Print', 'CFBundleVersion'));
-}
-
-# Get the tag saved in version.properties, e.g. '0.12.0'
-sub version_from_props_file {
-  open(FILE, get_file_or_die('resources/version.properties')) or die $!;
-  while (<FILE>) { m/^tag/ && s/^tag=v([0-9.]+)[^0-9.]*$/$1/ && (return trim($_)); };
-}
-
-# This is the name of the subdirectory on s3, e.g. 'v.0.12.0'
-sub upload_subdir {
-  return 'v' . version_from_props_file();
-}
-
-# Next version after version(), e.g. '0.11.3.2'
-sub next_version {
-    my ($old_version_tag, $old_version_point_release) = (version() =~ /^(\d+\.\d+\.\d+)\.(\d+)$/);
-
-    Readonly my $tag_from_props_file => version_from_props_file();
-
-    # Now calculate the new version, which is ($tag.$point_release)
-    # Check and see if tag has changed in version.properties; if so, new version is the first "point release" of that tag.
-    return $old_version_tag eq $tag_from_props_file ? ($old_version_tag . '.' . ($old_version_point_release + 1)) : "$tag_from_props_file.0";
-}
-
-sub bump_version {
-    Readonly my $new_version => next_version();
-    announce 'Bumping version: ' . version() . " -> $new_version";
-
-    plist_buddy_exec('Set', ':CFBundleVersion', $new_version);
-    plist_buddy_exec('Set', ':CFBundleShortVersionString', $new_version);
-}
-
-sub clean {
-    system('xcodebuild', '-UseNewBuildSystem=NO', 'clean', '-project', $xcode_project) == 0 or die $!;
-    remove_tree(OSX_ARTIFACTS_DIR);
-}
-
-# Build Metabase.app
-sub build {
-    announce "Building $app...";
-
-    Readonly my $xcarchive => artifact('Metabase.xcarchive');
-
-    # remove old artifacts if they exist
-    remove_tree($xcarchive, $app);
-
-    # Build the project and generate Metabase.xcarchive
-    system('xcodebuild',
-           '-UseNewBuildSystem=NO',
-           '-project', $xcode_project,
-           '-scheme', 'Metabase',
-           '-configuration', 'Release',
-           '-archivePath', $xcarchive,
-           'archive') == 0 or die $!;
-
-    # Ok, now create the Metabase.app artifact
-    system('xcodebuild',
-           '-UseNewBuildSystem=NO',
-           '-exportArchive',
-           '-exportOptionsPlist', $export_options,
-           '-archivePath', $xcarchive,
-           '-exportPath', OSX_ARTIFACTS_DIR) == 0 or die $!;
-
-    # Ok, we can remove the .xcarchive file now
-    remove_tree($xcarchive);
-}
-
-sub codesign_file {
-    my ($filename) = @_;
-
-    Readonly my $codesigning_cert_name => config_or_die('codesigningIdentity');
-    Readonly my $entitlements_file => get_file_or_die('OSX/Metabase/Metabase.entitlements');
-
-    announce "Codesigning $filename...";
-
-    system('codesign', '--force', '--verify',
-           '--sign', $codesigning_cert_name,
-           '-r=designated => anchor trusted',
-           '--timestamp',
-           '--options', 'runtime',
-           '--entitlements', $entitlements_file,
-           '--deep', get_file_or_die($filename)) == 0 or die "Code signing failed: $!\n";
-}
-
-# Codesign Metabase.app
-sub codesign {
-    codesign_file($app) or die $1;
-}
-
-sub verify_file_codesign {
-    my ($filename) = @_;
-    get_file_or_die($filename);
-
-    config_or_die('codesigningIdentity');
-
-    announce "Verifying codesigning for $filename...";
-
-    system('codesign', '--verify', '--deep',
-           '--display',
-           '--strict',
-           '--verbose=4',
-           get_file_or_die($filename)) == 0 or die "Code signing verification failed: $!\n";
-
-    announce "codesign --verify $filename successful";
-
-    # Double-check with System Policy Security tool
-    system('spctl', '--assess', '--verbose=4', get_file_or_die($filename)) == 0 or die "Codesigning verification (spctl) failed: $!\n";
-
-    announce "spctl --assess $filename successful";
-
-}
-
-# Verify that Metabase.app was signed correctly
-sub verify_codesign {
-    verify_file_codesign($app) or die $!;
-}
-
-
-# ------------------------------------------------------------ PACKAGING FOR SPARKLE ------------------------------------------------------------
-
-sub verify_zip_codesign {
-    remove_tree('/tmp/Metabase.zip');
-
-    system('unzip', get_file_or_die($zipfile),
-           '-d', '/tmp/Metabase.zip');
-
-    verify_file_codesign('/tmp/Metabase.zip/Metabase.app') or die $!;
-}
-
-# Create ZIP containing Metabase.app
-sub archive {
-    announce "Creating $zipfile...";
-
-    remove_tree($zipfile);
-
-    get_file_or_die($app);
-
-    # Use ditto instead of zip to preserve the codesigning -- see https://forums.developer.apple.com/thread/116831
-    system('cd ' . OSX_ARTIFACTS_DIR . ' && ditto -c -k --sequesterRsrc --keepParent Metabase.app Metabase.zip') == 0 or die $!;
-    get_file_or_die($zipfile);
-
-    verify_zip_codesign;
-}
-
-sub generate_signature {
-    Readonly my $private_key_file => getcwd() . '/OSX/dsa_priv.pem';
-
-    unless (-e $private_key_file) {
-        warn "Missing private key file: $private_key_file\n";
-        return;
-    }
-
-    Readonly my $sign_update_script => get_file_or_die('bin/lib/sign_update.rb');
-
-    get_file_or_die($zipfile);
-
-    return trim(`$sign_update_script "$zipfile" "$private_key_file"`);
-}
-
-# Generate the appcast.xml RSS feed file that Sparkle reads to check for updates
-sub generate_appcast {
-    announce "Generating $appcast...";
-
-    remove_tree($appcast);
-
-    Readonly my $aws_bucket  => config_or_die('awsBucket');
-    Readonly my $signature   => generate_signature() or die 'Failed to generate appcast signature';
-
-    open(my $out, '>', $appcast) or die "Unable to write to $appcast: $!";
-    print $out Text::Caml->new->render_file(get_file_or_die('bin/templates/appcast.xml.template'), {
-        VERSION   => version(),
-        SIGNATURE => $signature,
-        LENGTH    => stat(get_file_or_die($zipfile))->size,
-        S3_BUCKET => $aws_bucket,
-        S3_SUBDIR => upload_subdir()
-    });
-    close $out;
-}
-
-sub edit_release_notes {
-    remove_tree($release_notes);
-
-    copy(get_file_or_die('bin/templates/release-notes.html.template'), $release_notes) or die $!;
-    system('nano', get_file_or_die($release_notes)) == 0 or die $!;
-}
-
-
-# ------------------------------------------------------------ CREATING DMG ------------------------------------------------------------
-
-sub create_dmg_from_source_dir {
-    my ($source_dir, $dmg_filename) = @_;
-    announce "Creating DMG: $dmg_filename from source $source_dir...";
-
-    system('hdiutil', 'create',
-           '-srcfolder', $source_dir,
-           '-volname', 'Metabase',
-           '-fs', 'HFS+',
-           '-fsargs', '-c c=64,a=16,e=16',
-           '-format', 'UDRW',
-           '-size', '512MB',          # has to be big enough to hold everything uncompressed, but doesn't matter if there's extra space -- compression slims it down
-           $dmg_filename) == 0 or die $!;
-
-    announce "$dmg_filename created.";
-}
-
-# Mount the disk image, return the device name
-sub mount_dmg {
-    my ($dmg_filename) = @_;
-    announce "Mounting DMG...";
-
-    my $device = `hdiutil attach -readwrite -noverify -noautoopen $dmg_filename`;
-    # Find the device name: the part of the output looks like /dev/disk11
-    for my $token (split(/\s+/, $device)) {
-        if ($token =~ m|^/dev/|) {
-            $device = $token;
-            last;
-        }
-    }
-
-    announce "Mounted $dmg_filename at $device";
-    return $device;
-}
-
-sub dmg_add_applications_shortcut {
-    announce "Adding shortcut to /Applications...";
-
-    system('osascript', '-e',
-           'tell application "Finder"
-              tell disk "Metabase"
-                open
-                set current view of container window to icon view
-                set toolbar visible of container window to false
-                set statusbar visible of container window to false
-                set the bounds of container window to {400, 100, 885, 430}
-                set theViewOptions to the icon view options of container window
-                set arrangement of theViewOptions to not arranged
-                set icon size of theViewOptions to 72
-                make new alias file at container window to POSIX file "/Applications" with properties {name:"Applications"}
-                set position of item "Metabase.app" of container window to {100, 100}
-                set position of item "Applications" of container window to {375, 100}
-                update without registering applications
-                delay 5
-                close
-              end tell
-           end tell') == 0 or die $!;
-}
-
-sub finalize_dmg {
-    my ($device, $dmg_filename) = @_;
-    announce "Finalizing DMG...";
-
-    # Remove any hidden files that creeped into the DMG
-    remove_tree('/Volumes/Metabase/.Trashes',
-                '/Volumes/Metabase/.fseventsd');
-
-
-    # Set DMG permissions, force completion of pending disk writes
-    system('chmod', '-Rf', 'go-w', '/Volumes/Metabase') == 0 or warn $!; # this might issue warnings about not being able to affect .Trashes
-    system('sync');
-    system('sync');
-
-    # wait a few seconds for the sync to complete so DMG isn't "busy" when we try to unmount it
-    system('sleep', '5');
-
-    # unmount the temp DMG
-    announce "Unmounting $device...";
-    system('hdiutil', 'detach', $device) == 0 or die $!;
-
-    # compress the DMG
-    announce "Compressing DMG...";
-    system('hdiutil', 'convert', $dmg_filename,
-           '-format', 'UDZO',
-           '-imagekey', 'zlib-level-9',
-           '-o', $dmg) == 0 or die $!;
-}
-
-sub create_dmg {
-    announce "Preparing DMG files...";
-
-    # detach any existing Metabase DMGs
-    system('hdiutil', 'detach', '/Volumes/Metabase') if -d '/Volumes/Metabase';
-
-    Readonly my $temp_dmg       => artifact('Metabase.temp.dmg');
-    Readonly my $dmg_source_dir => artifact('dmg');
-
-    # Clean up old artifacts
-    remove_tree($dmg_source_dir, $temp_dmg, $dmg);
-    mkdir $dmg_source_dir or die $!;
-
-    # Copy Metabase.app into the source dir
-    rcopy(get_file_or_die($app), $dmg_source_dir . '/Metabase.app') or die $!;
-
-    # Ok, now proceed with the steps to create the DMG
-    create_dmg_from_source_dir($dmg_source_dir, $temp_dmg);
-
-    Readonly my $device => mount_dmg($temp_dmg);
-
-    dmg_add_applications_shortcut;
-
-    finalize_dmg($device, $temp_dmg);
-
-    announce "DMG created successfully: $dmg";
-
-    # Cleanup: remove temp file & temp dir
-    remove_tree($temp_dmg, $dmg_source_dir);
-}
-
-# ------------------------------------------------------------ NOTORIZATION ------------------------------------------------------------
-
-sub getAppleID {
-    return $ENV{'METABASE_MAC_APP_BUILD_APPLE_ID'} or die 'Make sure you export the env var METABASE_MAC_APP_BUILD_APPLE_ID';
-}
-
-sub getAscProvider {
-    return config_or_die('appStoreConnectProviderShortName');
-}
-
-sub notarize_file {
-    my ($filename) = @_;
-
-    announce "Notarizing $filename...";
-
-    Readonly my $appleID     => getAppleID;
-    Readonly my $ascProvider => getAscProvider;
-
-    system('xcrun', 'altool', '--notarize-app',
-           '--primary-bundle-id', 'com.metabase.Metabase',
-           '--username', $appleID,
-           '--password', '@keychain:METABASE_MAC_APP_BUILD_PASSWORD',
-           '--asc-provider', $ascProvider,
-           '--file', $filename
-          ) == 0 or die $!;
-
-    print 'You can keep an eye on the notarization status (and get the LogFileURL) with the command:' . "\n\n";
-    print '    xcrun altool --notarization-info <RequestUUID> -u "$METABASE_MAC_APP_BUILD_APPLE_ID" -p "@keychain:METABASE_MAC_APP_BUILD_PASSWORD"' . "\n\n";
-}
-
-sub wait_for_notarization {
-    announce "Waiting for notarization...";
-
-    Readonly my $appleID     => getAppleID;
-    Readonly my $ascProvider => getAscProvider;
-
-    my $status = `xcrun altool --notarization-history 0 -u "$appleID" -p "\@keychain:METABASE_MAC_APP_BUILD_PASSWORD" --asc-provider $ascProvider` or die $!;
-
-    print "$status\n";
-
-    if ($status =~ m/in progress/) {
-        print "Notarization is still in progress, waiting a few seconds and trying again...\n";
-        sleep 5;
-        wait_for_notarization();
-    } else {
-        announce "Notarization successful.";
-        return "Done";
-    }
-}
-
-sub staple_notorization {
-    my ($filename) = @_;
-
-    announce "Stapling notarization to $filename...";
-
-    system('xcrun', 'stapler', 'staple',
-           '-v', $filename) == 0 or die $!;
-
-    announce "Notarization stapled successfully.";
-}
-
-# Verify that an app is Signed & Notarized correctly. See https://help.apple.com/xcode/mac/current/#/dev1cc22a95c
-sub verify_notarization {
-    # e.g. /Applications/Metabase.app
-    my ($appFile) = @_;
-
-    announce "Verifying that $appFile is notarized correctly...";
-
-    system('spctl', '-a', '-v', $appFile) == 0 or die $!;
-
-    announce "Verification successful.";
-}
-
-
-sub notarize_files {
-    notarize_file(get_file_or_die($zipfile));
-    notarize_file(get_file_or_die($dmg));
-    wait_for_notarization();
-    staple_notorization(get_file_or_die($dmg));
-    verify_notarization(get_file_or_die($app));
-}
-
-
-# ------------------------------------------------------------ UPLOADING ------------------------------------------------------------
-
-
-# Upload artifacts to AWS
-# Make sure to run `aws configure --profile metabase` first to set up your ~/.aws/config file correctly
-sub upload {
-    Readonly my $aws_profile => config_or_die('awsProfile');
-    Readonly my $aws_bucket  => config_or_die('awsBucket');
-
-    # Make a folder that contains the files we want to upload
-    Readonly my $upload_dir => artifact('upload');
-    remove_tree($upload_dir) if -d $upload_dir;
-    mkdir $upload_dir or die $!;
-
-    # appcast.xml goes in the root directory
-    copy(get_file_or_die($appcast), $upload_dir) or die $!;
-
-    # zipfile, release notes, and DMG go in a dir like v0.12.0
-    Readonly my $upload_subdir => $upload_dir . '/' . upload_subdir();
-    mkdir $upload_subdir or die $!;
-
-    for my $file ($zipfile, $release_notes, $dmg) {
-      copy(get_file_or_die($file), $upload_subdir) or die $!;
-    }
-
-    announce "Uploading files to $aws_bucket...";
-    system('aws', '--recursive',
-           '--profile', $aws_profile,
-           '--region', 'us-east-1',
-           's3', 'cp', $upload_dir,
-           "s3://$aws_bucket") == 0 or die "Upload failed: $!\n";
-
-    announce "Upload finished."
-}
-
-sub create_cloudfront_invalidation {
-    announce "Creating CloudFront invalidation...";
-
-    system ('aws', 'configure',
-            'set', 'preview.cloudfront', 'true') == 0 or die $!;
-
-    system ('aws', 'cloudfront', 'create-invalidation',
-            '--profile', config_or_die('awsProfile'),
-            '--distribution-id', config_or_die('cloudFrontDistributionID'),
-            '--paths', '/appcast.xml') == 0 or die $!;
-
-    announce "CloudFront invalidation created successfully.";
-}
-
-
-# ------------------------------------------------------------ RUN ALL ------------------------------------------------------------
-
-sub all {
-    clean;
-    bump_version;
-    build;
-    codesign;
-    verify_codesign;
-    archive;
-    generate_appcast;
-    edit_release_notes;
-    create_dmg;
-    notarize_files;
-    upload;
-    create_cloudfront_invalidation;
-}
-
-# Run all the commands specified in command line args, otherwise default to running 'all'
-@ARGV = ('all') unless @ARGV;
-no strict 'refs';
-map { $_->(); } @ARGV;
-
-print_giant_success_banner();

docs/developers-guide-osx.md

@@ -36,104 +36,90 @@ The following steps need to be done before building the Mac App:
     Assuming the OpenJDK folks have resolved this issue going forward, you are fine to use whatever the latest JRE version available is. I have been using the HotSpot JRE instead of the
     OpenJ9 one but it ultimately shouldn't make a difference.
 
-    
-1.  Copy Metabase uberjar to OSX resources dir
+1)  Copy Metabase uberjar to OSX resources dir
 
     ```bash
     cp /path/to/metabase.jar OSX/Resources/metabase.jar
     ```
-    
+
     Every time you want to build a new version of the Mac App, you can simple update the bundled uberjar the same way.
-        
+
 At this point, you should try opening up the Xcode project and building the Mac App in Xcode by clicking the run button. The app should build and launch at this point. If it doesn't, ask Cam for help!
 
 ### Releasing
 
 The following steps are prereqs for releasing the Mac App:
 
-
 1.  Install XCode command-line tools. In `Xcode` > `Preferences` > `Locations` select your current Xcode version in the `Command Line Tools` drop-down.
 
-1.  Install CPAN modules
-
-    ```bash
-    sudo cpan
-    install force File::Copy::Recursive Readonly String::Util Text::Caml JSON
-    quit
-    ```
-
-    You can install [PerlBrew](https://perlbrew.pl/) if you want to install CPAN modules without having to use `sudo`.
-
-    Normally you shouldn't have to use `install force` to install the modules above, but `File::Copy::Recursive` seems fussy lately and has a failing test that prevents it from installing normally.
-
-1.  Install AWS command-line client (if needed)
+1)  Install AWS command-line client (if needed)
 
     ```bash
     brew install awscli
     ```
-    
-1.  Configure AWS Credentials for `metabase` profile (used to upload artifacts to S3)    
+
+1)  Configure AWS Credentials for `metabase` profile (used to upload artifacts to S3)
 
     You'll need credentials that give you permission to write the metabase-osx-releases S3 bucket.
     You just need the access key ID and secret key; use the defaults for locale and other options.
-    
+
     ```bash
     aws configure --profile metabase
     ```
-    
-1.  Obtain a copy of the private key for signing app updates (ask Cam) and put a copy of it at `OSX/dsa_priv.pem`
+
+1)  Obtain a copy of the private key for signing app updates (ask Cam) and put a copy of it at `OSX/dsa_priv.pem`
 
     ```bash
     cp /path/to/private/key.pem OSX/dsa_priv.pem
     ```
-    
-1.  Add `Apple Developer ID Application Certificate` to your computer's keychain.
+
+1)  Add `Apple Developer ID Application Certificate` to your computer's keychain.
 
     You'll need to generate a Certificate Signing Request from Keychain Access, and have Sameer go to [the Apple Developer Site](https://developer.apple.com/account/mac/certificate/) and generate one for you, then load the file on your computer.
-    
-1.  Export your Apple ID for building the app as `METABASE_MAC_APP_BUILD_APPLE_ID`. (This Apple ID must be part of the Metabase org in the Apple developer site. Ask Cam or Sameer to add you if it isn't.)
-     
+
+1)  Export your Apple ID for building the app as `METABASE_MAC_APP_BUILD_APPLE_ID`. (This Apple ID must be part of the Metabase org in the Apple developer site. Ask Cam or Sameer to add you if it isn't.)
+
     ```bash
     #  Add this to .zshrc or .bashrc
     export METABASE_MAC_APP_BUILD_APPLE_ID=my_email@whatever.com
-    ```        
+    ```
 
-1.  Create an App-Specific password for the Apple ID in the previous step
+1)  Create an App-Specific password for the Apple ID in the previous step
+
+    Go to https://appleid.apple.com/account/manage then `Security` > `App-Specific Passwords` > `Generate Password`
 
-     Go to https://appleid.apple.com/account/manage then `Security` > `App-Specific Passwords` > `Generate Password`
-        
     1.  Store the password in Keychain
-    
+
         ```bash
         xcrun altool \
         --store-password-in-keychain-item "METABASE_MAC_APP_BUILD_PASSWORD" \
         -u "$METABASE_MAC_APP_BUILD_APPLE_ID" \
         -p <secret_password>
         ```
-      
+
+1)  Install Clojure CLI
+
+    ```bash
+    brew install clojure
+    ```
+
 ## Building & Releasing the Mac App
-        
+
 After following the configuration steps above, to build and release the app you can use the `./bin/osx-release` script:
 
 1. Copy latest uberjar to the Mac App build directory
 
-    ```bash
-    cp path/to/metabase.jar OSX/Resources/metabase.jar
-    ```
-    
+   ```bash
+   cp path/to/metabase.jar OSX/Resources/metabase.jar
+   ```
+
 1. Bump version number (`tag`) in `./bin/version.properties` unless uberjar was built locally
 
    The build script reads this file, which is generated by `./bin/build`; it assumes the uberjar was built locally, which is a bad assumption. This is something we should fix
-    
-1. Bundle entire app, and upload to s3
 
-    ```bash
-    ./bin/osx-release
-    ```
-
-## Debugging ./bin/osx-release
+1. Bundle entire app, and upload to s3
 
-*  You can run individual steps of the release script by passing in the appropriate step subroutines. e.g. `./bin/osx-release create_dmg upload`.
-   The entire sequence of different steps can be found at the bottom of `./bin/osx-release`.
-*  Generating the DMG seems to be somewhat finicky, so if it fails with a message like "Device busy" trying the step again a few times usually resolves the issue.
-   You can continue the build process from the DMG creation step by running `./bin/osx-release create_dmg upload`.
+   ```bash
+   cd OSX
+   clojure -m macos-release
+   ```

frontend/src/metabase/meta/Dashboard.js

@@ -206,6 +206,10 @@ export function getParameterMappingOptions(
   card: Card,
 ): ParameterMappingUIOption[] {
   const options = [];
+  if (card.display === "text") {
+    // text cards don't have parameters
+    return [];
+  }
 
   const query = new Question(card, metadata).query();
 

frontend/src/metabase/meta/types/Parameter.js

 /* @flow */
 
 import type { CardId } from "./Card";
+import type { FieldId } from "./Field";
 import type { LocalFieldReference, ForeignFieldReference } from "./Query";
 
 export type ParameterId = string;
@@ -18,6 +19,7 @@ export type Parameter = {
   type: ParameterType,
   slug: string,
   default?: string,
+  field_ids?: FieldId[],
 
   target?: ParameterTarget,
 };

frontend/src/metabase/parameters/components/Parameters.jsx

 /* @flow */
 
 import React, { Component } from "react";
+import { connect } from "react-redux";
 
 import StaticParameterWidget from "./ParameterWidget";
 import Icon from "metabase/components/Icon";
 import { color } from "metabase/lib/colors";
 
+import { getMetadata } from "metabase/selectors/metadata";
+
 import querystring from "querystring";
 import cx from "classnames";
 
@@ -14,9 +17,12 @@ import type {
   ParameterId,
   Parameter,
   ParameterValues,
+  ParameterValueOrArray,
 } from "metabase/meta/types/Parameter";
 
 import type { DashboardWithCards } from "metabase/meta/types/Dashboard";
+import type Field from "metabase-lib/lib/metadata/Field";
+import type Metadata from "metabase-lib/lib/metadata/Metadata";
 
 type Props = {
   className?: string,
@@ -33,6 +39,7 @@ type Props = {
   vertical?: boolean,
   commitImmediately?: boolean,
 
+  metadata?: Metadata,
   query?: QueryParams,
 
   setParameterName?: (parameterId: ParameterId, name: string) => void,
@@ -46,6 +53,7 @@ type Props = {
   setEditingParameter?: (parameterId: ParameterId) => void,
 };
 
+@connect(state => ({ metadata: getMetadata(state) }))
 export default class Parameters extends Component {
   props: Props;
 
@@ -57,13 +65,17 @@ export default class Parameters extends Component {
 
   componentWillMount() {
     // sync parameters from URL query string
-    const { parameters, setParameterValue, query } = this.props;
+    const { parameters, setParameterValue, query, metadata } = this.props;
     if (setParameterValue) {
       for (const parameter of parameters) {
-        if (query && query[parameter.slug] != null) {
-          setParameterValue(parameter.id, query[parameter.slug]);
-        } else if (parameter.default != null) {
-          setParameterValue(parameter.id, parameter.default);
+        const queryParam = query && query[parameter.slug];
+        if (queryParam != null || parameter.default != null) {
+          const value = queryParam != null ? queryParam : parameter.default;
+          const fieldIds = parameter.field_ids || [];
+          // $FlowFixMe
+          const fields = fieldIds.map(id => metadata.field(id));
+          // $FlowFixMe
+          setParameterValue(parameter.id, parseQueryParam(value, fields));
         }
       }
     }
@@ -231,3 +243,22 @@ const SortableParameterWidget = SortableElement(StaticParameterWidget);
 const SortableParameterWidgetList = SortableContainer(
   StaticParameterWidgetList,
 );
+
+export function parseQueryParam(
+  value: ParameterValueOrArray,
+  fields: Field[],
+): any {
+  if (Array.isArray(value)) {
+    return value.map(v => parseQueryParam(v, fields));
+  }
+  // [].every is always true, so only check if there are some fields
+  if (fields.length > 0) {
+    if (fields.every(f => f.isNumeric())) {
+      return parseFloat(value);
+    }
+    if (fields.every(f => f.isBoolean())) {
+      return value === "true" ? true : value === "false" ? false : value;
+    }
+  }
+  return value;
+}