Skip to content
Snippets Groups Projects
Commit 731d0834 authored by Allen Gilliland's avatar Allen Gilliland
Browse files

Merge branch 'master' into release-0.15.0

parents 0311d013 3abf3e91
No related branches found
No related tags found
No related merge requests found
Showing
with 160 additions and 114 deletions
If this is a bug report, please fill in the blanks:
* I am using the _____ browser.
* My computer's OS is _____.
* My database is _____.
* My Metabase version is _____.
Before we can merge your pull request, you'll need to sign our [Contributor License Agreement](https://docs.google.com/a/metabase.com/forms/d/1oV38o7b9ONFSwuzwmERRMi9SYrhYeOrkbmNaq9pOJ_E/viewform)
(unless it's a tiny documentation change). Fill this out when you get a chance, and let us know in the PR! :100:
[![Circle CI](https://circleci.com/gh/metabase/metabase.svg?style=svg&circle-token=3ccf0aa841028af027f2ac9e8df17ce603e90ef9)](https://circleci.com/gh/metabase/metabase)
[![Dependencies Status](https://jarkeeper.com/metabase/metabase/status.png)](https://jarkeeper.com/metabase/metabase)
[![Leiningen Dependencies Status](https://jarkeeper.com/metabase/metabase/status.png)](https://jarkeeper.com/metabase/metabase)
[![NPM Dependencies Status](https://david-dm.org/metabase/metabase.svg)](https://david-dm.org/metabase/metabase)
# Overview
......
......@@ -34,7 +34,7 @@
ring/ring-core]]
[com.draines/postal "1.11.4"] ; SMTP library
[com.google.apis/google-api-services-bigquery ; Google BigQuery Java Client Library
"v2-rev270-1.21.0"]
"v2-rev271-1.21.0"]
[com.h2database/h2 "1.4.191"] ; embedded SQL database
[com.mattbertolini/liquibase-slf4j "2.0.0"] ; Java Migrations lib
[com.novemberain/monger "3.0.2"] ; MongoDB Driver
......@@ -55,7 +55,7 @@
[net.sourceforge.jtds/jtds "1.3.1"] ; Open Source SQL Server driver
[org.xhtmlrenderer/flying-saucer-core "9.0.8"]
[org.liquibase/liquibase-core "3.4.2"] ; migration management (Java lib)
[org.slf4j/slf4j-log4j12 "1.7.16"]
[org.slf4j/slf4j-log4j12 "1.7.18"]
[org.yaml/snakeyaml "1.17"] ; YAML parser (required by liquibase)
[org.xerial/sqlite-jdbc "3.8.11.2"] ; SQLite driver
[postgresql "9.3-1102.jdbc41"] ; Postgres driver
......@@ -89,7 +89,7 @@
:exclusions [org.clojure/clojure]]
[lein-bikeshed "0.3.0"] ; Linting
[lein-expectations "0.0.8"] ; run unit tests with 'lein expectations'
[lein-instant-cheatsheet "2.1.5" ; use awesome instant cheatsheet created by yours truly w/ 'lein instant-cheatsheet'
[lein-instant-cheatsheet "2.2.1" ; use awesome instant cheatsheet created by yours truly w/ 'lein instant-cheatsheet'
:exclusions [org.clojure/clojure
org.clojure/tools.namespace]]]
:env {:mb-run-mode "dev"}
......
......@@ -445,6 +445,3 @@
obj)
([entity id]
(check-403 (models/can-write? entity id))))
(u/require-dox-in-this-namespace)
......@@ -107,7 +107,10 @@
;;; ## ROUTE TYPING / AUTO-PARSE SHARED FNS
(defn parse-int [value]
(defn parse-int
"Parse VALUE (presumabily a string) as an Integer, or throw a 400 exception.
Used to automatically to parse `id` parameters in `defendpoint` functions."
[^String value]
(try (Integer/parseInt value)
(catch java.lang.NumberFormatException _
(throw (ex-info (format "Not a valid integer: '%s'" value) {:status-code 400})))))
......@@ -117,10 +120,10 @@
:route-param-regex Regex pattern that should be used for params in Compojure route forms
:parser Function that should be used to parse args"
{:int {:route-param-regex #"[0-9]+"
:parser 'metabase.api.common.internal/parse-int}
{:int {:route-param-regex #"[0-9]+"
:parser 'metabase.api.common.internal/parse-int}
:uuid {:route-param-regex #"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"
:parser nil}})
:parser nil}})
(def ^:private ^:const auto-parse-arg-name-patterns
"Sequence of `[param-pattern parse-type]` pairs.
......@@ -262,7 +265,9 @@
(symbol? arg-symb)]}
`[~arg-symb (~((resolve 'metabase.api.common/-arg-annotation-fn) annotation-kw) '~arg-symb ~arg-symb)])
(defn process-arg-annotations [annotations-map]
(defn process-arg-annotations
"Internal function used by `defendpoint` for handling a parameter annotations map. Don't call this directly! "
[annotations-map]
{:pre [(or (nil? annotations-map)
(map? annotations-map))]}
(some->> annotations-map
......
......@@ -10,11 +10,11 @@
[metabase.integrations.slack :as slack]
(metabase.models [card :refer [Card]]
[database :refer [Database]]
[pulse :refer [Pulse] :as pulse]
[pulse :refer [Pulse retrieve-pulse] :as pulse]
[pulse-channel :refer [channel-types]])
[metabase.pulse :as p]
[metabase.task.send-pulses :refer [send-pulse]]
[metabase.models.pulse :refer [retrieve-pulse]]))
[metabase.task.send-pulses :refer [send-pulse!]]
[metabase.util :as u]))
(defendpoint GET "/"
......@@ -59,10 +59,9 @@
(defendpoint DELETE "/:id"
"Delete a `Pulse`."
[id]
(let [pulse (db/sel :one Pulse :id id)
result (db/cascade-delete Pulse :id id)]
(events/publish-event :pulse-delete (assoc pulse :actor_id *current-user-id*))
result))
(let-404 [pulse (Pulse id)]
(u/prog1 (db/cascade-delete Pulse :id id)
(events/publish-event :pulse-delete (assoc pulse :actor_id *current-user-id*)))))
(defendpoint GET "/form_input"
......@@ -75,8 +74,10 @@
;; no Slack integration, so we are g2g
chan-types
;; if we have Slack enabled build a dynamic list of channels/users
(let [slack-channels (mapv (fn [ch] (str "#" (get ch "name"))) (get (slack/channels-list) "channels"))
slack-users (mapv (fn [u] (str "@" (get u "name"))) (get (slack/users-list) "members"))]
(let [slack-channels (for [channel (slack/channels-list)]
(str \# (:name channel)))
slack-users (for [user (slack/users-list)]
(str \@ (:name user)))]
(assoc-in chan-types [:slack :fields 0 :options] (concat slack-channels slack-users))))}))
......@@ -86,7 +87,9 @@
(let [card (Card id)]
(read-check Database (:database (:dataset_query card)))
(let [data (:data (driver/dataset-query (:dataset_query card) {:executed_by *current-user-id*}))]
{:status 200, :body (html [:html [:body {:style "margin: 0;"} (p/render-pulse-card card data p/render-img-data-uri :include-title :include-buttons)]])})))
{:status 200, :body (html [:html [:body {:style "margin: 0;"} (binding [p/*include-title* true
p/*include-buttons* true]
(p/render-pulse-card card data))]])})))
(defendpoint GET "/preview_card_info/:id"
"Get JSON object containing HTML rendering of a `Card` with ID and other information."
......@@ -96,7 +99,8 @@
(let [result (driver/dataset-query (:dataset_query card) {:executed_by *current-user-id*})
data (:data result)
card-type (p/detect-pulse-card-type card data)
card-html (html (p/render-pulse-card card data p/render-img-data-uri :include-title (not :include-buttons)))]
card-html (html (binding [p/*include-title* true]
(p/render-pulse-card card data)))]
{:id id
:pulse_card_type card-type
:pulse_card_html card-html
......@@ -108,7 +112,8 @@
(let [card (Card id)]
(read-check Database (:database (:dataset_query card)))
(let [data (:data (driver/dataset-query (:dataset_query card) {:executed_by *current-user-id*}))
ba (p/render-pulse-card-to-png card data true)]
ba (binding [p/*include-title* true]
(p/render-pulse-card-to-png card data))]
{:status 200, :headers {"Content-Type" "image/png"}, :body (new java.io.ByteArrayInputStream ba) })))
(defendpoint POST "/test"
......@@ -117,7 +122,7 @@
{name [Required NonEmptyString]
cards [Required ArrayOfMaps]
channels [Required ArrayOfMaps]}
(send-pulse body)
(send-pulse! body)
{:ok true})
(define-routes)
......@@ -32,7 +32,7 @@
"Wrap API-ROUTES so they may only be accessed with proper authentiaction credentials."
middleware/enforce-authentication)
(defroutes routes
(defroutes ^{:doc "Ring routes for API endpoints."} routes
(context "/activity" [] (+auth activity/routes))
(context "/card" [] (+auth card/routes))
(context "/dashboard" [] (+auth dashboard/routes))
......
(ns metabase.api.slack
"/api/slack endpoints"
(:require [clojure.tools.logging :as log]
[clojure.set :as set]
[cheshire.core :as cheshire]
[compojure.core :refer [GET PUT DELETE POST]]
(:require [clojure.set :as set]
[clojure.tools.logging :as log]
[compojure.core :refer [PUT]]
[metabase.api.common :refer :all]
[metabase.config :as config]
[metabase.integrations [slack :refer [slack-api-get]]]
[metabase.models.setting :as setting]))
(defn- humanize-error-messages
"Convert raw error message responses from Slack into our normal api error response structure."
[response body]
(case (get body "error")
"invalid_auth" {:errors {:slack-token "Invalid token"}}
{:message "Sorry, something went wrong. Please try again."}))
[metabase.integrations.slack :as slack]))
(defendpoint PUT "/settings"
"Update multiple `Settings` values. You must be a superuser to do this."
[:as {settings :body}]
{settings [Required Dict]}
"Update the `slack-token`. You must be a superuser to do this."
[:as {{slack-token :slack-token} :body}]
{slack-token [Required NonEmptyString]}
(check-superuser)
(let [slack-token (:slack-token settings)
response (if-not config/is-test?
;; in normal conditions, validate connection
(slack-api-get slack-token "channels.list" {:exclude_archived 1})
;; for unit testing just respond with a success message
{:status 200 :body "{\"ok\":true}"})
body (if (= 200 (:status response)) (cheshire/parse-string (:body response)))]
(if (= true (get body "ok"))
;; test was good, save our settings
(setting/set :slack-token slack-token)
;; test failed, return response message
{:status 500
:body (humanize-error-messages response body)})))
(try
;; just check that channels.list doesn't throw an exception (that the connection works)
(when-not config/is-test?
(slack/GET :channels.list, :exclude_archived 1, :token slack-token))
{:ok true}
(catch clojure.lang.ExceptionInfo info
{:status 400, :body (ex-data info)})))
(define-routes)
......@@ -43,21 +43,13 @@
;; These are convenience functions for accessing config values that ensures a specific return type
(defn ^Integer config-int [k] (some-> k config-str Integer/parseInt))
(defn ^Boolean config-bool [k] (some-> k config-str Boolean/parseBoolean))
(defn ^Keyword config-kw [k] (some-> k config-str keyword))
(defn ^Integer config-int "Fetch a configuration key and parse it as an integer." [k] (some-> k config-str Integer/parseInt))
(defn ^Boolean config-bool "Fetch a configuration key and parse it as a boolean." [k] (some-> k config-str Boolean/parseBoolean))
(defn ^Keyword config-kw "Fetch a configuration key and parse it as a keyword." [k] (some-> k config-str keyword))
(def ^:const config-all
"Global application configuration as a dictionary.
Combines hard coded defaults with optional user specified overrides from environment variables."
(into {} (for [k (keys app-defaults)]
[k (config-str k)])))
(def ^:const ^Boolean is-dev? (= :dev (config-kw :mb-run-mode)))
(def ^:const ^Boolean is-prod? (= :prod (config-kw :mb-run-mode)))
(def ^:const ^Boolean is-test? (= :test (config-kw :mb-run-mode)))
(def ^:const ^Boolean is-dev? "Are we running in `dev` mode (i.e. in a REPL or via `lein ring server`)?" (= :dev (config-kw :mb-run-mode)))
(def ^:const ^Boolean is-prod? "Are we running in `prod` mode (i.e. from a JAR)?" (= :prod (config-kw :mb-run-mode)))
(def ^:const ^Boolean is-test? "Are we running in `test` mode (i.e. via `lein test`)?" (= :test (config-kw :mb-run-mode)))
;;; Version stuff
......
......@@ -210,7 +210,7 @@
;; Ok, now block forever while Jetty does its thing
(when (config/config-bool :mb-jetty-join)
(.join ^org.eclipse.jetty.server.Server @jetty-instance))
(catch Exception e
(catch Throwable e
(.printStackTrace e)
(log/error "Metabase Initialization FAILED: " (.getMessage e))
(System/exit 1))))
......
......@@ -185,7 +185,9 @@
(require 'metabase.db.migrations)
(@(resolve 'metabase.db.migrations/run-all)))
(defn setup-db-if-needed [& args]
(defn setup-db-if-needed
"Call `setup-db` if DB is not already setup; otherwise no-op."
[& args]
(when-not @setup-db-has-been-called?
(apply setup-db args)))
......
......@@ -64,7 +64,9 @@
;;; Low-level sel implementation
(defmacro sel-fn [& forms]
(defmacro sel-fn
"Part of the internal implementation for `sel`, don't call this directly!"
[& forms]
(let [forms (sel-apply-kwargs forms)
entity (gensym "ENTITY")]
(loop [query `(k/select* ~entity), [[f & args] & more] forms]
......@@ -74,7 +76,10 @@
:else `[(fn [~entity]
~query) ~(str query)]))))
(defn sel-exec [entity [select-fn log-str]]
(defn sel-exec
"Part of the internal implementation for `sel`, don't call this directly!
Execute the korma form generated by the `sel` macro and process the results."
[entity [select-fn log-str]]
(let [[entity field-keys] (destructure-entity entity)
entity (entity->korma entity)
entity+fields (assoc entity :fields (or field-keys
......@@ -90,86 +95,136 @@
(for [obj (k/exec (select-fn entity+fields))]
(models/do-post-select entity obj))))
(defmacro sel* [entity & forms]
(defmacro sel*
"Part of the internal implementation for `sel`, don't call this directly!"
[entity & forms]
`(sel-exec ~entity (sel-fn ~@forms)))
;;; :field
(defmacro sel:field [[entity field] & forms]
(defmacro sel:field
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :field ...)` forms."
[[entity field] & forms]
`(let [field# ~field]
(map field# (sel* [~entity field#] ~@forms))))
;;; :id
(defmacro sel:id [entity & forms]
(defmacro sel:id
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :id ...)` forms."
[entity & forms]
`(sel:field [~entity :id] ~@forms))
;;; :fields
(defn sel:fields* [fields results]
(defn sel:fields*
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :fields ...)` forms."
[fields results]
(for [result results]
(select-keys result fields)))
(defmacro sel:fields [[entity & fields] & forms]
(defmacro sel:fields
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :fields ...)` forms."
[[entity & fields] & forms]
`(let [fields# ~(vec fields)]
(sel:fields* (set fields#) (sel* `[~~entity ~@fields#] ~@forms))))
;;; :id->fields
(defn sel:id->fields* [fields results]
(defn sel:id->fields*
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :id->fields ...)` forms."
[fields results]
(->> results
(map (u/rpartial select-keys fields))
(zipmap (map :id results))))
(defmacro sel:id->fields [[entity & fields] & forms]
(defmacro sel:id->fields
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :id->fields ...)` forms."
[[entity & fields] & forms]
`(let [fields# ~(conj (set fields) :id)]
(sel:id->fields* fields# (sel* `[~~entity ~@fields#] ~@forms))))
;;; :field->field
(defn sel:field->field* [f1 f2 results]
(defn sel:field->field*
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :field->field ...)` forms."
[f1 f2 results]
(into {} (for [result results]
{(f1 result) (f2 result)})))
(defmacro sel:field->field [[entity f1 f2] & forms]
(defmacro sel:field->field
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :field->field ...)` forms."
[[entity f1 f2] & forms]
`(let [f1# ~f1
f2# ~f2]
(sel:field->field* f1# f2# (sel* [~entity f1# f2#] ~@forms))))
;;; :field->fields
(defn sel:field->fields* [key-field other-fields results]
(defn sel:field->fields*
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :field->fields ...)` forms."
[key-field other-fields results]
(into {} (for [result results]
{(key-field result) (select-keys result other-fields)})))
(defmacro sel:field->fields [[entity key-field & other-fields] & forms]
(defmacro sel:field->fields
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :field->fields ...)` forms."
[[entity key-field & other-fields] & forms]
`(let [key-field# ~key-field
other-fields# ~(vec other-fields)]
(sel:field->fields* key-field# other-fields# (sel* `[~~entity ~key-field# ~@other-fields#] ~@forms))))
;;; : id->field
(defmacro sel:id->field [[entity field] & forms]
(defmacro sel:id->field
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :id->field ...)` forms."
[[entity field] & forms]
`(sel:field->field [~entity :id ~field] ~@forms))
;;; :field->id
(defmacro sel:field->id [[entity field] & forms]
(defmacro sel:field->id
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :field->id ...)` forms."
[[entity field] & forms]
`(sel:field->field [~entity ~field :id] ~@forms))
;;; :field->obj
(defn sel:field->obj* [field results]
(defn sel:field->obj*
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :field->obj ...)` forms."
[field results]
(into {} (for [result results]
{(field result) result})))
(defmacro sel:field->obj [[entity field] & forms]
(defmacro sel:field->obj
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel ... :field->obj ...)` forms."
[[entity field] & forms]
`(sel:field->obj* ~field (sel* ~entity ~@forms)))
;;; :one & :many
(defmacro sel:one [& args]
(defmacro sel:one
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel :one ...)` forms."
[& args]
`(first (metabase.db/sel ~@args (k/limit 1))))
(defmacro sel:many [& args]
(defmacro sel:many
"Part of the internal implementation for `sel`, don't call this directly!
Handle `(sel :many ...)` forms."
[& args]
`(metabase.db/sel ~@args))
......@@ -328,10 +328,13 @@
;; ## Driver Lookup
(defn engine->driver
"Return the driver instance that should be used for given ENGINE.
This loads the corresponding driver if needed; it is expected that it resides in a var named
"Return the driver instance that should be used for given ENGINE keyword.
This loads the corresponding driver if needed; this is done with a call like
metabase.driver.<engine>/<engine>"
(require 'metabase.driver.<engine>)
The namespace itself should register itself by passing an instance of a class that
implements `IDriver` to `metabase.driver/register-driver!`."
[engine]
{:pre [engine]}
(or ((keyword engine) @registered-drivers)
......@@ -445,7 +448,7 @@
(log/error (u/pprint-to-str 'red query-result))
(throw (Exception. (str (get query-result :error "general error")))))
(query-complete query-execution query-result))
(catch Exception e
(catch Throwable e
(log/error (u/format-color 'red "Query failure: %s" (.getMessage e)))
(query-fail query-execution (.getMessage e))))))
......@@ -494,6 +497,3 @@
query-execution)
;; first time saving execution, so insert it
(m/mapply ins QueryExecution query-execution)))
(u/require-dox-in-this-namespace)
......@@ -481,7 +481,9 @@
;;; ### process-structured-query
(defn process-structured-query [do-query query]
(defn process-structured-query
"Process a structured query for a Druid DB."
[do-query query]
(binding [*query* query]
(let [[query-type druid-query] (build-druid-query query)]
(log-druid-query druid-query)
......
......@@ -46,7 +46,8 @@
(defprotocol ^:private IGenericSQLFormattable
(formatted [this]))
(formatted [this]
"Return an appropriate korma form for an object."))
(extend-protocol IGenericSQLFormattable
nil (formatted [_] nil)
......@@ -149,10 +150,10 @@
:not (kfns/pred-not (kengine/pred-map (filter-subclause->predicate subclause)))
nil (filter-subclause->predicate clause)))
(defn apply-filter [_ korma-form {clause :filter}]
(defn- apply-filter [_ korma-form {clause :filter}]
(k/where korma-form (filter-clause->predicate clause)))
(defn apply-join-tables [_ korma-form {join-tables :join-tables, {source-table-name :name, source-schema :schema} :source-table}]
(defn- apply-join-tables [_ korma-form {join-tables :join-tables, {source-table-name :name, source-schema :schema} :source-table}]
(loop [korma-form korma-form, [{:keys [table-name pk-field source-field schema]} & more] join-tables]
(let [table-name (if (seq schema)
(str schema \. table-name)
......@@ -167,10 +168,10 @@
(recur korma-form more)
korma-form))))
(defn apply-limit [_ korma-form {value :limit}]
(defn- apply-limit [_ korma-form {value :limit}]
(k/limit korma-form value))
(defn apply-order-by [_ korma-form {subclauses :order-by}]
(defn- apply-order-by [_ korma-form {subclauses :order-by}]
(loop [korma-form korma-form, [{:keys [field direction]} & more] subclauses]
(let [korma-form (k/order korma-form (formatted field) (case direction
:ascending :ASC
......@@ -179,7 +180,7 @@
(recur korma-form more)
korma-form))))
(defn apply-page [_ korma-form {{:keys [items page]} :page}]
(defn- apply-page [_ korma-form {{:keys [items page]} :page}]
(-> korma-form
(k/limit items)
(k/offset (* items (dec page)))))
......
......@@ -21,8 +21,7 @@
[metabase.driver.sync :as sync])
(:import com.mongodb.DB))
(declare driver field-values-lazy-seq)
(declare field-values-lazy-seq)
;;; ## MongoDriver
......
......@@ -453,6 +453,3 @@
{:style/indent 0}
[& body]
`(run-query* (query ~@body)))
(u/require-dox-in-this-namespace)
......@@ -137,7 +137,7 @@
(nil? value)
nil
:else
(throw (Exception. (format "Invalid value '%s': expected a DateTime." value))))))
......@@ -253,6 +253,3 @@
resolve-fields
resolve-database
resolve-tables))
(u/require-dox-in-this-namespace)
......@@ -100,7 +100,7 @@
(.connect transport host port user pass)))
{:error :SUCCESS
:message nil}
(catch Exception e
(catch Throwable e
(println "err" (.getMessage e))
{:error :ERROR
:message (.getMessage e)})))
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment