Skip to content
Snippets Groups Projects
Commit f7a8dbab authored by Cam Saül's avatar Cam Saül
Browse files

Merge pull request #488 from metabase/unix_timestamp_aggregations 

Unix timestamp aggregation support
parents 16cc67ef db23d846
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 500 additions and 191 deletions
circle.yml
+ 2
− 0
View file @ f7a8dbab
machine:
timezone:
America/Los_Angeles
java:
version:
oraclejdk8
......
src/metabase/driver.clj
+ 11
− 7
View file @ f7a8dbab
......@@ -5,6 +5,7 @@
[metabase.db :refer [exists? ins sel upd]]
(metabase.driver [interface :as i]
[query-processor :as qp])
[metabase.driver.query-processor.expand :as expand]
(metabase.models [database :refer [Database]]
[query-execution :refer [QueryExecution]])
[metabase.models.setting :refer [defsetting]]
......@@ -50,6 +51,7 @@
;; ## Driver Lookup
(def ^{:arglists '([engine])} engine->driver
"Return the driver instance that should be used for given ENGINE.
This loads the corresponding driver if needed; it is expected that it resides in a var named
......@@ -133,13 +135,15 @@
[query]
{:pre [(map? query)]}
(try
(binding [qp/*query* query
qp/*internal-context* (atom {})]
(let [driver (database-id->driver (:database query))
query (qp/preprocess query)
results (binding [qp/*query* query]
(i/process-query driver (dissoc-in query [:query :cum_sum])))] ; strip out things that individual impls don't need to know about / deal with
(qp/post-process driver query results)))
(let [driver (database-id->driver (:database query))]
(binding [qp/*query* query
qp/*expanded-query* (expand/expand query)
qp/*internal-context* (atom {})
qp/*driver* driver]
(let [query (qp/preprocess query)
results (binding [qp/*query* query]
(i/process-query driver (dissoc-in query [:query :cum_sum])))] ; strip out things that individual impls don't need to know about / deal with
(qp/post-process driver query results))))
(catch Throwable e
(.printStackTrace e)
{:status :failed
......
src/metabase/driver/context.clj
+ 4
− 0
View file @ f7a8dbab
(ns metabase.driver.context)
;;; DEPRECATED !
;; The functionality in this namespace is part of some old QP stuff and no longer serves any important purpose.
;; TODO - Remove this namespace
(def ^:dynamic *database*
"Current DB."
nil)
......
src/metabase/driver/generic_sql.clj
+ 7
− 1
View file @ f7a8dbab
......@@ -12,7 +12,13 @@
connection-details->connection-spec
database->connection-details
sql-string-length-fn
timezone->set-timezone-sql]
timezone->set-timezone-sql
;; These functions take a string name of a Field and return the raw SQL to select it as a DATE
cast-timestamp-seconds-field-to-date-fn
cast-timestamp-milliseconds-field-to-date-fn
;; This should be a regex that will match the column returned by the driver when unix timestamp -> date casting occured
;; e.g. #"CAST\(TIMESTAMPADD\('(?:MILLI)?SECOND', ([^\s]+), DATE '1970-01-01'\) AS DATE\)" for H2
uncastify-timestamp-regex]
IDriver
;; Connection
(can-connect? [this database]
......
src/metabase/driver/generic_sql/query_processor.clj
+ 4
− 10
View file @ f7a8dbab
......@@ -41,6 +41,7 @@
[column-name]
(let [column-name (name column-name)]
(keyword (or (second (re-find #"CAST\(([^\s]+) AS [\w]+\)" column-name))
(second (re-find (:uncastify-timestamp-regex qp/*driver*) column-name))
column-name))))
(defn process-structured
......@@ -132,9 +133,6 @@
;; [">" 1413 1]
;; [">=" 1412 4]]
(defn- field-id->special-type [field-id]
(sel :one :field [Field :special_type] :id field-id))
(defn- filter-subclause->predicate
"Given a filter SUBCLAUSE, return a Korma filter predicate form for use in korma `where`.
......@@ -150,13 +148,9 @@
[_ field-id & _] {(field-id->kw field-id)
;; If the field in question is a date field we need to cast the YYYY-MM-DD string that comes back from the UI to a SQL date
(let [cast-value-if-needed (fn [v]
(cond (or (= (type v) java.sql.Date)
(= (type v) java.util.Date)) `(raw ~(format "CAST('%s' AS DATE)" (.toString ^java.sql.Date v)))
(not (string? v)) v
(date-field-id? field-id) (u/parse-date-yyyy-mm-dd v)
(= (field-id->special-type field-id)
:timestamp_seconds) (u/date-yyyy-mm-dd->unix-timestamp v)
:else v))]
(if-not (or (= (type v) java.sql.Date)
(= (type v) java.util.Date)) v
`(raw ~(format "CAST('%s' AS DATE)" (.toString ^java.sql.Date v)))))]
(match subclause
["NOT_NULL" _] ['not= nil]
["IS_NULL" _] ['= nil]
......
src/metabase/driver/generic_sql/util.clj
+ 22
− 20
View file @ f7a8dbab
......@@ -9,6 +9,7 @@
[metabase.db :refer [sel]]
[metabase.driver :as driver]
[metabase.driver.context :as context]
[metabase.driver.query-processor :as qp]
(metabase.models [database :refer [Database]]
[field :refer [Field]]
[table :refer [Table]])))
......@@ -32,9 +33,6 @@
(fn [database]
(-db->korma-db (select-keys database [:engine :details])))))
#_(defn db->korma-db [database]
(kdb/create-db (db->connection-spec database)))
(def ^:dynamic ^java.sql.DatabaseMetaData *jdbc-metadata*
"JDBC metadata object for a database. This is set by `with-jdbc-metadata`."
nil)
......@@ -93,32 +91,36 @@
(defn castify-field
"Wrap Field in a SQL `CAST` statement if needed (i.e., it's a `:DateTimeField`).
(castify :name :TextField) -> :name
(castify :date :DateTimeField) -> (raw \"CAST(\"date\" AS DATE)"
[field-name field-base-type]
(castify :name :TextField nil) -> :name
(castify :date :DateTimeField nil) -> (raw \"CAST(\"date\" AS DATE)
(castify :timestamp :IntegerField :timestamp_seconds) -> (raw \"CAST(TO_TIMESTAMP(\"timestamp\") AS DATE))"
[field-name base-type special-type]
{:pre [(string? field-name)
(keyword? field-base-type)]}
;; do we need to cast DateFields ? or just DateTimeFields ?
(if (contains? #{:DateField :DateTimeField} field-base-type) `(korma/raw ~(format "CAST(\"%s\" AS DATE)" field-name))
(keyword field-name)))
(keyword? base-type)]}
(cond
(contains? #{:DateField :DateTimeField} base-type) `(korma/raw ~(format "CAST(\"%s\" AS DATE)" field-name))
(= special-type :timestamp_seconds) `(korma/raw ~((:cast-timestamp-seconds-field-to-date-fn qp/*driver*) field-name))
(= special-type :timestamp_milliseconds) `(korma/raw ~((:cast-timestamp-milliseconds-field-to-date-fn qp/*driver*) field-name))
:else (keyword field-name)))
(defn field-name+base-type->castified-key
"Like `castify-field`, but returns a keyword that should match the one returned in results."
[field-name field-base-type]
[field-name field-base-type special-type]
{:pre [(string? field-name)
(keyword? field-base-type)]
:post [(keyword? %)]}
(if (contains? #{:DateField :DateTimeField} field-base-type) (keyword (format "CAST(%s AS DATE)" field-name))
(keyword field-name)))
(keyword
(cond
(contains? #{:DateField :DateTimeField} field-base-type) (format "CAST(%s AS DATE)" field-name)
:else field-name)))
(def field-id->kw
(defn field-id->kw
"Given a metabase `Field` ID, return a keyword for use in the Korma form (or a casted raw string for date fields)."
(memoize ; This can be memozied since the names and base_types of Fields never change
(fn [field-id] ; * if a field is renamed the old field will just be marked as `inactive` and a new Field will be created
{:pre [(integer? field-id)]} ; * if a field's type *actually* changes we have no logic in driver.generic-sql.sync to handle that case any way (TODO - fix issue?)
(if-let [{field-name :name, field-type :base_type} (sel :one [Field :name :base_type] :id field-id)]
(castify-field field-name field-type)
(throw (Exception. (format "Field with ID %d doesn't exist!" field-id)))))))
[field-id]
{:pre [(integer? field-id)]}
(if-let [{field-name :name, field-type :base_type, special-type :special_type} (sel :one [Field :name :base_type :special_type] :id field-id)]
(castify-field field-name field-type special-type)
(throw (Exception. (format "Field with ID %d doesn't exist!" field-id)))))
(def date-field-id?
"Does FIELD-ID correspond to a field that is a Date?"
......
src/metabase/driver/h2.clj
+ 20
− 6
View file @ f7a8dbab
......@@ -82,12 +82,26 @@
:YEAR :IntegerField
(keyword "DOUBLE PRECISION") :FloatField})
;; ## QP Functions
(defn- cast-timestamp-seconds-field-to-date-fn [field-name]
(format "CAST(TIMESTAMPADD('SECOND', \"%s\", DATE '1970-01-01') AS DATE)" field-name))
(defn- cast-timestamp-milliseconds-field-to-date-fn [field-name]
(format "CAST(TIMESTAMPADD('MILLISECOND', \"%s\", DATE '1970-01-01') AS DATE)" field-name))
(def ^:private ^:const uncastify-timestamp-regex
#"CAST\(TIMESTAMPADD\('(?:MILLI)?SECOND', ([^\s]+), DATE '1970-01-01'\) AS DATE\)")
;; ## DRIVER
(def ^:const driver
(def driver
(generic-sql/map->SqlDriver
{:column->base-type column->base-type
:connection-details->connection-spec connection-details->connection-spec
:database->connection-details database->connection-details
:sql-string-length-fn :LENGTH
:timezone->set-timezone-sql nil}))
{:column->base-type column->base-type
:connection-details->connection-spec connection-details->connection-spec
:database->connection-details database->connection-details
:sql-string-length-fn :LENGTH
:timezone->set-timezone-sql nil
:cast-timestamp-seconds-field-to-date-fn cast-timestamp-seconds-field-to-date-fn
:cast-timestamp-milliseconds-field-to-date-fn cast-timestamp-milliseconds-field-to-date-fn
:uncastify-timestamp-regex uncastify-timestamp-regex}))
src/metabase/driver/postgres.clj
+ 18
− 5
View file @ f7a8dbab
......@@ -127,13 +127,26 @@
(defn- timezone->set-timezone-sql [timezone]
(format "SET LOCAL timezone TO '%s';" timezone))
(defn- cast-timestamp-seconds-field-to-date-fn [field-name]
{:pre [(string? field-name)]}
(format "CAST(TO_TIMESTAMP(\"%s\") AS DATE)" field-name))
(defn- cast-timestamp-milliseconds-field-to-date-fn [field-name]
{:pre [(string? field-name)]}
(format "CAST(TO_TIMESTAMP(\"%s\" / 1000) AS DATE)" field-name))
(def ^:private ^:const uncastify-timestamp-regex
#"CAST\(TO_TIMESTAMP\(([^\s+])(?: / 1000)?\) AS DATE\)")
;; ## DRIVER
(def ^:const driver
(generic-sql/map->SqlDriver
{:column->base-type column->base-type
:connection-details->connection-spec connection-details->connection-spec
:database->connection-details database->connection-details
:sql-string-length-fn :CHAR_LENGTH
:timezone->set-timezone-sql timezone->set-timezone-sql}))
{:column->base-type column->base-type
:connection-details->connection-spec connection-details->connection-spec
:database->connection-details database->connection-details
:sql-string-length-fn :CHAR_LENGTH
:timezone->set-timezone-sql timezone->set-timezone-sql
:cast-timestamp-seconds-field-to-date-fn cast-timestamp-seconds-field-to-date-fn
:cast-timestamp-milliseconds-field-to-date-fn cast-timestamp-milliseconds-field-to-date-fn
:uncastify-timestamp-regex uncastify-timestamp-regex}))
src/metabase/driver/query_processor.clj
+ 27
− 39
View file @ f7a8dbab
......@@ -15,8 +15,8 @@
(declare add-implicit-breakout-order-by
add-implicit-limit
add-implicit-fields
expand-date-values
get-special-column-info
preprocess-rewrite-timestamp-equals-filter
preprocess-cumulative-sum
preprocess-structured
remove-empty-clauses)
......@@ -39,7 +39,11 @@
;; # DYNAMIC VARS
(def ^:dynamic *query*
"The query we're currently processing (i.e., the body of the query API call)."
"The query we're currently processing, in its original, unexpanded form."
nil)
(def ^:dynamic *expanded-query*
"The query we're currently processing, in its expanded form."
nil)
(def ^:dynamic *disable-qp-logging*
......@@ -50,6 +54,10 @@
"A neat place to store 'notes-to-self': things individual implementations don't need to know about, like if the `fields` clause was added implicitly."
(atom nil))
(def ^:dynamic *driver*
"The driver currently being used to process this query."
(atom nil))
;; # PREPROCESSOR
......@@ -68,7 +76,7 @@
add-implicit-breakout-order-by
add-implicit-limit
add-implicit-fields
preprocess-rewrite-timestamp-equals-filter
expand-date-values
preprocess-cumulative-sum))]
(when-not *disable-qp-logging*
(log/debug (colorize.core/cyan "\n******************** PREPROCESSED: ********************\n"
......@@ -80,7 +88,7 @@
;; ## PREPROCESSOR FNS
;; ### REMOVE-EMPTY-CLAUSES
(def ^:const clause->empty-forms
(def ^:private ^:const clause->empty-forms
"Clause values that should be considered empty and removed during preprocessing."
{:breakout #{[nil]}
:filter #{[nil nil]}})
......@@ -142,39 +150,16 @@
(assoc query :fields (sel :many :id Field :table_id source_table, :active true, :preview_display true,
:field_type [not= "sensitive"], (order :position :asc), (order :id :desc))))))
(def ^:private ^:const seconds-per-day (* 24 60 60))
(def ^:private ^:const milliseconds-per-day (* seconds-per-day 1000))
(defn- rewrite-timestamp-filter=
"Rewrite an `=` filter clause for a timestamp `Field`. "
[{:keys [field], {timestamp :value, special-type :special-type, :as value} :value}]
;; The timestamps we create 00:00 on the day in question, re-write the filter as a ["BETWEEN" field timestamp (+ timestamp 1-day)]
(expand/map->Filter:Between {:type :between
:field field
:min value
:max (expand/map->Value (assoc value
:value (+ timestamp (case special-type
:timestamp_seconds seconds-per-day
:timestamp_milliseconds milliseconds-per-day))))}))
(defn preprocess-rewrite-timestamp-equals-filter
"In order for `=` filter clauses to work with timestamps (allowing the user to match a given day) we need to rewrite them as
`BETWEEN` clauses. Check and see if the `filter` clause contains any subclauses that fit the bill and rewrite them accordingly."
;;; ### EXPAND-DATES
(defn expand-date-values
"Expand any dates in the `:filter` clause.
This is done so various implementations can cast date values appropriately by simply checking their types.
In the future when drivers are re-worked to deal with the Expanded Query directly this step will no longer be needed."
[query]
(if-not (:filter query)
;; If there's no filter clause there's nothing to do
query
;; Otherwise rewrite as needed
(update-in query [:filter] (fn [filter-clause]
(-> filter-clause
expand/expand-filter
(update-in [:subclauses] #(for [{:keys [filter-type], {:keys [special-type]} :field, :as subclause} %]
(if (and (= filter-type :=)
(contains? #{:timestamp_seconds
:timestamp_milliseconds} special-type))
(rewrite-timestamp-filter= subclause)
subclause)))
expand/collapse)))))
(cond-> query
(:filter query) (assoc :filter (some-> *expanded-query* :query :filter expand/collapse)))) ; collapse the filter clause from the expanded query and use that as the replacement
;; ### PREPROCESS-CUMULATIVE-SUM
......@@ -266,7 +251,8 @@
(update-in results [:rows] #(for [row %]
(for [[i val] (m/indexed row)]
(cond
(contains? timestamp-seconds-col-indecies i) (java.sql.Date. (* val 1000))
(instance? java.util.Date val) val ; already converted to Date as part of preprocessing,
(contains? timestamp-seconds-col-indecies i) (java.sql.Date. (* val 1000)) ; nothing to do here
(contains? timestamp-millis-col-indecies i) (java.sql.Date. val)
:else val)))))))
......@@ -301,11 +287,11 @@
(count (set (:columns results))))
(format "Duplicate columns in results: %s" (vec (:columns results))))
(->> results
convert-unix-timestamps-to-dates
limit-max-result-rows
(#(case (keyword (:type query))
:native %
:query (post-process-cumulative-sum (:query query) %)))
convert-unix-timestamps-to-dates
add-row-count-and-status))
......@@ -442,7 +428,9 @@
(= col-kw :count) {:base_type :IntegerField
:special_type :number}
;; Otherwise something went wrong !
:else (throw (Exception. (format "Don't know what to do with Field '%s'." col-kw)))))))
:else (throw (Exception. (format "Annotation failed: don't know what to do with Field '%s'.\nExpected these Fields:\n%s"
col-kw
(u/pprint-to-str field-kw->field))))))))
;; Add FK info the the resulting Fields
add-fields-extra-info)))
......
src/metabase/driver/query_processor/expand.clj
+ 52
− 9
View file @ f7a8dbab
......@@ -50,7 +50,9 @@
[metabase.util :as u])
(:import (clojure.lang Keyword)))
(declare parse-filter
(declare parse-aggregation
parse-breakout
parse-filter
with-resolved-fields)
;; ## -------------------- Protocols --------------------
......@@ -76,9 +78,23 @@
IResolveField {:resolve-field (fn [this _] this)}
ICollapse {:collapse-one identity})
(extend nil
IResolveField {:resolve-field (constantly nil)})
;; ## -------------------- Public Interface --------------------
(defn- parse [query-dict]
(update-in query-dict [:query] #(assoc %
:aggregation (parse-aggregation (:aggregation %))
:breakout (parse-breakout (:breakout %))
:filter (parse-filter (:filter %)))))
(defn expand
"Expand a query-dict."
[query-dict]
(with-resolved-fields parse query-dict))
(defn expand-filter
"Expand a `filter` clause."
[filter-clause]
......@@ -140,11 +156,11 @@
[{:keys [value base-type special-type] :as qp-value}]
(assoc qp-value
:original-value value
:value (or (when (string? value) ; Since Value *doesn't* revert to YYYY-MM-DD when collapsing make sure we're not parsing it twice
(cond
(contains? #{:DateField :DateTimeField} base-type) (u/parse-date-yyyy-mm-dd value)
(= special-type :timestamp_seconds) (u/date-yyyy-mm-dd->unix-timestamp value)
(= special-type :timestamp_milliseconds) (* (u/date-yyyy-mm-dd->unix-timestamp value) 1000)))
;; Since Value *doesn't* revert to YYYY-MM-DD when collapsing make sure we're not parsing it twice
:value (or (when (and (string? value)
(or (contains? #{:DateField :DateTimeField} base-type)
(contains? #{:timestamp_seconds :timestamp_milliseconds} special-type)))
(u/parse-date-yyyy-mm-dd value))
value)))
;; Replace values with these during first pass over Query.
......@@ -203,8 +219,35 @@
"Convenience for writing a parser function, i.e. one that pattern-matches against a lone argument."
[fn-name & match-forms]
`(defn ~(vary-meta fn-name assoc :private true) [form#]
(match form#
~@match-forms)))
(when (and form#
(or (not (sequential? form#))
(and (seq form#)
(every? identity form#))))
(match form#
~@match-forms))))
;; ## -------------------- Aggregation --------------------
(defrecord Aggregation [^Keyword aggregation-type
^Field field])
(defparser parse-aggregation
["rows"] (->Aggregation :rows nil)
["count"] (->Aggregation :count nil)
["avg" field-id] (->Aggregation :avg (ph field-id))
["count" field-id] (->Aggregation :count (ph field-id))
["distinct" field-id] (->Aggregation :distinct (ph field-id))
["stddev" field-id] (->Aggregation :stddev (ph field-id))
["sum" field-id] (->Aggregation :sum (ph field-id))
["cum_sum" field-id] (->Aggregation :cumulative-sum (ph field-id)))
;; ## -------------------- Breakout --------------------
(defrecord Breakout [fields])
(defparser parse-breakout
[& field-ids] (mapv ph field-ids))
;; ## -------------------- Filter --------------------
......@@ -237,7 +280,7 @@
(collapse-one [_]
["BETWEEN" field min-val max-val]))
(defn- collapse-filter-type [^clojure.lang.Keyword filter-type]
(defn- collapse-filter-type [^Keyword filter-type]
(-> filter-type
name
(s/replace #"-" "_")
......
src/metabase/util.clj
+ 7
− 2
View file @ f7a8dbab
......@@ -46,16 +46,21 @@
(coerce/to-long)
(java.sql.Date.)))
(def ^:private ^java.text.SimpleDateFormat simple-date-format
(def ^:private ^java.text.SimpleDateFormat yyyy-mm-dd-simple-date-format
(java.text.SimpleDateFormat. "yyyy-MM-dd"))
(defn parse-date-yyyy-mm-dd
"Parse a date in the `yyyy-mm-dd` format and return a `java.sql.Date`."
^java.sql.Date [^String date]
(-> (.parse simple-date-format date)
(-> (.parse yyyy-mm-dd-simple-date-format date)
.getTime
java.sql.Date.))
(defn date->yyyy-mm-dd
"Convert a date to a `YYYY-MM-DD` string."
^String [^java.util.Date date]
(.format yyyy-mm-dd-simple-date-format date))
(defn date-yyyy-mm-dd->unix-timestamp
"Convert a string DATE in the `YYYY-MM-DD` format to a Unix timestamp in seconds."
^Float [^String date]
......
test/metabase/driver/query_processor_test.clj
+ 40
− 13
View file @ f7a8dbab
......@@ -7,7 +7,7 @@
(metabase.models [field :refer [Field]]
[table :refer [Table]])
[metabase.test.data :refer :all]
(metabase.test.data [dataset-definitions :refer [us-history-1607-to-1774]]
(metabase.test.data [dataset-definitions :as defs]
[datasets :as datasets :refer [*dataset*]])))
......@@ -778,15 +778,42 @@
;; ## Unix timestamp special type fields <3
;; There are 4 events in us-history-1607-to-1774 whose year is < 1765
(datasets/expect-with-all-datasets 4
(with-temp-db [db (dataset-loader) us-history-1607-to-1774]
(-> (driver/process-query {:database (:id db)
:type :query
:query {:source_table (:id &events)
:aggregation ["count"]
:filter ["<" (:id &events.timestamp) "1765-01-01"]}})
:data
:rows
first
first)))
;; There were 9 "sad toucan incidents" on 2015-06-02
(datasets/expect-with-datasets #{:generic-sql}
9
(with-temp-db [db (dataset-loader) defs/sad-toucan-incidents]
(->> (driver/process-query {:database (:id db)
:type "query"
:query {:source_table (:id &incidents)
:filter ["AND"
[">" (:id &incidents.timestamp) "2015-06-01"]
["<" (:id &incidents.timestamp) "2015-06-03"]]
:order_by [[(:id &incidents.timestamp) "ascending"]]}})
:data
:rows
count)))
;;; Unix timestamp breakouts -- SQL only
(datasets/expect-with-datasets #{:generic-sql}
[["2015-06-01" 6]
["2015-06-02" 9]
["2015-06-03" 5]
["2015-06-04" 9]
["2015-06-05" 8]
["2015-06-06" 9]
["2015-06-07" 8]
["2015-06-08" 9]
["2015-06-09" 7]
["2015-06-10" 8]]
(with-temp-db [db (dataset-loader) defs/sad-toucan-incidents]
(->> (driver/process-query {:database (:id db)
:type "query"
:query {:source_table (:id &incidents)
:aggregation ["count"]
:breakout [(:id &incidents.timestamp)]
:limit 10}})
:data
:rows
(map (fn [[^java.util.Date date count]]
[(.toString date) (int count)])))))
test/metabase/test/data.clj
+ 60
− 54
View file @ f7a8dbab
......@@ -78,63 +78,69 @@
(defn get-or-create-database!
"Create DBMS database associated with DATABASE-DEFINITION, create corresponding Metabase `Databases`/`Tables`/`Fields`, and sync the `Database`.
DATASET-LOADER should be an object that implements `IDatasetLoader`."
[dataset-loader {:keys [database-name], :as ^DatabaseDefinition database-definition}]
(let [engine (engine dataset-loader)]
(or (metabase-instance database-definition engine)
(do
;; Create the database
(log/info (color/blue (format "Creating %s database %s..." (name engine) database-name)))
(create-physical-db! dataset-loader database-definition)
;; Load data
(log/info (color/blue "Loading data..."))
(doseq [^TableDefinition table-definition (:table-definitions database-definition)]
(log/info (color/blue (format "Loading data for table '%s'..." (:table-name table-definition))))
(load-table-data! dataset-loader database-definition table-definition)
(log/info (color/blue (format "Inserted %d rows." (count (:rows table-definition))))))
;; Add DB object to Metabase DB
(log/info (color/blue "Adding DB to Metabase..."))
(let [db (ins Database
:name database-name
:engine (name engine)
:details (database->connection-details dataset-loader database-definition))]
;; Sync the database
(log/info (color/blue "Syncing DB..."))
(driver/sync-database! db)
;; Add extra metadata like Field field-type, base-type, etc.
(log/info (color/blue "Adding schema metadata..."))
(doseq [^TableDefinition table-definition (:table-definitions database-definition)]
(let [table-name (:table-name table-definition)
table (delay (let [table (metabase-instance table-definition db)]
(assert table)
table))]
(doseq [{:keys [field-name field-type special-type], :as field-definition} (:field-definitions table-definition)]
(let [field (delay (let [field (metabase-instance field-definition @table)]
(assert field)
field))]
(when field-type
(log/info (format "SET FIELD TYPE %s.%s -> %s" table-name field-name field-type))
(upd Field (:id @field) :field_type (name field-type)))
(when special-type
(log/info (format "SET SPECIAL TYPE %s.%s -> %s" table-name field-name special-type))
(upd Field (:id @field) :special_type (name special-type)))))))
(log/info (color/blue "Finished."))
db)))))
DATASET-LOADER should be an object that implements `IDatasetLoader`; it defaults to the value returned by the method `dataset-loader` for the
current dataset (`*dataset*`), which is H2 by default."
([^DatabaseDefinition database-definition]
(get-or-create-database! (dataset-loader) database-definition))
([dataset-loader {:keys [database-name], :as ^DatabaseDefinition database-definition}]
(let [engine (engine dataset-loader)]
(or (metabase-instance database-definition engine)
(do
;; Create the database
(log/info (color/blue (format "Creating %s database %s..." (name engine) database-name)))
(create-physical-db! dataset-loader database-definition)
;; Load data
(log/info (color/blue "Loading data..."))
(doseq [^TableDefinition table-definition (:table-definitions database-definition)]
(log/info (color/blue (format "Loading data for table '%s'..." (:table-name table-definition))))
(load-table-data! dataset-loader database-definition table-definition)
(log/info (color/blue (format "Inserted %d rows." (count (:rows table-definition))))))
;; Add DB object to Metabase DB
(log/info (color/blue "Adding DB to Metabase..."))
(let [db (ins Database
:name database-name
:engine (name engine)
:details (database->connection-details dataset-loader database-definition))]
;; Sync the database
(log/info (color/blue "Syncing DB..."))
(driver/sync-database! db)
;; Add extra metadata like Field field-type, base-type, etc.
(log/info (color/blue "Adding schema metadata..."))
(doseq [^TableDefinition table-definition (:table-definitions database-definition)]
(let [table-name (:table-name table-definition)
table (delay (let [table (metabase-instance table-definition db)]
(assert table)
table))]
(doseq [{:keys [field-name field-type special-type], :as field-definition} (:field-definitions table-definition)]
(let [field (delay (let [field (metabase-instance field-definition @table)]
(assert field)
field))]
(when field-type
(log/info (format "SET FIELD TYPE %s.%s -> %s" table-name field-name field-type))
(upd Field (:id @field) :field_type (name field-type)))
(when special-type
(log/info (format "SET SPECIAL TYPE %s.%s -> %s" table-name field-name special-type))
(upd Field (:id @field) :special_type (name special-type)))))))
(log/info (color/blue "Finished."))
db))))))
(defn remove-database!
"Delete Metabase `Database`, `Fields` and `Tables` associated with DATABASE-DEFINITION, then remove the physical database from the associated DBMS.
DATASET-LOADER should be an object that implements `IDatasetLoader`."
[dataset-loader ^DatabaseDefinition database-definition]
;; Delete the Metabase Database and associated objects
(cascade-delete Database :id (:id (metabase-instance database-definition (engine dataset-loader))))
;; now delete the DBMS database
(drop-physical-db! dataset-loader database-definition))
DATASET-LOADER should be an object that implements `IDatasetLoader`; by default it is the value returned by the method `dataset-loader` for the
current dataset, bound to `*dataset*`."
([^DatabaseDefinition database-definition]
(remove-database! (dataset-loader) database-definition))
([dataset-loader ^DatabaseDefinition database-definition]
;; Delete the Metabase Database and associated objects
(cascade-delete Database :id (:id (metabase-instance database-definition (engine dataset-loader))))
;; now delete the DBMS database
(drop-physical-db! dataset-loader database-definition)))
;; ## Temporary Dataset Macros
......
test/metabase/test/data/dataset_definitions.clj
+ 21
− 25
View file @ f7a8dbab
(ns metabase.test.data.dataset-definitions
"Definitions of various datasets for use in tests with `with-temp-db`."
(:require [metabase.test.data.interface :refer [def-database-definition]]))
(:require [clojure.tools.reader.edn :as edn]
[metabase.test.data.interface :refer [def-database-definition]]))
;; ## Helper Functions
(defn create-unix-timestamp
"Create a Unix timestamp (in seconds).
(defn- unix-timestamp-ms
"Create a Unix timestamp (in milliseconds).
(create-unix-timestamp :year 2012 :month 12 :date 27)"
(unix-timestamp-ms :year 2012 :month 12 :date 27)"
^Long [& {:keys [year month date hour minute second nano]
:or {year 0, month 1, date 1, hour 0, minute 0, second 0, nano 0}}]
(-> (java.sql.Timestamp. (- year 1900) (- month 1) date hour minute second nano)
.getTime
(/ 1000)
long)) ; coerce to long since Korma doesn't know how to insert bigints
(defn- unix-timestamp
"Create a Unix timestamp, in seconds."
^Long [& args]
(/ (apply unix-timestamp-ms args) 1000))
;; ## Datasets
(def-database-definition us-history-1607-to-1774
["events" [{:field-name "name"
:base-type :CharField}
{:field-name "timestamp"
:base-type :BigIntegerField
:special-type :timestamp_seconds}]
[["Jamestown Settlement Founded" (create-unix-timestamp :year 1607 :month 5 :date 14)]
["Mayflower Compact Signed" (create-unix-timestamp :year 1620 :month 11 :date 11)]
["Ben Franklin's Kite Experiment" (create-unix-timestamp :year 1752 :month 96 :date 15)]
["French and Indian War Begins" (create-unix-timestamp :year 1754 :month 5 :date 28)]
["Stamp Act Enacted" (create-unix-timestamp :year 1765 :month 3 :date 22)]
["Quartering Act Enacted" (create-unix-timestamp :year 1765 :month 3 :date 24)]
["Stamp Act Congress Meets" (create-unix-timestamp :year 1765 :month 10 :date 19)]
["Stamp Act Repealed" (create-unix-timestamp :year 1766 :month 3 :date 18)]
["Townshend Acts Passed" (create-unix-timestamp :year 1767 :month 6 :date 29)]
["Boston Massacre" (create-unix-timestamp :year 1770 :month 3 :date 5)]
["Tea Act Passed" (create-unix-timestamp :year 1773 :month 5 :date 10)]
["Boston Tea Party" (create-unix-timestamp :year 1773 :month 12 :date 16)]
["Boston Port Act Passed" (create-unix-timestamp :year 1774 :month 3 :date 31)]
["First Continental Congress Held" (create-unix-timestamp :year 1774 :month 9 :date 5)]]])
(def ^:private ^:const edn-definitions-dir "./test/metabase/test/data/dataset_definitions/")
;; TODO - move this to interface
;; TODO - make rows be lazily loadable for DB definitions from a file
(defmacro ^:private def-database-definition-edn [dbname]
`(def-database-definition ~dbname
(edn/read-string (slurp ~(str edn-definitions-dir (name dbname) ".edn")))))
;; Times when the Toucan cried
(def-database-definition-edn sad-toucan-incidents)
test/metabase/test/data/dataset_definitions/sad-toucan-incidents.edn 0 → 100644
+ 205
− 0
View file @ f7a8dbab
["incidents" [{:field-name "severity"
:base-type :IntegerField}
{:field-name "timestamp"
:base-type :BigIntegerField
:special-type :timestamp_milliseconds}]
[[4 1433587200000]
[0 1433965860000]
[5 1433864520000]
[3 1435016940000]
[3 1434764700000]
[3 1433253540000]
[5 1434995940000]
[2 1433383260000]
[1 1434247980000]
[0 1434389160000]
[2 1433276880000]
[2 1433857980000]
[5 1433879640000]
[1 1435482840000]
[3 1433745540000]
[2 1434700080000]
[5 1433536440000]
[3 1434157800000]
[1 1435150440000]
[2 1434702960000]
[3 1433749020000]
[2 1435255140000]
[5 1434358080000]
[4 1433410440000]
[2 1434737820000]
[0 1433794800000]
[5 1433323500000]
[4 1434914760000]
[3 1433397480000]
[5 1435158240000]
[4 1434952620000]
[0 1434060000000]
[3 1433395440000]
[2 1435029300000]
[5 1433272620000]
[0 1433944080000]
[5 1434577620000]
[0 1434753060000]
[1 1433991600000]
[5 1433578500000]
[4 1435365600000]
[4 1433243460000]
[1 1433279820000]
[2 1433288820000]
[3 1435010460000]
[3 1435106400000]
[4 1433535060000]
[1 1433641260000]
[1 1433184900000]
[4 1434937080000]
[0 1435441740000]
[2 1434872700000]
[4 1434705600000]
[2 1435095120000]
[1 1433898300000]
[1 1434519780000]
[1 1435240020000]
[2 1434663960000]
[5 1435363560000]
[4 1434663000000]
[5 1435351860000]
[4 1434975600000]
[2 1434971400000]
[2 1433675100000]
[2 1435088280000]
[0 1433549160000]
[1 1434094740000]
[1 1434904080000]
[5 1433211180000]
[1 1433751900000]
[4 1434982440000]
[1 1433826360000]
[5 1435060020000]
[5 1434450780000]
[1 1434236700000]
[1 1433280000000]
[1 1434135600000]
[5 1434338340000]
[4 1435389960000]
[3 1434302820000]
[0 1434102900000]
[3 1435444560000]
[1 1433174760000]
[4 1434933840000]
[0 1433959800000]
[0 1433977440000]
[0 1433233200000]
[4 1434164460000]
[4 1435193040000]
[4 1435124760000]
[0 1434969660000]
[1 1434867540000]
[3 1433440560000]
[2 1433688720000]
[2 1434946500000]
[1 1433973720000]
[4 1434517080000]
[3 1434709320000]
[5 1433583780000]
[5 1433693040000]
[2 1435229280000]
[2 1435362780000]
[2 1435107540000]
[5 1435048440000]
[5 1434709800000]
[4 1433449500000]
[1 1434947760000]
[5 1433832300000]
[1 1433548500000]
[0 1434071940000]
[1 1434263820000]
[2 1433592360000]
[5 1433652720000]
[2 1435506960000]
[2 1433492460000]
[5 1433785620000]
[3 1433309820000]
[2 1433886480000]
[2 1435106220000]
[0 1434353280000]
[1 1435506780000]
[4 1434954000000]
[5 1434502080000]
[1 1433794440000]
[5 1434456660000]
[4 1434751200000]
[3 1433193540000]
[2 1435190460000]
[2 1433594280000]
[3 1433790660000]
[5 1433365620000]
[5 1433192640000]
[5 1435532520000]
[0 1434284520000]
[4 1433654760000]
[3 1433948340000]
[5 1433223420000]
[4 1435068540000]
[4 1433939580000]
[2 1434707040000]
[5 1435233180000]
[1 1433179380000]
[0 1434963540000]
[2 1433538780000]
[4 1434607980000]
[2 1433481420000]
[2 1435148820000]
[1 1433994840000]
[4 1435476420000]
[2 1435405440000]
[3 1433553960000]
[1 1433764800000]
[2 1433542920000]
[2 1435425840000]
[1 1434731340000]
[5 1433846040000]
[0 1434582480000]
[0 1435514580000]
[2 1434812580000]
[3 1434521820000]
[4 1434166320000]
[0 1435103460000]
[0 1434291000000]
[3 1433517180000]
[1 1433383980000]
[1 1435210860000]
[0 1434403920000]
[4 1433714580000]
[2 1433954940000]
[5 1435044600000]
[0 1435365360000]
[1 1434212880000]
[1 1434920580000]
[0 1433551620000]
[2 1433494440000]
[5 1434398340000]
[4 1433154660000]
[0 1435334040000]
[1 1435123680000]
[0 1433674140000]
[4 1434714240000]
[3 1435336860000]
[5 1433377980000]
[1 1434252120000]
[4 1435038120000]
[3 1434278880000]
[0 1433366220000]
[0 1434029880000]
[4 1433789280000]
[2 1435343340000]
[4 1434343500000]
[5 1433398500000]
[3 1434805860000]
[1 1435215180000]
[3 1435010160000]
[1 1434436140000]
[5 1434972240000]
[5 1434851640000]
[1 1434107400000]
[4 1435492320000]]]
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment