diff --git a/resources/migrations/000_migrations.yaml b/resources/migrations/000_migrations.yaml
index 1927dce96d75ea0166612ce6beb37fbcd8df1d99..cc4aadbceafab5c518d433881b462c68c5fe8eca 100644
--- a/resources/migrations/000_migrations.yaml
+++ b/resources/migrations/000_migrations.yaml
@@ -3649,6 +3649,7 @@ databaseChangeLog:
- changeSet:
id: 56
author: wwwiiilll
+ comment: 'Added 0.25.0'
changes:
- addColumn:
tableName: core_user
@@ -3662,6 +3663,7 @@ databaseChangeLog:
- changeSet:
id: 57
author: camsaul
+ comment: 'Added 0.25.0'
changes:
- addColumn:
tableName: report_card
@@ -3673,6 +3675,7 @@ databaseChangeLog:
- changeSet:
id: 58
author: senior
+ comment: 'Added 0.25.0'
changes:
- createTable:
tableName: dimension
@@ -3744,6 +3747,7 @@ databaseChangeLog:
- changeSet:
id: 59
author: camsaul
+ comment: 'Added 0.26.0'
changes:
- addColumn:
tableName: metabase_field
@@ -3752,3 +3756,25 @@ databaseChangeLog:
name: fingerprint
type: text
remarks: 'Serialized JSON containing non-identifying information about this Field, such as min, max, and percent JSON. Used for classification.'
+ - changeSet:
+ id: 60
+ author: camsaul
+ comment: 'Added 0.26.0'
+ changes:
+ - addColumn:
+ tableName: metabase_database
+ columns:
+ - column:
+ name: metadata_sync_schedule
+ type: varchar(254)
+ remarks: 'The cron schedule string for when this database should undergo the metadata sync process (and analysis for new fields).'
+ defaultValue: '0 50 * * * ? *' # run at the end of every hour
+ constraints:
+ nullable: false
+ - column:
+ name: cache_field_values_schedule
+ type: varchar(254)
+ remarks: 'The cron schedule string for when FieldValues for eligible Fields should be updated.'
+ defaultValue: '0 50 0 * * ? *' # run at 12:50 AM
+ constraints:
+ nullable: false
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index 06d5200b0328efa654eb21f5f401cf90c50b54a8..5988c55b6178efe4a2f6af3c3a594b945d69b498 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -238,7 +238,7 @@
(defn- init-driver-in-namespace! [ns-symb]
(require ns-symb)
- (if-let [register-driver-fn (ns-resolve ns-symb (symbol "-init-driver"))]
+ (if-let [register-driver-fn (ns-resolve ns-symb '-init-driver)]
(register-driver-fn)
(log/warn (format "No -init-driver function found for '%s'" (name ns-symb)))))
@@ -326,8 +326,7 @@
This loads the corresponding driver if needed."
(let [db-id->engine (memoize (fn [db-id] (db/select-one-field :engine Database, :id db-id)))]
(fn [db-id]
- {:pre [db-id]}
- (when-let [engine (db-id->engine db-id)]
+ (when-let [engine (db-id->engine (u/get-id db-id))]
(engine->driver engine)))))
(defn ->driver
diff --git a/src/metabase/events/sync_database.clj b/src/metabase/events/sync_database.clj
index a9490440b9474e83f6c29652e65c906b1af7eb33..b0c71feaf6c6e868be563b0dfa95ba8f900913d3 100644
--- a/src/metabase/events/sync_database.clj
+++ b/src/metabase/events/sync_database.clj
@@ -4,7 +4,8 @@
[metabase
[events :as events]
[sync :as sync]]
- [metabase.models.database :refer [Database]]))
+ [metabase.models.database :refer [Database]]
+ [metabase.sync.sync-metadata :as sync-metadata]))
(def ^:const sync-database-topics
"The `Set` of event topics which are subscribed to for use in database syncing."
@@ -28,7 +29,10 @@
(when-let [database (Database (events/object->model-id topic object))]
;; just kick off a sync on another thread
(future (try
- (sync/sync-database! database)
+ ;; only do the 'full' sync if this is a "full sync" database. Otherwise just do metadata sync only
+ (if (:is_full_sync database)
+ (sync/sync-database! database)
+ (sync-metadata/sync-db-metadata! database))
(catch Throwable t
(log/error (format "Error syncing Database: %d" (:id database)) t))))))
(catch Throwable e
diff --git a/src/metabase/models/database.clj b/src/metabase/models/database.clj
index aa448ae66ede2aa96ebace5883564d7cb3b0506e..a16dcd17be92b00d825c9846c9a2463398343e44 100644
--- a/src/metabase/models/database.clj
+++ b/src/metabase/models/database.clj
@@ -1,5 +1,6 @@
(ns metabase.models.database
(:require [cheshire.generate :refer [add-encoder encode-map]]
+ [clojure.tools.logging :as log]
[metabase
[db :as mdb]
[util :as u]]
@@ -12,7 +13,6 @@
[db :as db]
[models :as models]]))
-
;;; ------------------------------------------------------------ Constants ------------------------------------------------------------
;; TODO - should this be renamed `saved-cards-virtual-id`?
@@ -34,23 +34,68 @@
(models/defmodel Database :metabase_database)
+
+(defn- schedule-tasks!
+ "(Re)schedule sync operation tasks for DATABASE. (Existing scheduled tasks will be deleted first.)"
+ [database]
+ (try
+ ;; this is done this way to avoid circular dependencies
+ (require 'metabase.task.sync-databases)
+ ((resolve 'metabase.task.sync-databases/schedule-tasks-for-db!) database)
+ (catch Throwable e
+ (log/error "Error scheduling tasks for DB:" (.getMessage e) "\n"
+ (u/pprint-to-str (u/filtered-stacktrace e))))))
+
+(defn- unschedule-tasks!
+ "Unschedule any currently pending sync operation tasks for DATABASE."
+ [database]
+ (try
+ (require 'metabase.task.sync-databases)
+ ((resolve 'metabase.task.sync-databases/unschedule-tasks-for-db!) database)
+ (catch Throwable e
+ (log/error "Error unscheduling tasks for DB:" (.getMessage e) "\n"
+ (u/pprint-to-str (u/filtered-stacktrace e))))))
+
(defn- post-insert [{database-id :id, :as database}]
(u/prog1 database
;; add this database to the all users and metabot permissions groups
(doseq [{group-id :id} [(perm-group/all-users)
(perm-group/metabot)]]
- (perms/grant-full-db-permissions! group-id database-id))))
+ (perms/grant-full-db-permissions! group-id database-id))
+ ;; schedule the Database sync tasks
+ (schedule-tasks! database)))
(defn- post-select [{:keys [engine] :as database}]
(if-not engine database
(assoc database :features (set (when-let [driver ((resolve 'metabase.driver/engine->driver) engine)]
((resolve 'metabase.driver/features) driver))))))
-(defn- pre-delete [{:keys [id]}]
+(defn- pre-delete [{id :id, :as database}]
+ (unschedule-tasks! database)
(db/delete! 'Card :database_id id)
(db/delete! 'Permissions :object [:like (str (perms/object-path id) "%")])
(db/delete! 'Table :db_id id))
+;; TODO - this logic would make more sense in post-update if such a method existed
+(defn- pre-update [{new-metadata-schedule :metadata_sync_schedule, new-fieldvalues-schedule :cache_field_values_schedule, :as database}]
+ (u/prog1 database
+ ;; if the sync operation schedules have changed, we need to reschedule this DB
+ (when (or new-metadata-schedule new-fieldvalues-schedule)
+ (let [{old-metadata-schedule :metadata_sync_schedule
+ old-fieldvalues-schedule :cache_field_values_schedule} (db/select-one [Database :metadata_sync_schedule :cache_field_values_schedule] :id (u/get-id database))
+ ;; if one of the schedules wasn't passed continue using the old one
+ new-metadata-schedule (or new-metadata-schedule old-metadata-schedule)
+ new-fieldvalues-schedule (or new-fieldvalues-schedule old-fieldvalues-schedule)]
+ (when (or (not= new-metadata-schedule old-metadata-schedule)
+ (not= new-fieldvalues-schedule old-fieldvalues-schedule))
+ (log/info "DB's schedules have changed!\n"
+ (format "Sync metadata was: '%s', is now: '%s'\n" old-metadata-schedule new-metadata-schedule)
+ (format "Cache FieldValues was: '%s', is now: '%s'\n" old-fieldvalues-schedule new-fieldvalues-schedule))
+ ;; reschedule the database. Make sure we're passing back the old schedule if one of the two wasn't supplied
+ (schedule-tasks! (assoc database
+ :metadata_sync_schedule new-metadata-schedule
+ :cache_field_values_schedule new-fieldvalues-schedule)))))))
+
(defn- perms-objects-set [database _]
#{(perms/object-path (u/get-id database))})
@@ -60,16 +105,20 @@
models/IModel
(merge models/IModelDefaults
{:hydration-keys (constantly [:database :db])
- :types (constantly {:details :encrypted-json, :engine :keyword})
+ :types (constantly {:details :encrypted-json
+ :engine :keyword
+ :metadata_sync_schedule :cron-string
+ :cache_field_values_schedule :cron-string})
:properties (constantly {:timestamped? true})
:post-insert post-insert
:post-select post-select
+ :pre-update pre-update
:pre-delete pre-delete})
i/IObjectPermissions
(merge i/IObjectPermissionsDefaults
- {:perms-objects-set perms-objects-set
- :can-read? (partial i/current-user-has-partial-permissions? :read)
- :can-write? i/superuser?}))
+ {:perms-objects-set perms-objects-set
+ :can-read? (partial i/current-user-has-partial-permissions? :read)
+ :can-write? i/superuser?}))
;;; ------------------------------------------------------------ Hydration / Util Fns ------------------------------------------------------------
@@ -106,10 +155,12 @@
"The string to replace passwords with when serializing Databases."
"**MetabasePass**")
-(add-encoder DatabaseInstance (fn [db json-generator]
- (encode-map (cond
- (not (:is_superuser @*current-user*)) (dissoc db :details)
- (get-in db [:details :password]) (assoc-in db [:details :password] protected-password)
- (get-in db [:details :pass]) (assoc-in db [:details :pass] protected-password) ; MongoDB uses "pass" instead of password
- :else db)
- json-generator)))
+(add-encoder
+ DatabaseInstance
+ (fn [db json-generator]
+ (encode-map (cond
+ (not (:is_superuser @*current-user*)) (dissoc db :details)
+ (get-in db [:details :password]) (assoc-in db [:details :password] protected-password)
+ (get-in db [:details :pass]) (assoc-in db [:details :pass] protected-password) ; MongoDB uses "pass" instead of password
+ :else db)
+ json-generator)))
diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.clj
index c7c2ace99bdf05c96530b78bfa3dd98537870b7b..3e956aa0f1e8cbcdf055dd8ff111c7ce7eadb84c 100644
--- a/src/metabase/models/interface.clj
+++ b/src/metabase/models/interface.clj
@@ -2,7 +2,10 @@
(:require [cheshire.core :as json]
[clojure.core.memoize :as memoize]
[metabase.util :as u]
- [metabase.util.encryption :as encryption]
+ [metabase.util
+ [encryption :as encryption]
+ [schema :as su]]
+ [schema.core :as s]
[taoensso.nippy :as nippy]
[toucan.models :as models])
(:import java.sql.Blob))
@@ -59,6 +62,13 @@
:in compress
:out decompress)
+(defn- validate-cron-string [s]
+ (s/validate (s/maybe su/CronScheduleString) s))
+
+(models/add-type! :cron-string
+ :in validate-cron-string
+ :out identity)
+
;;; properties
diff --git a/src/metabase/sync.clj b/src/metabase/sync.clj
index 4dde3530da20d8a6752f30d8730489919be2435b..da9999f3356448a483d6ec135130126e17e4b4c4 100644
--- a/src/metabase/sync.clj
+++ b/src/metabase/sync.clj
@@ -16,25 +16,20 @@
[schema.core :as s]
[metabase.sync.util :as sync-util]))
-(def ^:private SyncDatabaseOptions
- {(s/optional-key :full-sync?) s/Bool})
-
(s/defn ^:always-validate sync-database!
"Perform all the different sync operations synchronously for DATABASE.
- You may optionally supply OPTIONS, which can be used to disable so-called 'full-sync',
- meaning just metadata will be synced, but no 'analysis' (special type determination and
- FieldValues syncing) will be done."
- ([database]
- (sync-database! database {:full-sync? true}))
- ([database :- i/DatabaseInstance, options :- SyncDatabaseOptions]
- (sync-util/sync-operation :sync database (format "Sync %s with options: %s" (sync-util/name-for-logging database) options)
- ;; First make sure Tables, Fields, and FK information is up-to-date
- (sync-metadata/sync-db-metadata! database)
- (when (:full-sync? options)
- ;; Next, run the 'analysis' step where we do things like scan values of fields and update special types accordingly
- (analyze/analyze-db! database)
- ;; Finally, update FieldValues
- (field-values/update-field-values! database)))))
+ This is considered a 'full sync' in that all the different sync operations are performed at the same time.
+ Please note that this function is *not* what is called by the scheduled tasks. Those call different steps
+ independently."
+ {:style/indent 1}
+ [database :- i/DatabaseInstance]
+ (sync-util/sync-operation :sync database (format "Sync %s" (sync-util/name-for-logging database))
+ ;; First make sure Tables, Fields, and FK information is up-to-date
+ (sync-metadata/sync-db-metadata! database)
+ ;; Next, run the 'analysis' step where we do things like scan values of fields and update special types accordingly
+ (analyze/analyze-db! database)
+ ;; Finally, update cached FieldValues
+ (field-values/update-field-values! database)))
(s/defn ^:always-validate sync-table!
diff --git a/src/metabase/sync/interface.clj b/src/metabase/sync/interface.clj
index 6ba02b7b9e4fa469644df39aed82ac7a3ec91b32..7087972b7970e2c340baaa1259cbdd3bb20e75cb 100644
--- a/src/metabase/sync/interface.clj
+++ b/src/metabase/sync/interface.clj
@@ -97,6 +97,7 @@
(def Fingerprint
"Schema for a Field 'fingerprint' generated as part of the analysis stage. Used to power the 'classification' sub-stage of
analysis. Stored as the `fingerprint` column of Field."
- {(s/optional-key :global) GlobalFingerprint
+ {(s/optional-key :version) su/IntGreaterThanZero ; Fingerprints with no version key are assumed to have version of 1
+ (s/optional-key :global) GlobalFingerprint
(s/optional-key :type) TypeSpecificFingerprint
(s/optional-key :experimental) {s/Keyword s/Any}})
diff --git a/src/metabase/task.clj b/src/metabase/task.clj
index 5a75c79423c9a51964d1aced2f7167ddcf593b0c..3cbe37d1e50d64129e45a272004da5be1f44c2e1 100644
--- a/src/metabase/task.clj
+++ b/src/metabase/task.clj
@@ -1,20 +1,36 @@
(ns metabase.task
- "Background task scheduling via Quartzite. Individual tasks are defined in `metabase.task.*`.
+ "Background task scheduling via Quartzite. Individual tasks are defined in `metabase.task.*`.
## Regarding Task Initialization:
The most appropriate way to initialize tasks in any `metabase.task.*` namespace is to implement the
- `task-init` function which accepts zero arguments. This function is dynamically resolved and called
- exactly once when the application goes through normal startup procedures. Inside this function you
+ `task-init` function which accepts zero arguments. This function is dynamically resolved and called
+ exactly once when the application goes through normal startup procedures. Inside this function you
can do any work needed and add your task to the scheduler as usual via `schedule-task!`."
(:require [clojure.tools.logging :as log]
[clojurewerkz.quartzite.scheduler :as qs]
- [metabase.util :as u]))
+ [metabase.util :as u]
+ [schema.core :as s])
+ (:import [org.quartz JobDetail JobKey Scheduler Trigger TriggerKey]))
+;;; +------------------------------------------------------------------------------------------------------------------------+
+;;; | SCHEDULER INSTANCE |
+;;; +------------------------------------------------------------------------------------------------------------------------+
(defonce ^:private quartz-scheduler
(atom nil))
+(defn- scheduler
+ "Fetch the instance of our Quartz scheduler. Call this function rather than dereffing the atom directly
+ because there are a few places (e.g., in tests) where we swap the instance out."
+ ^Scheduler []
+ @quartz-scheduler)
+
+
+;;; +------------------------------------------------------------------------------------------------------------------------+
+;;; | FINDING & LOADING TASKS |
+;;; +------------------------------------------------------------------------------------------------------------------------+
+
(defn- find-and-load-tasks!
"Search Classpath for namespaces that start with `metabase.tasks.`, then `require` them so initialization can happen."
[]
@@ -26,6 +42,11 @@
(when-let [init-fn (ns-resolve ns-symb 'task-init)]
(init-fn))))
+
+;;; +------------------------------------------------------------------------------------------------------------------------+
+;;; | STARTING/STOPPING SCHEDULER |
+;;; +------------------------------------------------------------------------------------------------------------------------+
+
(defn start-scheduler!
"Start our Quartzite scheduler which allows jobs to be submitted and triggers to begin executing."
[]
@@ -41,21 +62,25 @@
[]
(log/debug "Stopping Quartz Scheduler")
;; tell quartz to stop everything
- (when @quartz-scheduler
- (qs/shutdown @quartz-scheduler))
+ (when-let [scheduler (scheduler)]
+ (qs/shutdown scheduler))
;; reset our scheduler reference
(reset! quartz-scheduler nil))
-(defn schedule-task!
+;;; +------------------------------------------------------------------------------------------------------------------------+
+;;; | SCHEDULING/DELETING TASKS |
+;;; +------------------------------------------------------------------------------------------------------------------------+
+
+(s/defn ^:always-validate schedule-task!
"Add a given job and trigger to our scheduler."
- [job trigger]
- (when @quartz-scheduler
- (qs/schedule @quartz-scheduler job trigger)))
+ [job :- JobDetail, trigger :- Trigger]
+ (when-let [scheduler (scheduler)]
+ (qs/schedule scheduler job trigger)))
-(defn delete-task!
+(s/defn ^:always-validate delete-task!
"delete a task from the scheduler"
- [job-key trigger-key]
- (when @quartz-scheduler
- (qs/delete-trigger @quartz-scheduler trigger-key)
- (qs/delete-job @quartz-scheduler job-key)))
+ [job-key :- JobKey, trigger-key :- TriggerKey]
+ (when-let [scheduler (scheduler)]
+ (qs/delete-trigger scheduler trigger-key)
+ (qs/delete-job scheduler job-key)))
diff --git a/src/metabase/task/sync_databases.clj b/src/metabase/task/sync_databases.clj
index 0589d11391169fa77284d77b1c30b1eb1b13d920..a2dd9ac0174e3cf624a39fec1d2d548ac0510d54 100644
--- a/src/metabase/task/sync_databases.clj
+++ b/src/metabase/task/sync_databases.clj
@@ -1,49 +1,169 @@
(ns metabase.task.sync-databases
- (:require [clj-time.core :as t]
- [clojure.tools.logging :as log]
+ "Scheduled tasks for syncing metadata/analyzing and caching FieldValues for connected Databases."
+ (:require [clojure.tools.logging :as log]
[clojurewerkz.quartzite
+ [conversion :as qc]
[jobs :as jobs]
[triggers :as triggers]]
[clojurewerkz.quartzite.schedule.cron :as cron]
[metabase
- [driver :as driver]
- [sync :as sync]
- [task :as task]]
+ [task :as task]
+ [util :as u]]
[metabase.models.database :refer [Database]]
- [toucan.db :as db]))
-
-(def ^:private ^:const sync-databases-job-key "metabase.task.sync-databases.job")
-(def ^:private ^:const sync-databases-trigger-key "metabase.task.sync-databases.trigger")
-
-(defonce ^:private sync-databases-job (atom nil))
-(defonce ^:private sync-databases-trigger (atom nil))
-
-;; simple job which looks up all databases and runs a sync on them
-(jobs/defjob SyncDatabases [_]
- (doseq [database (db/select Database, :is_sample false)] ; skip Sample Dataset DB
- (try
- ;; NOTE: this happens synchronously for now to avoid excessive load if there are lots of databases
- ;; most of the time we do a quick sync and avoid the lengthy analysis process
- ;; at midnight we run the full sync
- (let [full-sync? (not (and (zero? (t/hour (t/now)))
- (driver/driver-supports? (driver/engine->driver (:engine database)) :dynamic-schema)))]
- (sync/sync-database! database {:full-sync? full-sync?}))
- (catch Throwable e
- (log/error (format "Error syncing database %d: " (:id database)) e)))))
+ [metabase.sync
+ [analyze :as analyze]
+ [field-values :as field-values]
+ [sync-metadata :as sync-metadata]]
+ [metabase.util.schema :as su]
+ [schema.core :as s]
+ [toucan.db :as db])
+ (:import metabase.models.database.DatabaseInstance
+ [org.quartz CronTrigger JobDetail JobKey TriggerKey]))
+
+;;; +------------------------------------------------------------------------------------------------------------------------+
+;;; | JOB LOGIC |
+;;; +------------------------------------------------------------------------------------------------------------------------+
+
+(s/defn ^:private ^:always-validate job-context->database :- DatabaseInstance
+ "Get the Database referred to in JOB-CONTEXT. Guaranteed to return a valid Database."
+ [job-context]
+ (Database (u/get-id (get (qc/from-job-data job-context) "db-id"))))
+
+
+(jobs/defjob SyncAndAnalyzeDatabase [job-context]
+ (let [database (job-context->database job-context)]
+ (sync-metadata/sync-db-metadata! database)
+ ;; only run analysis if this is a "full sync" database
+ (when (:is_full_sync database)
+ (analyze/analyze-db! database))))
+
+
+(jobs/defjob UpdateFieldValues [job-context]
+ (field-values/update-field-values! (job-context->database job-context)))
+
+
+;;; +------------------------------------------------------------------------------------------------------------------------+
+;;; | TASK INFO AND GETTER FUNCTIONS |
+;;; +------------------------------------------------------------------------------------------------------------------------+
+
+(def ^:private TaskInfo
+ "One-off schema for information about the various sync tasks we run for a DB."
+ {:key s/Keyword
+ :db-schedule-column s/Keyword
+ :job-class Class})
+
+
+(def ^:private task-infos
+ "Maps containing info about the different independent sync tasks we schedule for each DB."
+ [{:key :sync-and-analyze
+ :db-schedule-column :metadata_sync_schedule
+ :job-class SyncAndAnalyzeDatabase}
+ {:key :update-field-values
+ :db-schedule-column :cache_field_values_schedule
+ :job-class UpdateFieldValues}])
+
+
+;; These getter functions are not strictly neccesary but are provided primarily so we can get some extra validation by using them
+
+(s/defn ^:private ^:always-validate job-key :- JobKey
+ "Return an appropriate string key for the job described by TASK-INFO for DATABASE-OR-ID."
+ [database :- DatabaseInstance, task-info :- TaskInfo]
+ (jobs/key (format "metabase.task.%s.job.%d" (name (:key task-info)) (u/get-id database))))
+
+(s/defn ^:private ^:always-validate trigger-key :- TriggerKey
+ "Return an appropriate string key for the trigger for TASK-INFO and DATABASE-OR-ID."
+ [database :- DatabaseInstance, task-info :- TaskInfo]
+ (triggers/key (format "metabase.task.%s.trigger.%d" (name (:key task-info)) (u/get-id database))))
+
+(s/defn ^:private ^:always-validate cron-schedule :- su/CronScheduleString
+ "Fetch the appropriate cron schedule string for DATABASE and TASK-INFO."
+ [database :- DatabaseInstance, task-info :- TaskInfo]
+ (get database (:db-schedule-column task-info)))
+
+(s/defn ^:private ^:always-validate job-class :- Class
+ "Get the Job class for TASK-INFO."
+ [task-info :- TaskInfo]
+ (:job-class task-info))
+
+(s/defn ^:private ^:always-validate description :- s/Str
+ "Return an appropriate description string for a job/trigger for Database described by TASK-INFO."
+ [database :- DatabaseInstance, task-info :- TaskInfo]
+ (format "%s Database %d" (name (:key task-info)) (u/get-id database)))
+
+
+;;; +------------------------------------------------------------------------------------------------------------------------+
+;;; | DELETING TASKS FOR A DB |
+;;; +------------------------------------------------------------------------------------------------------------------------+
+
+(s/defn ^:private ^:always-validate delete-task!
+ "Cancel a single sync job for DATABASE-OR-ID and TASK-INFO."
+ [database :- DatabaseInstance, task-info :- TaskInfo]
+ (let [job-key (job-key database task-info)
+ trigger-key (trigger-key database task-info)]
+ (log/debug (u/format-color 'red "Unscheduling task for Database %d: job: %s; trigger: %s" (u/get-id database) (.getName job-key) (.getName trigger-key)))
+ (task/delete-task! job-key trigger-key)))
+
+(s/defn ^:always-validate unschedule-tasks-for-db!
+ "Cancel *all* scheduled sync and FieldValues caching tassks for DATABASE-OR-ID."
+ [database :- DatabaseInstance]
+ (doseq [task-info task-infos]
+ (delete-task! database task-info)))
+
+
+;;; +------------------------------------------------------------------------------------------------------------------------+
+;;; | (RE)SCHEDULING TASKS FOR A DB |
+;;; +------------------------------------------------------------------------------------------------------------------------+
+
+(s/defn ^:private ^:always-validate job :- JobDetail
+ "Build a Quartz Job for DATABASE and TASK-INFO."
+ [database :- DatabaseInstance, task-info :- TaskInfo]
+ (jobs/build
+ (jobs/with-description (description database task-info))
+ (jobs/of-type (job-class task-info))
+ (jobs/using-job-data {"db-id" (u/get-id database)})
+ (jobs/with-identity (job-key database task-info))))
+
+(s/defn ^:private ^:always-validate trigger :- CronTrigger
+ "Build a Quartz Trigger for DATABASE and TASK-INFO."
+ [database :- DatabaseInstance, task-info :- TaskInfo]
+ (triggers/build
+ (triggers/with-description (description database task-info))
+ (triggers/with-identity (trigger-key database task-info))
+ (triggers/start-now)
+ (triggers/with-schedule
+ (cron/schedule
+ (cron/cron-schedule (cron-schedule database task-info))
+ ;; drop tasks if they start to back up
+ (cron/with-misfire-handling-instruction-do-nothing)))))
+
+
+(s/defn ^:private ^:always-validate schedule-task-for-db!
+ "Schedule a new Quartz job for DATABASE and TASK-INFO."
+ [database :- DatabaseInstance, task-info :- TaskInfo]
+ (let [job (job database task-info)
+ trigger (trigger database task-info)]
+ (log/debug (u/format-color 'green "Scheduling task for Database %d: job: %s; trigger: %s" (u/get-id database) (.getName (.getKey job)) (.getName (.getKey trigger))))
+ (task/schedule-task! job trigger)))
+
+
+(s/defn ^:always-validate schedule-tasks-for-db!
+ "Schedule all the different sync jobs we have for DATABASE.
+ Unschedules any existing jobs."
+ [database :- DatabaseInstance]
+ ;; unschedule any tasks that might already be scheduled
+ (unschedule-tasks-for-db! database)
+ ;; now (re)schedule all the tasks
+ (doseq [task-info task-infos]
+ (schedule-task-for-db! database task-info)))
+
+
+;;; +------------------------------------------------------------------------------------------------------------------------+
+;;; | TASK INITIALIZATION |
+;;; +------------------------------------------------------------------------------------------------------------------------+
(defn task-init
- "Automatically called during startup; start the job for syncing databases."
+ "Automatically called during startup; start the jobs for syncing/analyzing and updating FieldValues for all Dtabases besides
+ the sample dataset."
[]
- ;; build our job
- (reset! sync-databases-job (jobs/build
- (jobs/of-type SyncDatabases)
- (jobs/with-identity (jobs/key sync-databases-job-key))))
- ;; build our trigger
- (reset! sync-databases-trigger (triggers/build
- (triggers/with-identity (triggers/key sync-databases-trigger-key))
- (triggers/start-now)
- (triggers/with-schedule
- ;; run at the end of every hour
- (cron/cron-schedule "0 50 * * * ? *"))))
- ;; submit ourselves to the scheduler
- (task/schedule-task! @sync-databases-job @sync-databases-trigger))
+ (doseq [database (db/select Database, :is_sample false)]
+ (schedule-tasks-for-db! database)))
diff --git a/src/metabase/util/schema.clj b/src/metabase/util/schema.clj
index 01600347032c10226b6e2fd1ae632d03c171d3d0..552a49d6c6d469efed8ef5e034f1ac4bc4c00086 100644
--- a/src/metabase/util/schema.clj
+++ b/src/metabase/util/schema.clj
@@ -5,7 +5,8 @@
[medley.core :as m]
[metabase.util :as u]
[metabase.util.password :as password]
- [schema.core :as s]))
+ [schema.core :as s])
+ (:import org.quartz.CronExpression))
(defn with-api-error-message
"Return SCHEMA with an additional API-ERROR-MESSAGE that will be used to explain the error if a parameter fails validation."
@@ -137,3 +138,16 @@
"Schema for a valid map of embedding params."
(with-api-error-message (s/maybe {s/Keyword (s/enum "disabled" "enabled" "locked")})
"value must be a valid embedding params map."))
+
+
+(def CronScheduleString
+ "Schema for a valid cron schedule string."
+ ;; See `http://www.quartz-scheduler.org/documentation/quartz-2.x/tutorials/crontrigger#format` for details on Cron string format
+ (s/constrained
+ NonBlankString
+ (fn [^String s]
+ (try (CronExpression/validateExpression s)
+ true
+ (catch Throwable _
+ false)))
+ "Invalid cron schedule string."))
diff --git a/test/metabase/api/database_test.clj b/test/metabase/api/database_test.clj
index 52086a920eda4aa379cd31a604608502563bdf84..b4f46eef4c70c9d22784cefd814ed75bcfe68c49 100644
--- a/test/metabase/api/database_test.clj
+++ b/test/metabase/api/database_test.clj
@@ -53,13 +53,15 @@
(second @~result)))))
(def ^:private default-db-details
- {:engine "h2"
- :name "test-data"
- :is_sample false
- :is_full_sync true
- :description nil
- :caveats nil
- :points_of_interest nil})
+ {:engine "h2"
+ :name "test-data"
+ :is_sample false
+ :is_full_sync true
+ :description nil
+ :caveats nil
+ :points_of_interest nil
+ :cache_field_values_schedule "0 50 0 * * ? *"
+ :metadata_sync_schedule "0 50 * * * ? *"})
(defn- db-details
diff --git a/test/metabase/api/field_test.clj b/test/metabase/api/field_test.clj
index 6a2b2b7c19d104a3bc4a170de4b3e7ab65aac24e..c0fe291f301155052b4da9d454dc9a981084c5b9 100644
--- a/test/metabase/api/field_test.clj
+++ b/test/metabase/api/field_test.clj
@@ -22,17 +22,19 @@
(defn- db-details []
(tu/match-$ (db)
- {:created_at $
- :engine "h2"
- :caveats nil
- :points_of_interest nil
- :id $
- :updated_at $
- :name "test-data"
- :is_sample false
- :is_full_sync true
- :description nil
- :features (mapv name (driver/features (driver/engine->driver :h2)))}))
+ {:created_at $
+ :engine "h2"
+ :caveats nil
+ :points_of_interest nil
+ :id $
+ :updated_at $
+ :name "test-data"
+ :is_sample false
+ :is_full_sync true
+ :description nil
+ :features (mapv name (driver/features (driver/engine->driver :h2)))
+ :cache_field_values_schedule "0 50 0 * * ? *"
+ :metadata_sync_schedule "0 50 * * * ? *"}))
;; ## GET /api/field/:id
diff --git a/test/metabase/api/table_test.clj b/test/metabase/api/table_test.clj
index 62c60bf1444a9b02f40b42825bb32db5933dd978..7312bb511408aa7bfce7b5f6293e47ec0f057504 100644
--- a/test/metabase/api/table_test.clj
+++ b/test/metabase/api/table_test.clj
@@ -9,16 +9,17 @@
[middleware :as middleware]
[sync :as sync]
[util :as u]]
+ [metabase.api.table :as table-api]
[metabase.models
[card :refer [Card]]
[database :as database :refer [Database]]
[field :refer [Field]]
[permissions :as perms]
[permissions-group :as perms-group]
- [table :refer [Table]]]
+ [table :as table :refer [Table]]]
[metabase.test
[data :as data]
- [util :as tu :refer [match-$ resolve-private-vars]]]
+ [util :as tu :refer [match-$]]]
[metabase.test.data
[dataset-definitions :as defs]
[users :refer [user->client]]]
@@ -27,12 +28,6 @@
[hydrate :as hydrate]]
[toucan.util.test :as tt]))
-(resolve-private-vars metabase.models.table pk-field-id)
-(resolve-private-vars metabase.api.table
- dimension-options-for-response datetime-dimension-indexes numeric-dimension-indexes
- numeric-default-index date-default-index coordinate-default-index)
-
-
;; ## /api/org/* AUTHENTICATION Tests
;; We assume that all endpoints for a given context are enforced by the same middleware, so we don't run the same
;; authentication test on every single individual endpoint
@@ -45,17 +40,19 @@
(defn- db-details []
(match-$ (data/db)
- {:created_at $
- :engine "h2"
- :id $
- :updated_at $
- :name "test-data"
- :is_sample false
- :is_full_sync true
- :description nil
- :caveats nil
- :points_of_interest nil
- :features (mapv name (driver/features (driver/engine->driver :h2)))}))
+ {:created_at $
+ :engine "h2"
+ :id $
+ :updated_at $
+ :name "test-data"
+ :is_sample false
+ :is_full_sync true
+ :description nil
+ :caveats nil
+ :points_of_interest nil
+ :features (mapv name (driver/features (driver/engine->driver :h2)))
+ :cache_field_values_schedule "0 50 0 * * ? *"
+ :metadata_sync_schedule "0 50 * * * ? *"}))
(defn- table-defaults []
{:description nil
@@ -140,7 +137,7 @@
:display_name "Venues"
:rows 100
:updated_at $
- :pk_field (pk-field-id $$)
+ :pk_field (#'table/pk-field-id $$)
:id (data/id :venues)
:db_id (data/id)
:raw_table_id $
@@ -156,7 +153,7 @@
((user->client :rasta) :get 403 (str "table/" table-id))))
(defn- query-metadata-defaults []
- (->> dimension-options-for-response
+ (->> #'table-api/dimension-options-for-response
var-get
walk/keywordize-keys
(assoc (table-defaults) :dimension_options)))
@@ -229,8 +226,8 @@
:display_name "Last Login"
:base_type "type/DateTime"
:visibility_type "normal"
- :dimension_options (var-get datetime-dimension-indexes)
- :default_dimension_option (var-get date-default-index)
+ :dimension_options (var-get #'table-api/datetime-dimension-indexes)
+ :default_dimension_option (var-get #'table-api/date-default-index)
)
(assoc (field-details (Field (data/id :users :name)))
:special_type "type/Name"
@@ -275,8 +272,8 @@
:name "LAST_LOGIN"
:display_name "Last Login"
:base_type "type/DateTime"
- :dimension_options (var-get datetime-dimension-indexes)
- :default_dimension_option (var-get date-default-index))
+ :dimension_options (var-get #'table-api/datetime-dimension-indexes)
+ :default_dimension_option (var-get #'table-api/date-default-index))
(assoc (field-details (Field (data/id :users :name)))
:table_id (data/id :users)
:special_type "type/Name"
@@ -338,7 +335,7 @@
:name $
:rows 15
:display_name "Userz"
- :pk_field (pk-field-id $$)
+ :pk_field (#'table/pk-field-id $$)
:id $
:raw_table_id $
:created_at $}))
@@ -535,12 +532,12 @@
;; Ensure dimensions options are sorted numerically, but returned as strings
(expect
- (map str (sort (map #(Long/parseLong %) (var-get datetime-dimension-indexes))))
- (var-get datetime-dimension-indexes))
+ (map str (sort (map #(Long/parseLong %) (var-get #'table-api/datetime-dimension-indexes))))
+ (var-get #'table-api/datetime-dimension-indexes))
(expect
- (map str (sort (map #(Long/parseLong %) (var-get numeric-dimension-indexes))))
- (var-get numeric-dimension-indexes))
+ (map str (sort (map #(Long/parseLong %) (var-get #'table-api/numeric-dimension-indexes))))
+ (var-get #'table-api/numeric-dimension-indexes))
;; Numeric fields without min/max values should not have binning strategies
(expect
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index 7a9c434707cd1823661d17a94aacd0e9e88e7d1e..5441ca57baca534920995586a7de5f5063a50ac8 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -221,7 +221,7 @@
(drop-if-exists-and-create-db! "dropped_views_test")
;; create the DB object
(tt/with-temp Database [database {:engine :postgres, :details (assoc details :dbname "dropped_views_test")}]
- (let [sync! #(sync/sync-database! database {:full-sync? true})]
+ (let [sync! #(sync/sync-database! database)]
;; populate the DB and create a view
(exec! ["CREATE table birds (name VARCHAR UNIQUE NOT NULL);"
"INSERT INTO birds (name) VALUES ('Rasta'), ('Lucky'), ('Kanye Nest');"
diff --git a/test/metabase/sync_database_test.clj b/test/metabase/sync_database_test.clj
index badf18be90d20769468ae6a5105ce4e3b0b14629..7633c53604621c16468f850c76c4a9d0f3d5cdeb 100644
--- a/test/metabase/sync_database_test.clj
+++ b/test/metabase/sync_database_test.clj
@@ -351,7 +351,7 @@
;; create the `blueberries_consumed` table and insert a 100 values
(exec! ["CREATE TABLE blueberries_consumed (num INTEGER NOT NULL);"
(insert-range-sql (range 100))])
- (sync-database! db {:full-sync? true})
+ (sync-database! db)
(let [table-id (db/select-one-id Table :db_id (u/get-id db))
field-id (db/select-one-id Field :table_id table-id)]
;; field values should exist...
diff --git a/test/metabase/task/sync_databases_test.clj b/test/metabase/task/sync_databases_test.clj
new file mode 100644
index 0000000000000000000000000000000000000000..8b8e5838b7ccd60385db92a34c06ac303f269de0
--- /dev/null
+++ b/test/metabase/task/sync_databases_test.clj
@@ -0,0 +1,209 @@
+(ns metabase.task.sync-databases-test
+ "Tests for the logic behind scheduling the various sync operations of Databases. Most of the actual logic we're testing is part of `metabase.models.database`,
+ so there's an argument to be made that these sorts of tests could just as easily belong to a `database-test` namespace."
+ (:require [clojure.string :as str]
+ [expectations :refer :all]
+ [metabase.models.database :refer [Database]]
+ metabase.task.sync-databases
+ [metabase.test.util :as tu]
+ [metabase.util :as u]
+ [toucan.db :as db]
+ [toucan.util.test :as tt])
+ (:import [metabase.task.sync_databases SyncAndAnalyzeDatabase UpdateFieldValues]))
+
+(defn- replace-trailing-id-with-<id> [s]
+ (str/replace s #"\d+$" "<id>"))
+
+(defn- replace-ids-with-<id> [current-tasks]
+ (vec (for [task current-tasks]
+ (-> task
+ (update :description replace-trailing-id-with-<id>)
+ (update :key replace-trailing-id-with-<id>)
+ (update-in [:data "db-id"] class)
+ (update :triggers (fn [triggers]
+ (vec (for [trigger triggers]
+ (update trigger :key replace-trailing-id-with-<id>)))))))))
+
+(defn- current-tasks []
+ (replace-ids-with-<id> (tu/scheduler-current-tasks)))
+
+;; Check that a newly created database automatically gets scheduled
+(expect
+ [{:description "sync-and-analyze Database <id>"
+ :class SyncAndAnalyzeDatabase
+ :key "metabase.task.sync-and-analyze.job.<id>"
+ :data {"db-id" Integer}
+ :triggers [{:key "metabase.task.sync-and-analyze.trigger.<id>"
+ :cron-schedule "0 50 * * * ? *"}]}
+ {:description "update-field-values Database <id>"
+ :class UpdateFieldValues
+ :key "metabase.task.update-field-values.job.<id>"
+ :data {"db-id" Integer}
+ :triggers [{:key "metabase.task.update-field-values.trigger.<id>"
+ :cron-schedule "0 50 0 * * ? *"}]}]
+ (tu/with-temp-scheduler
+ (tt/with-temp Database [database {:engine :postgres}]
+ (current-tasks))))
+
+
+;; Check that a custom schedule is respected when creating a new Database
+(expect
+ [{:description "sync-and-analyze Database <id>"
+ :class SyncAndAnalyzeDatabase
+ :key "metabase.task.sync-and-analyze.job.<id>"
+ :data {"db-id" Integer}
+ :triggers [{:key "metabase.task.sync-and-analyze.trigger.<id>"
+ :cron-schedule "0 30 4,16 * * ? *"}]}
+ {:description "update-field-values Database <id>"
+ :class UpdateFieldValues
+ :key "metabase.task.update-field-values.job.<id>"
+ :data {"db-id" Integer}
+ :triggers [{:key "metabase.task.update-field-values.trigger.<id>"
+ :cron-schedule "0 15 10 ? * 6#3"}]}]
+ (tu/with-temp-scheduler
+ (tt/with-temp Database [database {:engine :postgres
+ :metadata_sync_schedule "0 30 4,16 * * ? *" ; 4:30 AM and PM daily
+ :cache_field_values_schedule "0 15 10 ? * 6#3"}] ; 10:15 on the 3rd Friday of the Month
+ (current-tasks))))
+
+
+;; Check that a deleted database gets unscheduled
+(expect
+ []
+ (tu/with-temp-scheduler
+ (tt/with-temp Database [database {:engine :postgres}]
+ (db/delete! Database :id (u/get-id database))
+ (current-tasks))))
+
+;; Check that changing the schedule column(s) for a DB properly updates the scheduled tasks
+(expect
+ [{:description "sync-and-analyze Database <id>"
+ :class SyncAndAnalyzeDatabase
+ :key "metabase.task.sync-and-analyze.job.<id>"
+ :data {"db-id" Integer}
+ :triggers [{:key "metabase.task.sync-and-analyze.trigger.<id>"
+ :cron-schedule "0 15 10 ? * MON-FRI"}]}
+ {:description "update-field-values Database <id>"
+ :class UpdateFieldValues
+ :key "metabase.task.update-field-values.job.<id>"
+ :data {"db-id" Integer}
+ :triggers [{:key "metabase.task.update-field-values.trigger.<id>"
+ :cron-schedule "0 11 11 11 11 ?"}]}]
+ (tu/with-temp-scheduler
+ (tt/with-temp Database [database {:engine :postgres}]
+ (db/update! Database (u/get-id database)
+ :metadata_sync_schedule "0 15 10 ? * MON-FRI" ; 10:15 AM every weekday
+ :cache_field_values_schedule "0 11 11 11 11 ?") ; Every November 11th at 11:11 AM
+ (current-tasks))))
+
+;; Check that changing one schedule doesn't affect the other
+(expect
+ [{:description "sync-and-analyze Database <id>"
+ :class SyncAndAnalyzeDatabase
+ :key "metabase.task.sync-and-analyze.job.<id>"
+ :data {"db-id" Integer}
+ :triggers [{:key "metabase.task.sync-and-analyze.trigger.<id>"
+ :cron-schedule "0 50 * * * ? *"}]}
+ {:description "update-field-values Database <id>"
+ :class UpdateFieldValues
+ :key "metabase.task.update-field-values.job.<id>"
+ :data {"db-id" Integer}
+ :triggers [{:key "metabase.task.update-field-values.trigger.<id>"
+ :cron-schedule "0 15 10 ? * MON-FRI"}]}]
+ (tu/with-temp-scheduler
+ (tt/with-temp Database [database {:engine :postgres}]
+ (db/update! Database (u/get-id database)
+ :cache_field_values_schedule "0 15 10 ? * MON-FRI")
+ (current-tasks))))
+
+(expect
+ [{:description "sync-and-analyze Database <id>"
+ :class SyncAndAnalyzeDatabase
+ :key "metabase.task.sync-and-analyze.job.<id>"
+ :data {"db-id" Integer}
+ :triggers [{:key "metabase.task.sync-and-analyze.trigger.<id>"
+ :cron-schedule "0 15 10 ? * MON-FRI"}]}
+ {:description "update-field-values Database <id>"
+ :class UpdateFieldValues
+ :key "metabase.task.update-field-values.job.<id>"
+ :data {"db-id" Integer}
+ :triggers [{:key "metabase.task.update-field-values.trigger.<id>"
+ :cron-schedule "0 50 0 * * ? *"}]}]
+ (tu/with-temp-scheduler
+ (tt/with-temp Database [database {:engine :postgres}]
+ (db/update! Database (u/get-id database)
+ :metadata_sync_schedule "0 15 10 ? * MON-FRI")
+ (current-tasks))))
+
+;; Check that you can't INSERT a DB with an invalid schedule
+(expect
+ Exception
+ (db/insert! Database {:engine :postgres
+ :metadata_sync_schedule "0 * ABCD"}))
+
+(expect
+ Exception
+ (db/insert! Database {:engine :postgres
+ :cache_field_values_schedule "0 * ABCD"}))
+
+;; Check that you can't UPDATE a DB's schedule to something invalid
+(expect
+ Exception
+ (tt/with-temp Database [database {:engine :postgres}]
+ (db/update! Database (u/get-id database)
+ :metadata_sync_schedule "2 CANS PER DAY")))
+
+(expect
+ Exception
+ (tt/with-temp Database [database {:engine :postgres}]
+ (db/update! Database (u/get-id database)
+ :cache_field_values_schedule "2 CANS PER DAY")))
+
+
+;;; +------------------------------------------------------------------------------------------------------------------------+
+;;; | CHECKING THAT SYNC TASKS RUN CORRECT FNS |
+;;; +------------------------------------------------------------------------------------------------------------------------+
+
+(defn- check-if-sync-processes-ran-for-db {:style/indent 0} [db-info]
+ (let [sync-db-metadata-counter (atom 0)
+ analyze-db-counter (atom 0)
+ update-field-values-counter (atom 0)]
+ (with-redefs [metabase.sync.sync-metadata/sync-db-metadata! (fn [& _] (swap! sync-db-metadata-counter inc))
+ metabase.sync.analyze/analyze-db! (fn [& _] (swap! analyze-db-counter inc))
+ metabase.sync.field-values/update-field-values! (fn [& _] (swap! update-field-values-counter inc))]
+ (tu/with-temp-scheduler
+ (tt/with-temp Database [database db-info]
+ ;; give tasks some time to run
+ (Thread/sleep 2000)
+ {:ran-sync? (not (zero? @sync-db-metadata-counter))
+ :ran-analyze? (not (zero? @analyze-db-counter))
+ :ran-update-field-values? (not (zero? @update-field-values-counter))})))))
+
+(defn- cron-schedule-for-next-year []
+ (format "0 15 10 * * ? %d" (inc (u/date-extract :year))))
+
+;; Make sure that a database that *is* marked full sync *will* get analyzed
+(expect
+ {:ran-sync? true, :ran-analyze? true, :ran-update-field-values? false}
+ (check-if-sync-processes-ran-for-db
+ {:engine :postgres
+ :metadata_sync_schedule "* * * * * ? *"
+ :cache_field_values_schedule (cron-schedule-for-next-year)}))
+
+;; Make sure that a database that *isn't* marked full sync won't get analyzed
+(expect
+ {:ran-sync? true, :ran-analyze? false, :ran-update-field-values? false}
+ (check-if-sync-processes-ran-for-db
+ {:engine :postgres
+ :is_full_sync false
+ :metadata_sync_schedule "* * * * * ? *"
+ :cache_field_values_schedule (cron-schedule-for-next-year)}))
+
+;; Make sure the update field values task calls `update-field-values!`
+(expect
+ {:ran-sync? false, :ran-analyze? false, :ran-update-field-values? true}
+ (check-if-sync-processes-ran-for-db
+ {:engine :postgres
+ :is_full_sync false
+ :metadata_sync_schedule (cron-schedule-for-next-year)
+ :cache_field_values_schedule "* * * * * ? *"}))
diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.clj
index a6a23f89f37d45637da37d764f28154e58193be0..2d5512a5c3d46ee75096dd45959070833db38a11 100644
--- a/test/metabase/test/util.clj
+++ b/test/metabase/test/util.clj
@@ -1,9 +1,15 @@
(ns metabase.test.util
"Helper functions and macros for writing unit tests."
(:require [cheshire.core :as json]
+ [clojure
+ [string :as str]
+ [walk :as walk]]
[clojure.tools.logging :as log]
- [clojure.walk :as walk]
+ [clojurewerkz.quartzite.scheduler :as qs]
[expectations :refer :all]
+ [metabase
+ [task :as task]
+ [util :as u]]
[metabase.models
[card :refer [Card]]
[collection :refer [Collection]]
@@ -21,8 +27,8 @@
[table :refer [Table]]
[user :refer [User]]]
[metabase.test.data :as data]
- [metabase.util :as u]
- [toucan.util.test :as test]))
+ [toucan.util.test :as test])
+ (:import [org.quartz CronTrigger JobDetail JobKey Scheduler Trigger]))
;; ## match-$
@@ -136,7 +142,7 @@
(u/strict-extend (class Database)
test/WithTempDefaults
{:with-temp-defaults (fn [_] {:details {}
- :engine :yeehaw
+ :engine :yeehaw ; wtf?
:is_sample false
:name (random-name)})})
@@ -338,8 +344,64 @@
(update-in-if-present [:fingerprint :type :type/Number] round-fingerprint-fields [:min :max :avg])
(update-in-if-present [:fingerprint :type :type/Text] round-fingerprint-fields [:percent-json :percent-url :percent-email :average-length])))
-(defn round-fingerprint-cols [query-results]
+(defn round-fingerprint-cols [query-results]
(let [maybe-data-cols (if (contains? query-results :data)
[:data :cols]
[:cols])]
(update-in query-results maybe-data-cols #(map round-fingerprint %))))
+
+;;; +------------------------------------------------------------------------------------------------------------------------+
+;;; | SCHEDULER |
+;;; +------------------------------------------------------------------------------------------------------------------------+
+
+;; Various functions for letting us check that things get scheduled properly. Use these to put a temporary scheduler in place
+;; and then check the tasks that get scheduled
+
+(defn do-with-scheduler [scheduler f]
+ (with-redefs [metabase.task/scheduler (constantly scheduler)]
+ (f)))
+
+(defmacro with-scheduler
+ "Temporarily bind the Metabase Quartzite scheduler to SCHEULDER and run BODY."
+ {:style/indent 1}
+ [scheduler & body]
+ `(do-with-scheduler ~scheduler (fn [] ~@body)))
+
+(defn do-with-temp-scheduler [f]
+ (let [temp-scheduler (qs/start (qs/initialize))]
+ (with-scheduler temp-scheduler
+ (try (f)
+ (finally
+ (qs/shutdown temp-scheduler))))))
+
+(defmacro with-temp-scheduler
+ "Execute BODY with a temporary scheduler in place.
+
+ (with-temp-scheduler
+ (do-something-to-schedule-tasks)
+ ;; verify that the right thing happened
+ (scheduler-current-tasks))"
+ {:style/indent 0}
+ [& body]
+ `(do-with-temp-scheduler (fn [] ~@body)))
+
+(defn scheduler-current-tasks
+ "Return information about the currently scheduled tasks (jobs+triggers) for the current scheduler.
+ Intended so we can test that things were scheduled correctly."
+ []
+ (when-let [^Scheduler scheduler (#'task/scheduler)]
+ (vec
+ (sort-by
+ :key
+ (for [^JobKey job-key (.getJobKeys scheduler nil)]
+ (let [^JobDetail job-detail (.getJobDetail scheduler job-key)
+ triggers (.getTriggersOfJob scheduler job-key)]
+ {:description (.getDescription job-detail)
+ :class (.getJobClass job-detail)
+ :key (.getName job-key)
+ :data (into {} (.getJobDataMap job-detail))
+ :triggers (vec (for [^Trigger trigger triggers]
+ (merge
+ {:key (.getName (.getKey trigger))}
+ (when (instance? CronTrigger trigger)
+ {:cron-schedule (.getCronExpression ^CronTrigger trigger)}))))}))))))
diff --git a/test_resources/log4j.properties b/test_resources/log4j.properties
index a085ebb0927e78d16ea47dfe4d800017a298cd5e..75116bf562a111292e65f08dfd7e38bb8b0414e5 100644
--- a/test_resources/log4j.properties
+++ b/test_resources/log4j.properties
@@ -19,8 +19,8 @@ log4j.logger.com.mchange=ERROR
log4j.logger.org.eclipse.jetty.server.HttpChannel=ERROR
log4j.logger.metabase=ERROR
log4j.logger.metabase.test-setup=INFO
-log4j.logger.metabase.test.data.datasets=INFO
-log4j.logger.metabase.util.encryption=INFO
-# NOCOMMIT
log4j.logger.metabase.sync=DEBUG
+log4j.logger.metabase.task.sync-databases=INFO
+log4j.logger.metabase.test.data.datasets=INFO
log4j.logger.metabase.test.data=DEBUG
+log4j.logger.metabase.util.encryption=INFO