Automatically load `metabase.task.*` namespaces when starting the task

runner. Add task to execute all `Querys` on the `nightly-task-hook`

.dir-locals.el

@@ -7,6 +7,7 @@
                               (catch-api-exceptions 0)
                               (check 1)
                               (context 2)
+                              (execute-query 1)
                               (expect 1)
                               (expect-eval-actual-first 1)
                               (expect-let 1)

src/metabase/driver.clj

@@ -82,9 +82,12 @@
      :synchronously [true|false]      (default true)
      :cache_result [true|false]       (default false)
   "
+  {:arglists '([query caller-options])}
   [query {:keys [executed_by synchronously saved_query]
           :or {synchronously true}
           :as caller-options}]
+  {:pre [(integer? executed_by)
+         (map? saved_query)]}
   (let [options (merge {:cache_result false} caller-options)
         query-execution {:uuid (.toString (java.util.UUID/randomUUID))
                          :executor_id executed_by

src/metabase/models/query_execution.clj

@@ -4,10 +4,10 @@
             [metabase.api.common :refer [check]]
             [metabase.db :refer :all]
             (metabase.models [common :refer :all]
-              [hydrate :refer [realize-json]]
-              [user :refer [User]]
-              [database :refer [Database]]
-              [query :refer [Query]])
+                             [hydrate :refer [realize-json]]
+                             [user :refer [User]]
+                             [database :refer [Database]]
+                             [query :refer [Query]])
             [metabase.util :refer :all]))
 
 
@@ -49,14 +49,13 @@
   (assoc query-execution :json_query (if (string? json_query) json_query
                                         (json/write-str json_query))))
 
-(defmethod post-select QueryExecution [_ {:keys [query_id] :as query-execution}]
+(defmethod post-select QueryExecution [_ {:keys [query_id result_rows] :as query-execution}]
   (-> query-execution
-    (assoc :row_count (get query-execution :result_rows 0))  ;; sadly we have 2 ways to reference the row count :(
-    (realize-json :json_query)
-    (realize-json :result_data)
-    (assoc* :query (delay
-                     (check query_id 500 "Can't get execution: QueryExecution doesn't have a :query_id.")
-                     (sel :one Query :id query_id)))))
+    (realize-json :json_query :result_data)
+    (assoc* :row_count (or result_rows 0) ; sadly we have 2 ways to reference the row count :(
+            :query (delay
+                    (check query_id 500 "Can't get execution: QueryExecution doesn't have a :query_id.")
+                    (sel :one Query :id query_id)))))
 
 
 (defn build-response

src/metabase/task.clj

 ;; -*- comment-column: 60; -*-
 (ns metabase.task
-  (:require [clojure.tools.logging :as log]
+  "Function hooks; background task runner and related hooks.
+   Namespaces under `metabase.task.*` will be automatically loaded when the background task runner is started."
+  (:require clojure.java.classpath
+            [clojure.tools.logging :as log]
+            [clojure.tools.namespace.find :as ns-find]
             [metabase.util :as u])
   (:import java.util.Calendar))
 
@@ -98,6 +102,8 @@
   "Tasks to run nightly at midnight (according to the system calendar).
    Functions will be passed no arguments.")
 
+;; ## RUN-HOURLY-TASKS / RUN-NIGHTLY-TASKS
+
 (defn- hour
   "Current hour (0 - 23) according to the system calendar."
   []
@@ -143,13 +149,33 @@
     (run-hook #'nightly-tasks-hook :parallel)))
 (add-hook! #'hourly-tasks-hook run-nightly-tasks)
 
+
+;; ## COLLECT TASKS IN METABASE.TASK.* NAMESPACES
+
+(defn- find-and-load-tasks
+  "Search JARs + files in the classpath for Clojure namespaces that start with `metabase.task.`, then `require` them so tasks will be loaded as needed."
+  []
+  (->> (ns-find/find-namespaces (clojure.java.classpath/classpath))
+       (filter (fn [ns-symb]
+                 (re-find #"^metabase\.task\." (name ns-symb))))
+       set
+       (map (fn [task-ns]
+              (log/info "Loading tasks from namespace" task-ns "...")
+              (require task-ns)))
+       dorun))
+
+
+;; ## START/STOP TASK RUNNER
+
 (defonce ^:private task-runner
   (atom nil))
 
 (defn start-task-runner!
-  "Start a background thread that will run tasks on the `hourly-tasks-hook` and `nightly-tasks-hook` every hour / every night, respectively. "
+  "Start a background thread that will run tasks on the `hourly-tasks-hook` and `nightly-tasks-hook` every hour / every night, respectively.
+   This also loads all namespaces under `metabase.task.*`, so tasks can defined in them without needing to load them elsewhere."
   []
   (when-not @task-runner
+    (find-and-load-tasks)
     (log/info "Starting task runner...")
     (reset! task-runner (future (run-hourly-tasks)))))
 

src/metabase/task/execute_queries.clj

+(ns metabase.task.execute-queries
+  (require (metabase [db :refer :all]
+                     [driver :as driver]
+                     [task :refer :all])
+           [metabase.models.query :refer [Query]]))
+
+(defn execute-queries []
+  "Execute all `Querys` in the database, one-at-a-time."
+  (->> (sel :many Query)
+       (map (fn [{database-id :database_id
+                 creator-id :creator_id
+                 {:keys [sql timezone]} :details :as query}]
+              (let [dataset-query {:type :native                 ; TODO: this code looks too much like the code in POST /api/query/:id
+                                   :database database-id         ; it would make me happier if there was a nice way to de-duplicate it
+                                   :native {:query sql
+                                            :timezone timezone}}
+                    options {:executed_by creator-id             ; HACK: Technically, creator *isn't* executing this `Query`, but this is a required field
+                             :saved_query query
+                             :synchronously true                 ; probably makes sense to run these one-at-a-time to avoid putting too much stress on the DB
+                             :cache_result true}]
+                (driver/dataset-query dataset-query options))))
+       dorun))
+
+(add-hook! #'nightly-tasks-hook execute-queries)