Basic search index management (#47846)

dev/src/dev/search.clj

@@ -11,7 +11,7 @@
 (comment
   (#'search.index/drop-table! @#'search.index/active-table)
   (search.index/reset-index!)
-  (search.index/create-pending!)
+  (search.index/maybe-create-pending!)
   (search.index/activate-pending!)
 
   {:initialized? @@#'search.index/initialized? :reindexing? @@#'search.index/reindexing?}

src/metabase/api/search.clj

@@ -3,6 +3,7 @@
    [compojure.core :refer [GET]]
    [java-time.api :as t]
    [metabase.api.common :as api]
+   [metabase.public-settings :as public-settings]
    [metabase.search :as search]
    [metabase.server.middleware.offset-paging :as mw.offset-paging]
    [metabase.util :as u]
@@ -36,6 +37,22 @@
                respond
                raise))))
 
+(api/defendpoint POST "/force-reindex"
+  "If fulltext search is enabled, this will trigger a synchronous reindexing operation."
+  []
+  (api/check-superuser)
+  (cond
+    (not (public-settings/experimental-fulltext-search-enabled))
+    {:status-code 501, :message "Search index is not enabled."}
+
+    (search/supports-index?)
+    (do
+      (search/reindex!)
+      {:status-code 200})
+
+    :else
+    {:status-code 501, :message "Search index is not supported for this installation."}))
+
 ;; TODO maybe deprecate this and make it as a parameter in `GET /api/search/models`
 ;; so we don't have to keep the arguments between 2 API in sync
 (api/defendpoint GET "/models"

src/metabase/search.clj

@@ -35,6 +35,23 @@
                     search.impl/in-place))
     :in-place search.impl/in-place))
 
+(defn supports-index?
+  "Does this instance support a search index, e.g. has the right kind of AppDb"
+  []
+  (is-postgres?))
+
+(defn init-index!
+  "Ensure there is an index ready to be populated."
+  [& {:keys [force-reset?]}]
+  (when (is-postgres?)
+    (search.postgres/init! force-reset?)))
+
+(defn reindex!
+  "Populate a new index, and make it active. Simultaneously updates the current index."
+  []
+  (when (is-postgres?)
+    (search.postgres/reindex!)))
+
 (mu/defn search
   "Builds a search query that includes all the searchable entities and runs it"
   [search-ctx :- search.config/SearchContext]

src/metabase/search/postgres/core.clj

@@ -87,9 +87,16 @@
 (defn init!
   "Ensure that the search index exists, and has been populated with all the entities."
   [& [force-reset?]]
-  (when (or force-reset? (not (#'search.index/exists? @#'search.index/active-table)))
-    (search.index/reset-index!))
+  (search.index/ensure-ready! force-reset?)
   (search.ingestion/populate-index!))
 
+(defn reindex!
+  "Populate a new index"
+  []
+  (search.index/ensure-ready! false)
+  (search.index/maybe-create-pending!)
+  (search.ingestion/populate-index!)
+  (search.index/activate-pending!))
+
 (comment
   (init! true))

src/metabase/search/postgres/index.clj

@@ -30,7 +30,7 @@
         (sql.helpers/rename-table new)
         t2/query)))
 
-(defn create-pending!
+(defn maybe-create-pending!
   "Create a non-active search index table."
   []
   (when (not @reindexing?)
@@ -50,7 +50,12 @@
              [:database_id :int]
              [:table_id :int]
              ;; filter related
-             [:archived :boolean]])
+             [:archived :boolean]
+             ;; useful for tracking the speed and age of the index
+             [:created_at :timestamp
+              [:default [:raw "CURRENT_TIMESTAMP"]]
+              :not-null]])
+
           t2/query)
 
       (t2/query
@@ -116,6 +121,13 @@
   []
   (reset! reindexing? false)
   (drop-table! pending-table)
-  (create-pending!)
+  (maybe-create-pending!)
   (activate-pending!)
   (reset! initialized? true))
+
+(defn ensure-ready!
+  "Ensure the index is ready to be populated. Return false if it was already ready."
+  [force-recreation?]
+  (if (or force-recreation? (not (exists? active-table)))
+    (reset-index!)
+    (reset! initialized? true)))

src/metabase/task/search_index.clj

+(ns metabase.task.search-index
+  (:require
+   [clojurewerkz.quartzite.jobs :as jobs]
+   [clojurewerkz.quartzite.triggers :as triggers]
+   [metabase.public-settings :as public-settings]
+   [metabase.search :as search]
+   [metabase.task :as task]
+   [metabase.util.log :as log])
+  (:import
+   (org.quartz DisallowConcurrentExecution)))
+
+(set! *warn-on-reflection* true)
+
+;; We need each instance to initialize on start-up currently, this will need to be refined.
+(jobs/defjob ^{DisallowConcurrentExecution false
+               :doc                        "Populate Search Index"}
+  SearchIndexing [_ctx]
+  (when (public-settings/experimental-fulltext-search-enabled)
+    (log/info "Recreating search index from latest schema")
+    (search/init-index! {:force-reset? true})
+    (log/info "Populating search index")
+    (search/reindex!)
+    (log/info "Done indexing.")))
+
+(defmethod task/init! ::SearchIndex [_]
+  (let [job     (jobs/build
+                 (jobs/of-type SearchIndexing)
+                 (jobs/with-identity (jobs/key "metabase.task.search-index.job")))
+        trigger (triggers/build
+                 (triggers/with-identity (triggers/key "metabase.task.search-index.trigger"))
+                 (triggers/start-now))]
+    (task/schedule-task! job trigger)))