Basic test for search reindexing (#50560) (#50568)

Co-authored-by: Chris Truter <crisptrutski@users.noreply.github.com>

src/metabase/search/appdb/core.clj

@@ -92,7 +92,7 @@
     (search.ingestion/populate-index! :search.engine/fulltext)))
 
 (defmethod search.engine/reindex! :search.engine/fulltext
-  []
+  [_]
   (search.index/ensure-ready! false)
   (search.index/maybe-create-pending!)
   (u/prog1 (search.ingestion/populate-index! :search.engine/fulltext)

src/metabase/search/appdb/index.clj

@@ -57,11 +57,13 @@
 
 (defn maybe-create-pending!
   "Create a search index table."
-  []
-  (when (not @reindexing?)
-    (when-not (exists? pending-table)
-      (create-table! pending-table))
-    (reset! reindexing? true)))
+  ([]
+   (when (not @reindexing?)
+     (maybe-create-pending! pending-table)
+     (reset! reindexing? true)))
+  ([table-name]
+   (when-not (exists? table-name)
+     (create-table! table-name))))
 
 (defn activate-pending!
   "Make the pending index active if it exists. Returns true if it did so."
@@ -113,9 +115,11 @@
   "Create the given search index entries in bulk"
   [documents]
   (let [entries          (map document->entry documents)
+        ;; When stubbing the table in tests, always treat it as active
+        active?          (or @initialized? (not= :search_index *active-table*))
         ;; Ideally, we would reset these atoms if the corresponding tables don't exist.
         ;; We're about to rework this area, so just leaving this as a note for now.
-        active-updated?  (when @initialized? (safe-batch-upsert! *active-table* entries))
+        active-updated?  (when active? (safe-batch-upsert! *active-table* entries))
         pending-updated? (when @reindexing?  (safe-batch-upsert! pending-table entries))]
     (when (or active-updated? pending-updated?)
       (->> entries (map :model) frequencies))))
@@ -147,11 +151,17 @@
 (defn reset-index!
   "Ensure we have a blank slate; in case the table schema or stored data format has changed."
   []
-  (reset! reindexing? false)
-  (drop-table! pending-table)
-  (maybe-create-pending!)
-  (activate-pending!)
-  (reset! initialized? true))
+  ;; Moving to random tables will clean this up
+  (let [testing?   (not= *active-table* :search_index)
+        table-name (if testing? *active-table* pending-table)]
+    (when-not testing?
+      (reset! reindexing? false))
+    (drop-table! table-name)
+    (maybe-create-pending! table-name)
+    (when-not testing?
+      (activate-pending!)
+      (reset! initialized? true))
+    true))
 
 (defn ensure-ready!
   "Ensure the index is ready to be populated. Return false if it was already ready."

src/metabase/task/search_index.clj

@@ -35,22 +35,25 @@
 
 (defn reindex!
   "Reindex the whole AppDB"
-  []
-  (when (search/supports-index?)
-    (let [timer  (u/start-timer)
-          report (if (not @recreated?)
-                   (do (log/info "Recreating search index from the latest schema")
-                       ;; Each instance in a multi-instance deployment will recreate the table the first time it is
-                       ;; selected to run the job, resulting in a momentary lack of search results.  One solution to
-                       ;; this would be to store metadata about the index in another table, which we can use to
-                       ;; determine whether it was built by another version of Metabase and should be rebuilt.
-
-                       (u/prog1 (search/init-index! {:force-reset? (not @recreated?)})
-                         (reset! recreated? true)))
-                   (do (log/info "Reindexing searchable entities")
-                       (search/reindex!)))]
-      (report->prometheus! report)
-      (log/infof "Done indexing in %.0fms %s" (u/since-ms timer) (sort-by (comp - val) report)))))
+  ([]
+   (let [recreate? (not @recreated?)]
+     (u/prog1 (reindex! recreate?)
+       (when recreate?
+         (reset! recreated? true)))))
+  ([recreate?]
+   (when (search/supports-index?)
+     (let [timer  (u/start-timer)
+           report (if recreate?
+                    (do (log/info "Recreating search index from the latest schema")
+                        ;; Each instance in a multi-instance deployment will recreate the table the first time it is
+                        ;; selected to run the job, resulting in a momentary lack of search results.  One solution to
+                        ;; this would be to store metadata about the index in another table, which we can use to
+                        ;; determine whether it was built by another version of Metabase and should be rebuilt.
+                        (search/init-index! {:force-reset? recreate?}))
+                    (do (log/info "Reindexing searchable entities")
+                        (search/reindex!)))]
+       (report->prometheus! report)
+       (log/infof "Done indexing in %.0fms %s" (u/since-ms timer) (sort-by (comp - val) report))))))
 
 (defn- update-index! []
   (when (search/supports-index?)

test/metabase/task/search_index_test.clj

+(ns metabase.task.search-index-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.search.appdb.index :as search.index]
+   [metabase.search.test-util :as search.tu]
+   [metabase.task.search-index :as task]
+   [toucan2.core :as t2]))
+
+(defn- index-size []
+  (t2/count search.index/*active-table*))
+
+;; TODO this is coupled to appdb engines at the moment
+(deftest reindex!-test
+  (search.tu/with-temp-index-table
+    (is (zero? (t2/count search.index/*active-table*)))
+    (testing "It can recreate the index from scratch"
+      (task/reindex! true)
+      (let [initial-size (index-size)]
+        (is (pos? initial-size))
+        (t2/delete! search.index/*active-table* (t2/select-one-pk search.index/*active-table*))
+        (is (= (dec initial-size) (index-size)))
+        (testing "It can cycle the index gracefully"
+          (task/reindex! false)
+          (is (= initial-size (index-size))))))))