Skip to content
Snippets Groups Projects
Unverified Commit 0aa4246b authored by github-automation-metabase's avatar github-automation-metabase Committed by GitHub
Browse files

Basic test for search reindexing (#50560) (#50568)

parent 0617f368
No related branches found
No related tags found
No related merge requests found
......@@ -92,7 +92,7 @@
(search.ingestion/populate-index! :search.engine/fulltext)))
(defmethod search.engine/reindex! :search.engine/fulltext
[]
[_]
(search.index/ensure-ready! false)
(search.index/maybe-create-pending!)
(u/prog1 (search.ingestion/populate-index! :search.engine/fulltext)
......
......@@ -57,11 +57,13 @@
(defn maybe-create-pending!
"Create a search index table."
[]
(when (not @reindexing?)
(when-not (exists? pending-table)
(create-table! pending-table))
(reset! reindexing? true)))
([]
(when (not @reindexing?)
(maybe-create-pending! pending-table)
(reset! reindexing? true)))
([table-name]
(when-not (exists? table-name)
(create-table! table-name))))
(defn activate-pending!
"Make the pending index active if it exists. Returns true if it did so."
......@@ -113,9 +115,11 @@
"Create the given search index entries in bulk"
[documents]
(let [entries (map document->entry documents)
;; When stubbing the table in tests, always treat it as active
active? (or @initialized? (not= :search_index *active-table*))
;; Ideally, we would reset these atoms if the corresponding tables don't exist.
;; We're about to rework this area, so just leaving this as a note for now.
active-updated? (when @initialized? (safe-batch-upsert! *active-table* entries))
active-updated? (when active? (safe-batch-upsert! *active-table* entries))
pending-updated? (when @reindexing? (safe-batch-upsert! pending-table entries))]
(when (or active-updated? pending-updated?)
(->> entries (map :model) frequencies))))
......@@ -147,11 +151,17 @@
(defn reset-index!
"Ensure we have a blank slate; in case the table schema or stored data format has changed."
[]
(reset! reindexing? false)
(drop-table! pending-table)
(maybe-create-pending!)
(activate-pending!)
(reset! initialized? true))
;; Moving to random tables will clean this up
(let [testing? (not= *active-table* :search_index)
table-name (if testing? *active-table* pending-table)]
(when-not testing?
(reset! reindexing? false))
(drop-table! table-name)
(maybe-create-pending! table-name)
(when-not testing?
(activate-pending!)
(reset! initialized? true))
true))
(defn ensure-ready!
"Ensure the index is ready to be populated. Return false if it was already ready."
......
......@@ -35,22 +35,25 @@
(defn reindex!
"Reindex the whole AppDB"
[]
(when (search/supports-index?)
(let [timer (u/start-timer)
report (if (not @recreated?)
(do (log/info "Recreating search index from the latest schema")
;; Each instance in a multi-instance deployment will recreate the table the first time it is
;; selected to run the job, resulting in a momentary lack of search results. One solution to
;; this would be to store metadata about the index in another table, which we can use to
;; determine whether it was built by another version of Metabase and should be rebuilt.
(u/prog1 (search/init-index! {:force-reset? (not @recreated?)})
(reset! recreated? true)))
(do (log/info "Reindexing searchable entities")
(search/reindex!)))]
(report->prometheus! report)
(log/infof "Done indexing in %.0fms %s" (u/since-ms timer) (sort-by (comp - val) report)))))
([]
(let [recreate? (not @recreated?)]
(u/prog1 (reindex! recreate?)
(when recreate?
(reset! recreated? true)))))
([recreate?]
(when (search/supports-index?)
(let [timer (u/start-timer)
report (if recreate?
(do (log/info "Recreating search index from the latest schema")
;; Each instance in a multi-instance deployment will recreate the table the first time it is
;; selected to run the job, resulting in a momentary lack of search results. One solution to
;; this would be to store metadata about the index in another table, which we can use to
;; determine whether it was built by another version of Metabase and should be rebuilt.
(search/init-index! {:force-reset? recreate?}))
(do (log/info "Reindexing searchable entities")
(search/reindex!)))]
(report->prometheus! report)
(log/infof "Done indexing in %.0fms %s" (u/since-ms timer) (sort-by (comp - val) report))))))
(defn- update-index! []
(when (search/supports-index?)
......
(ns metabase.task.search-index-test
(:require
[clojure.test :refer :all]
[metabase.search.appdb.index :as search.index]
[metabase.search.test-util :as search.tu]
[metabase.task.search-index :as task]
[toucan2.core :as t2]))
(defn- index-size []
(t2/count search.index/*active-table*))
;; TODO this is coupled to appdb engines at the moment
(deftest reindex!-test
(search.tu/with-temp-index-table
(is (zero? (t2/count search.index/*active-table*)))
(testing "It can recreate the index from scratch"
(task/reindex! true)
(let [initial-size (index-size)]
(is (pos? initial-size))
(t2/delete! search.index/*active-table* (t2/select-one-pk search.index/*active-table*))
(is (= (dec initial-size) (index-size)))
(testing "It can cycle the index gracefully"
(task/reindex! false)
(is (= initial-size (index-size))))))))
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment