diff --git a/src/metabase/search/postgres/index.clj b/src/metabase/search/postgres/index.clj index 9c134994b57f1c2c4b57ed9b264c0ccd8f6ce4a3..8c992ba10e8088bf8606f6cb8097ab28429a6fa8 100644 --- a/src/metabase/search/postgres/index.clj +++ b/src/metabase/search/postgres/index.clj @@ -78,24 +78,27 @@ (drop-table! retired-table) true)) +(defn- entity->entry [entity] + (-> entity + (select-keys + [:model + :model_rank + :collection_id + :database_id + :table_id + :archived]) + (assoc + :model_id (:id entity) + :search_vector [:to_tsvector + [:inline tsv-language] + [:cast + (:searchable_text entity) + :text]]))) + (defn update! - "Create or update the given search index trny" + "Create the given search index entries" [entity] - (let [entry (-> entity - (select-keys - [:model - :model_rank - :collection_id - :database_id - :table_id - :archived]) - (assoc - :model_id (:id entity) - :search_vector [:to_tsvector - [:inline tsv-language] - [:cast - (:searchable_text entity) - :text]]))] + (let [entry (entity->entry entity)] (when @initialized? (t2/insert! active-table entry)) (when @reindexing? @@ -150,6 +153,15 @@ (str/join " | ") maybe-complete))) +(defn batch-update! + "Create the given search index entries in bulk" + [entities] + (let [entries (map entity->entry entities)] + (when @initialized? + (t2/insert! active-table entries)) + (when @reindexing? + (t2/insert! pending-table entries)))) + (defn search-query "Query fragment for all models corresponding to a query paramter `:search-term`." [search-term] diff --git a/src/metabase/search/postgres/ingestion.clj b/src/metabase/search/postgres/ingestion.clj index bb50f288b3e2fba67176470e93276576b8caece0..634243ec5e34e32bec674bd69517f59e7e353624 100644 --- a/src/metabase/search/postgres/ingestion.clj +++ b/src/metabase/search/postgres/ingestion.clj @@ -11,6 +11,8 @@ [toucan2.core :as t2] [toucan2.realize :as t2.realize])) +(def ^:private insert-batch-size 50) + (def ^:private model-rankings (zipmap search.config/models-search-order (range))) @@ -40,7 +42,7 @@ (defn- search-items-reducible [] (-> {:search-string nil - :models search.config/all-models + :models (disj search.config/all-models "indexed-entity") ;; we want to see everything :is-superuser? true ;; irrelevant, as we're acting as a super user @@ -62,5 +64,6 @@ (eduction (comp (map t2.realize/realize) - (map ->entry))) - (run! search.index/update!))) + (map ->entry) + (partition-all insert-batch-size))) + (run! search.index/batch-update!)))