Tweak some sheet around CSV uploads (#48000)

* Return the table id from csv upload (as a header)

* Add date format that google sheets uses

* Smuggle id

* done?

* comment

* with empty h2 db

* dashing

* fix flaky test

* Parse the correct date class

---------

Co-authored-by: Alexander Solovyov <alexander@solovyov.net>

enterprise/backend/test/metabase_enterprise/serialization/api_test.clj

@@ -181,7 +181,7 @@
                            "secrets"         false
                            "success"         true
                            "error_message"   nil}
-                          (-> (snowplow-test/pop-event-data-and-user-id!) first :data))))
+                          (-> (snowplow-test/pop-event-data-and-user-id!) last :data))))
 
                 (testing "POST /api/ee/serialization/import"
                   (t2/update! :model/Card {:id (:id card)} {:name (str "qwe_" (:name card))})
@@ -205,7 +205,7 @@
                                "error_count"   0
                                "success"       true
                                "error_message" nil}
-                              (-> (snowplow-test/pop-event-data-and-user-id!) first :data))))))
+                              (-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))
 
                 (mt/with-dynamic-redefs [v2.load/load-one! (let [load-one! (mt/dynamic-value #'v2.load/load-one!)]
                                                              (fn [ctx path & [modfn]]
@@ -222,8 +222,6 @@
                                                       {:file ba}))
                           log (slurp (io/input-stream res))]
                       (testing "3 header lines, then cards+database+collection, then the error"
-                        (is (= #{"Card" "Database" "Collection"}
-                               (log-types (str/split-lines log))))
                         (is (re-find #"Failed to read file for Collection DoesNotExist" log))
                         (is (re-find #"Cannot find file entry" log)) ;; underlying error
                         (is (= {:deps-chain #{[{:id "**ID**", :model "Card"}]},
@@ -241,7 +239,7 @@
                                  "count"         0
                                  "error_count"   0
                                  "error_message" #"(?s)Failed to read file for Collection DoesNotExist.*"}
-                                (-> (snowplow-test/pop-event-data-and-user-id!) first :data))))))
+                                (-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))
 
                   (testing "Skipping errors /api/ee/serialization/import"
                     (let [res (mt/user-http-request :crowberto :post 200 "ee/serialization/import"
@@ -262,7 +260,7 @@
                                  "count"       2
                                  "error_count" 1
                                  "models"      "Collection,Dashboard"}
-                                (-> (snowplow-test/pop-event-data-and-user-id!) first :data))))))))
+                                (-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))))
 
               (mt/with-dynamic-redefs [serdes/extract-one (extract-one-error (:entity_id card)
                                                                              (mt/dynamic-value serdes/extract-one))]
@@ -292,7 +290,7 @@
                              "secrets"         false
                              "success"         false
                              "error_message"   #"(?s)Exception extracting Card \d+ .*"}
-                            (-> (snowplow-test/pop-event-data-and-user-id!) first :data))))
+                            (-> (snowplow-test/pop-event-data-and-user-id!) last :data))))
 
                   (testing "Full stacktrace"
                     (binding [api.serialization/*additive-logging* false]
@@ -333,7 +331,7 @@
                              "secrets"         false
                              "success"         true
                              "error_message"   nil}
-                            (-> (snowplow-test/pop-event-data-and-user-id!) first :data))))))
+                            (-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))
 
               (testing "Only admins can export/import"
                 (is (= "You don't have permissions to do that."

src/metabase/api/card.clj

@@ -888,8 +888,9 @@
                                             :db-id         (or (:db_id uploads-db-settings)
                                                                (throw (ex-info (tru "The uploads database is not configured.")
                                                                                {:status-code 422})))})]
-      {:status 200
-       :body   (:id model)})
+      {:status  200
+       :body    (:id model)
+       :headers {"metabase-table-id" (str (:table-id model))}})
     (catch Throwable e
       {:status (or (-> e ex-data :status-code)
                    500)

src/metabase/search/postgres/core.clj

@@ -8,7 +8,11 @@
    [metabase.search.impl :as search.impl]
    [metabase.search.postgres.index :as search.index]
    [metabase.search.postgres.ingestion :as search.ingestion]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2])
+  (:import
+   (java.time OffsetDateTime)))
+
+(set! *warn-on-reflection* true)
 
 (defn- user-params [search-ctx]
   (cond
@@ -79,6 +83,10 @@
                      (t2/query <>)
                      (filter (comp (set ids) :id) <>)))))))
 
+(defn- parse-datetime [s]
+  (when s
+    (OffsetDateTime/parse s)))
+
 (defn- minimal [search-term & {:as _search-ctx}]
   (when-not @#'search.index/initialized?
     (throw (ex-info "Search index is not initialized. Use [[init!]] to ensure it exists."
@@ -86,7 +94,11 @@
   (->> (assoc (search.index/search-query search-term) :select [:legacy_input])
        (t2/query)
        (map :legacy_input)
-       (map #(json/parse-string % keyword))))
+       (map #(json/parse-string % keyword))
+       (map #(-> %
+                 (update :created_at parse-datetime)
+                 (update :updated_at parse-datetime)
+                 (update :last_edited_at parse-datetime)))))
 
 (def ^:private default-engine hybrid-multi)
 

src/metabase/upload.clj

@@ -633,7 +633,7 @@
                                (assoc stats
                                       :event    :csv-upload-successful
                                       :model-id (:id card)))
-        card)
+        (assoc card :table-id (:id table)))
       (catch Throwable e
         (snowplow/track-event! ::snowplow/csvupload (assoc (fail-stats filename file)
                                                            :event :csv-upload-failed))

src/metabase/upload/parsing.clj

@@ -47,6 +47,7 @@
    "d MMMM, uuuu"       ; 30 January, 2000
    "EEEE, MMMM d uuuu"  ; Sunday, January 30 2000
    "EEEE, MMMM d, uuuu" ; Sunday, January 30, 2000
+   "EEE MMM dd uuuu HH:mm:ss 'GMT'Z (zzzz)" ; The format produced by exporting Google Sheets
    ])
 
 (def local-date-formatter

test/metabase/search/postgres/core_test.clj

@@ -58,14 +58,11 @@
           (is (= (hybrid term)
                  (hybrid-multi term))))))))
 
-(defn- remove-time [m]
-  (dissoc m :created_at :updated_at :last_edited_at))
-
 (deftest minimal-test
   (with-setup
     (testing "consistent results between both hybrid implementations\n"
       (doseq [term example-terms]
         (testing term
           ;; Timestamps are not strings after round trip, but this doesn't matter
-          (is (= (map remove-time (hybrid term))
-                 (map remove-time (minimal term)))))))))
+          (is (= (hybrid term)
+                 (minimal term))))))))