Skip to content
Snippets Groups Projects
Unverified Commit 22e7764f authored by Sameer Al-Sakran's avatar Sameer Al-Sakran Committed by GitHub
Browse files

Merge pull request #6855 from metabase/add-minmax-to-datetime-fingerprints 

Add latest and earliest to datetime fingerprints
parents d97214bc 344e45c5
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 92 additions and 12 deletions
frontend/test/__runner__/test_db_fixture.db.h2.db
+ 0
− 0
View file @ 22e7764f
No preview for this file type
frontend/test/services/__snapshots__/MetabaseApi.integ.spec.js.snap
+ 30
− 0
View file @ 22e7764f
......@@ -427,6 +427,12 @@ Object {
"global": Object {
"distinct-count": 1381,
},
"type": Object {
"type/DateTime": Object {
"earliest": "2015-11-05T00:00:00.000-08:00",
"latest": "2019-11-02T00:00:00.000-07:00",
},
},
},
"fingerprint_version": 1,
"fk_target_field_id": null,
......@@ -1571,6 +1577,12 @@ Object {
"global": Object {
"distinct-count": 2293,
},
"type": Object {
"type/DateTime": Object {
"earliest": "1957-11-06T00:00:00.000-08:00",
"latest": "1999-10-31T00:00:00.000-07:00",
},
},
},
"fingerprint_version": 1,
"fk_target_field_id": null,
......@@ -1787,6 +1799,12 @@ Object {
"global": Object {
"distinct-count": 980,
},
"type": Object {
"type/DateTime": Object {
"earliest": "2015-11-02T00:00:00.000-08:00",
"latest": "2018-11-02T00:00:00.000-07:00",
},
},
},
"fingerprint_version": 1,
"fk_target_field_id": null,
......@@ -2769,6 +2787,12 @@ Object {
"global": Object {
"distinct-count": 178,
},
"type": Object {
"type/DateTime": Object {
"earliest": "2015-11-07T00:00:00.000-08:00",
"latest": "2018-11-02T00:00:00.000-07:00",
},
},
},
"fingerprint_version": 1,
"fk_target_field_id": null,
......@@ -3611,6 +3635,12 @@ Object {
"global": Object {
"distinct-count": 624,
},
"type": Object {
"type/DateTime": Object {
"earliest": "2015-12-16T00:00:00.000-08:00",
"latest": "2019-11-02T00:00:00.000-07:00",
},
},
},
"fingerprint_version": 1,
"fk_target_field_id": null,
......
src/metabase/sync/analyze/fingerprint.clj
+ 4
− 2
View file @ 22e7764f
......@@ -9,6 +9,7 @@
[interface :as i]
[util :as sync-util]]
[metabase.sync.analyze.fingerprint
[datetime :as datetime]
[global :as global]
[number :as number]
[sample :as sample]
......@@ -22,8 +23,9 @@
"Return type-specific fingerprint info for FIELD AND. a FieldSample of Values if it has an elligible base type"
[field :- i/FieldInstance, values :- i/FieldSample]
(condp #(isa? %2 %1) (:base_type field)
:type/Text {:type/Text (text/text-fingerprint values)}
:type/Number {:type/Number (number/number-fingerprint values)}
:type/Text {:type/Text (text/text-fingerprint values)}
:type/Number {:type/Number (number/number-fingerprint values)}
:type/DateTime {:type/DateTime (datetime/datetime-fingerprint values)}
nil))
(s/defn ^:private fingerprint :- i/Fingerprint
......
src/metabase/sync/analyze/fingerprint/datetime.clj 0 → 100644
+ 21
− 0
View file @ 22e7764f
(ns metabase.sync.analyze.fingerprint.datetime
"Logic for generating a `DateTimeFingerprint` from a sequence of values for a `:type/DateTime` Field."
(:require [clj-time
[coerce :as t.coerce]
[core :as t]]
[medley.core :as m]
[metabase.sync.interface :as i]
[metabase.util :as u]
[redux.core :as redux]
[schema.core :as s]))
(s/defn datetime-fingerprint :- i/DateTimeFingerprint
"Generate a fingerprint containing information about values that belong to a `DateTime` Field."
[values :- i/FieldSample]
(transduce (map u/str->date-time)
(redux/post-complete
(redux/fuse {:earliest t/min-date
:latest t/max-date})
(partial m/map-vals str))
[(t.coerce/from-long Long/MAX_VALUE) (t.coerce/from-long 0)]
values))
src/metabase/sync/interface.clj
+ 8
− 2
View file @ 22e7764f
......@@ -108,11 +108,17 @@
(s/optional-key :percent-email) Percent
(s/optional-key :average-length) (s/constrained Double #(>= % 0) "Valid number greater than or equal to zero")})
(def DateTimeFingerprint
"Schema for fingerprint information for Fields deriving from `:type/DateTime`."
{(s/optional-key :earliest) s/Str
(s/optional-key :latest) s/Str})
(def TypeSpecificFingerprint
"Schema for type-specific fingerprint information."
(s/constrained
{(s/optional-key :type/Number) NumberFingerprint
(s/optional-key :type/Text) TextFingerprint}
{(s/optional-key :type/Number) NumberFingerprint
(s/optional-key :type/Text) TextFingerprint
(s/optional-key :type/DateTime) DateTimeFingerprint}
(fn [m]
(= 1 (count (keys m))))
"Type-specific fingerprint with exactly one key"))
......
test/metabase/driver/postgres_test.clj
+ 4
− 3
View file @ 22e7764f
......@@ -282,11 +282,12 @@
"UTC"
(tu/db-timezone-id))
;; Make sure we're able to fingerprint TIME fields (#5911)
(expect-with-engine :postgres
#{#metabase.models.field.FieldInstance{:name "start_time", :fingerprint {:global {:distinct-count 1}}}
#metabase.models.field.FieldInstance{:name "end_time", :fingerprint {:global {:distinct-count 1}}}
#{#metabase.models.field.FieldInstance{:name "start_time", :fingerprint {:global {:distinct-count 1}
:type {:type/DateTime {:earliest "1970-01-01T22:00:00.000Z", :latest "1970-01-01T22:00:00.000Z"}}}}
#metabase.models.field.FieldInstance{:name "end_time", :fingerprint {:global {:distinct-count 1}
:type {:type/DateTime {:earliest "1970-01-01T09:00:00.000Z", :latest "1970-01-01T09:00:00.000Z"}}}}
#metabase.models.field.FieldInstance{:name "reason", :fingerprint {:global {:distinct-count 1}
:type {:type/Text {:percent-json 0.0
:percent-url 0.0
......
test/metabase/query_processor/expand_resolve_test.clj
+ 9
− 3
View file @ 22e7764f
......@@ -235,7 +235,9 @@
:special-type nil
:table-id (id :users)
:table-name "USERS__via__USER_ID"
:fingerprint {:global {:distinct-count 11}}})
:fingerprint {:global {:distinct-count 11}
:type {:type/DateTime {:earliest "2014-01-01T00:00:00.000Z"
:latest "2014-12-05T00:00:00.000Z"}}}})
:unit :year}
:value {:value (u/->Timestamp "1980-01-01")
:field {:field
......@@ -250,7 +252,9 @@
:visibility-type :normal
:table-id (id :users)
:table-name "USERS__via__USER_ID"
:fingerprint {:global {:distinct-count 11}}})
:fingerprint {:global {:distinct-count 11}
:type {:type/DateTime {:earliest "2014-01-01T00:00:00.000Z"
:latest "2014-12-05T00:00:00.000Z"}}}})
:unit :year}}}
:join-tables [{:source-field {:field-id (id :checkins :user_id)
:field-name "USER_ID"}
......@@ -314,7 +318,9 @@
:field-id true
:table-name "CHECKINS"
:schema-name "PUBLIC"
:fingerprint {:global {:distinct-count 618}}})
:fingerprint {:global {:distinct-count 618}
:type {:type/DateTime {:earliest "2013-01-03T00:00:00.000Z"
:latest "2015-12-29T00:00:00.000Z"}}}})
:unit :day-of-week}]
:join-tables [{:source-field {:field-id true
:field-name "VENUE_ID"}
......
test/metabase/query_processor_test.clj
+ 3
− 1
View file @ 22e7764f
......@@ -155,7 +155,9 @@
:name (data/format-name "last_login")
:display_name "Last Login"
:unit :default
:fingerprint {:global {:distinct-count 11}}})))
:fingerprint {:global {:distinct-count 11}
:type {:type/DateTime {:earliest "2014-01-01T00:00:00.000Z"
:latest "2014-12-05T00:00:00.000Z"}}}})))
;; #### venues
(defn venues-columns
......
test/metabase/sync/analyze/fingerprint/datetime_test.clj 0 → 100644
+ 9
− 0
View file @ 22e7764f
(ns metabase.sync.analyze.fingerprint.datetime-test
(:require [clj-time.core :as t]
[expectations :refer :all]
[metabase.sync.analyze.fingerprint.datetime :refer :all]))
(expect
{:earliest "2013-01-01T00:00:00.000Z"
:latest "2018-01-01T00:00:00.000Z"}
(datetime-fingerprint ["2013" "2018" "2015"]))
test/metabase/sync/analyze/fingerprint_test.clj
+ 4
− 1
View file @ 22e7764f
......@@ -8,6 +8,7 @@
[metabase.sync.analyze.fingerprint.sample :as sample]
[metabase.sync.interface :as i]
[metabase.test.data :as data]
[metabase.test.util]
[metabase.util :as u]
[toucan.db :as db]
[toucan.util.test :as tt]))
......@@ -36,7 +37,9 @@
;; a datetime field
(expect
{:global {:distinct-count 618}}
{:global {:distinct-count 618}
:type {:type/DateTime {:earliest "2013-01-03T00:00:00.000Z"
:latest "2015-12-29T00:00:00.000Z"}}}
(fingerprint (Field (data/id :checkins :date))))
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment