Sensitive Fields 2.0 (preprocessing implementation)

resources/frontend_client/app/admin/databases/databases.controllers.js

@@ -229,12 +229,6 @@ DatabasesControllers.controller('DatabaseTables', ['$scope', '$routeParams', '$l
                     Metabase.db_tables({ 'dbId': $routeParams.databaseId }).$promise
                     .then(function(tables) {
                         $scope.tables = tables;
-                        return $q.all(tables.map(function(table) {
-                            return Metabase.table_query_metadata({ 'tableId': table.id }).$promise
-                            .then(function(result) {
-                                $scope.tableFields[table.id] = result;
-                            });
-                        }));
                     })
                 ]);
             }
@@ -261,7 +255,8 @@ DatabasesControllers.controller('DatabaseTable', ['$scope', '$routeParams', '$lo
 
         function loadData() {
             Metabase.table_query_metadata({
-                'tableId': $routeParams.tableId
+                'tableId': $routeParams.tableId,
+                'include_sensitive_fields': true
             }, function(result) {
                 $scope.table = result;
                 $scope.getIdFields();

resources/frontend_client/app/services.js

@@ -456,6 +456,9 @@ CorvusServices.service('CorvusCore', ['$resource', 'User', function($resource, U
     }, {
         'id': 'dimension',
         'name': 'Dimension'
+    }, {
+        'id': 'sensitive',
+        'name': 'Sensitive Information'
     }];
 
     this.boolean_types = [{

src/metabase/api/common.clj

@@ -338,7 +338,7 @@
   [symb value :nillable]
   (try (Integer/parseInt value)
        (catch java.lang.NumberFormatException _
-         (format "Invalid value '%s' for '%s': cannot parse as an integer." value symb))))
+         (format "Invalid value '%s' for '%s': cannot parse as an integer." value symb)))) ; TODO - why aren't we re-throwing these exceptions ?
 
 (defannotation String->Dict
   "Param is converted from a JSON string to a dictionary."
@@ -347,6 +347,15 @@
        (catch java.lang.Exception _
          (format "Invalid value '%s' for '%s': cannot parse as json." value symb))))
 
+(defannotation String->Boolean
+  "Param is converted from `\"true\"` or `\"false\"` to the corresponding boolean."
+  [symb value :nillable]
+  (cond
+    (= value "true")  true
+    (= value "false") false
+    (nil? value)      nil
+    :else             (throw (ApiFieldValidationException. (name symb) (format "'%s' is not a valid boolean." value)))))
+
 (defannotation Integer
   "Param must be an integer (this does *not* cast the param)."
   [symb value :nillable]

src/metabase/api/meta/table.clj

@@ -54,15 +54,25 @@
   "Get all `Fields` for `Table` with ID."
   [id]
   (read-check Table id)
-  (sel :many Field :table_id id :active true (order :name :ASC)))
+  (sel :many Field :table_id id, :active true, :field_type [not= "sensitive"], (order :name :ASC)))
 
 (defendpoint GET "/:id/query_metadata"
   "Get metadata about a `Table` useful for running queries.
-   Returns DB, fields, field FKs, and field values."
-  [id]
+   Returns DB, fields, field FKs, and field values.
+
+  By passing `include_sensitive_fields=true`, information *about* sensitive `Fields` will be returned; in no case
+  will any of its corresponding values be returned. (This option is provided for use in the Admin Edit Metadata page)."
+  [id include_sensitive_fields]
+  {include_sensitive_fields String->Boolean}
   (->404 (sel :one Table :id id)
          read-check
-         (hydrate :db [:fields [:target]] :field_values)))
+         (hydrate :db [:fields [:target]] :field_values)
+         (update-in [:fields] (if include_sensitive_fields
+                                ;; If someone passes include_sensitive_fields return hydrated :fields as-is
+                                identity
+                                ;; Otherwise filter out all :sensitive fields
+                                (partial filter (fn [{:keys [field_type]}]
+                                                  (not= (keyword field_type) :sensitive)))))))
 
 (defendpoint GET "/:id/fks"
   "Get all `ForeignKeys` whose destination is a `Field` that belongs to this `Table`."

src/metabase/models/field.clj

@@ -61,10 +61,11 @@
     :UnknownField})
 
 (def ^:const field-types
-  "Not sure what this is for"
-  #{:metric
-    :dimension
-    :info})
+  "Possible values for `Field.field_type`."
+  #{:metric      ; A number that can be added, graphed, etc.
+    :dimension   ; A high or low-cardinality numerical string value that is meant to be used as a grouping
+    :info        ; Non-numerical value that is not meant to be used
+    :sensitive}) ; A Fields that should *never* be shown *anywhere*
 
 (defentity Field
   (table :metabase_field)
@@ -118,8 +119,9 @@
 (defmethod post-update Field [_ {:keys [id] :as field}]
   ;; if base_type or special_type were affected then we should asynchronously create corresponding FieldValues objects if need be
   (when (or (contains? field :base_type)
+            (contains? field :field_type)
             (contains? field :special_type))
-    (future (create-field-values-if-needed (sel :one [Field :id :table_id :base_type :special_type] :id id)))))
+    (future (create-field-values-if-needed (sel :one [Field :id :table_id :base_type :special_type :field_type] :id id)))))
 
 (defmethod pre-cascade-delete Field [_ {:keys [id]}]
   (cascade-delete ForeignKey (where (or (= :origin_id id)

src/metabase/models/field_values.clj

@@ -29,11 +29,13 @@
 (defn field-should-have-field-values?
   "Should this `Field` be backed by a corresponding `FieldValues` object?"
   {:arglists '([field])}
-  [{:keys [base_type special_type] :as field}]
-  {:pre [(contains? field :base_type)
+  [{:keys [base_type special_type field_type] :as field}]
+  {:pre [field_type
+         (contains? field :base_type)
          (contains? field :special_type)]}
-  (or (contains? #{:category :city :state :country} (keyword special_type))
-      (= (keyword base_type) :BooleanField)))
+  (and (not= (keyword field_type) :sensitive)
+       (or (contains? #{:category :city :state :country} (keyword special_type))
+           (= (keyword base_type) :BooleanField))))
 
 (def ^:private field-distinct-values
   (u/runtime-resolved-fn 'metabase.db.metadata-queries 'field-distinct-values))

src/metabase/models/table.clj

@@ -28,9 +28,7 @@
                                                     :active true
                                                     (order :position :asc)
                                                     (order :name :asc))]
-                                 (->> (sel :many FieldValues :field_id [in field-ids])
-                                      (map (fn [{:keys [field_id values]}] {field_id values}))
-                                      (apply merge))))
+                                 (sel :many :field->field [FieldValues :field_id :values] :field_id [in field-ids])))
                :description  (u/jdbc-clob->str description)
                :pk_field     (delay (:id (sel :one :fields [Field :id] :table_id id (where {:special_type "id"}))))
                :can_read     (delay @(:can_read @(:db <>)))

test/metabase/api/meta/table_test.clj

@@ -144,6 +144,193 @@
   ((user->client :rasta) :get 200 (format "meta/table/%d/query_metadata" (table->id :categories))))
 
 
+;;; GET api/meta/table/:id/query_metadata?include_sensitive_fields
+;;; Make sure that getting the User table *does* include info about the password field, but not actual values themselves
+(expect
+    (match-$ (sel :one Table :id (table->id :users))
+      {:description nil
+       :entity_type nil
+       :db (match-$ @test-db
+             {:created_at $
+              :engine "h2"
+              :id $
+              :details $
+              :updated_at $
+              :name "Test Database"
+              :organization_id @org-id
+              :description nil})
+       :name "USERS"
+       :fields [(match-$ (sel :one Field :id (field->id :users :id))
+                  {:description nil
+                   :table_id (table->id :users)
+                   :special_type "id"
+                   :name "ID"
+                   :updated_at $
+                   :active true
+                   :id $
+                   :field_type "info"
+                   :position 0
+                   :target nil
+                   :preview_display true
+                   :created_at $
+                   :base_type "BigIntegerField"})
+                (match-$ (sel :one Field :id (field->id :users :last_login))
+                  {:description nil
+                   :table_id (table->id :users)
+                   :special_type "category"
+                   :name "LAST_LOGIN"
+                   :updated_at $
+                   :active true
+                   :id $
+                   :field_type "info"
+                   :position 0
+                   :target nil
+                   :preview_display true
+                   :created_at $
+                   :base_type "DateTimeField"})
+                (match-$ (sel :one Field :id (field->id :users :name))
+                  {:description nil
+                   :table_id (table->id :users)
+                   :special_type "category"
+                   :name "NAME"
+                   :updated_at $
+                   :active true
+                   :id $
+                   :field_type "info"
+                   :position 0
+                   :target nil
+                   :preview_display true
+                   :created_at $
+                   :base_type "TextField"})
+                (match-$ (sel :one Field :table_id (table->id :users) :name "PASSWORD")
+                  {:description nil
+                   :table_id (table->id :users)
+                   :special_type "category"
+                   :name "PASSWORD"
+                   :updated_at $
+                   :active true
+                   :id $
+                   :field_type "sensitive"
+                   :position 0
+                   :target nil
+                   :preview_display true
+                   :created_at $
+                   :base_type "TextField"})]
+       :rows 15
+       :updated_at $
+       :entity_name nil
+       :active true
+       :id (table->id :users)
+       :db_id @db-id
+       :field_values {(keyword (str (field->id :users :last_login)))
+                      user-last-login-date-strs
+
+                      (keyword (str (field->id :users :name)))
+                      ["Broen Olujimi"
+                       "Conchúr Tihomir"
+                       "Dwight Gresham"
+                       "Felipinho Asklepios"
+                       "Frans Hevel"
+                       "Kaneonuskatew Eiran"
+                       "Kfir Caj"
+                       "Nils Gotam"
+                       "Plato Yeshua"
+                       "Quentin Sören"
+                       "Rüstem Hebel"
+                       "Shad Ferdynand"
+                       "Simcha Yan"
+                       "Spiros Teofil"
+                       "Szymon Theutrich"]}
+       :created_at $})
+  ((user->client :rasta) :get 200 (format "meta/table/%d/query_metadata?include_sensitive_fields=true" (table->id :users))))
+
+;;; GET api/meta/table/:id/query_metadata
+;;; Make sure that getting the User table does *not* include password info
+(expect
+    (match-$ (sel :one Table :id (table->id :users))
+      {:description nil
+       :entity_type nil
+       :db (match-$ @test-db
+             {:created_at $
+              :engine "h2"
+              :id $
+              :details $
+              :updated_at $
+              :name "Test Database"
+              :organization_id @org-id
+              :description nil})
+       :name "USERS"
+       :fields [(match-$ (sel :one Field :id (field->id :users :id))
+                  {:description nil
+                   :table_id (table->id :users)
+                   :special_type "id"
+                   :name "ID"
+                   :updated_at $
+                   :active true
+                   :id $
+                   :field_type "info"
+                   :position 0
+                   :target nil
+                   :preview_display true
+                   :created_at $
+                   :base_type "BigIntegerField"})
+                (match-$ (sel :one Field :id (field->id :users :last_login))
+                  {:description nil
+                   :table_id (table->id :users)
+                   :special_type "category"
+                   :name "LAST_LOGIN"
+                   :updated_at $
+                   :active true
+                   :id $
+                   :field_type "info"
+                   :position 0
+                   :target nil
+                   :preview_display true
+                   :created_at $
+                   :base_type "DateTimeField"})
+                (match-$ (sel :one Field :id (field->id :users :name))
+                  {:description nil
+                   :table_id (table->id :users)
+                   :special_type "category"
+                   :name "NAME"
+                   :updated_at $
+                   :active true
+                   :id $
+                   :field_type "info"
+                   :position 0
+                   :target nil
+                   :preview_display true
+                   :created_at $
+                   :base_type "TextField"})]
+       :rows 15
+       :updated_at $
+       :entity_name nil
+       :active true
+       :id (table->id :users)
+       :db_id @db-id
+       :field_values {(keyword (str (field->id :users :last_login)))
+                      user-last-login-date-strs
+
+                      (keyword (str (field->id :users :name)))
+                      ["Broen Olujimi"
+                       "Conchúr Tihomir"
+                       "Dwight Gresham"
+                       "Felipinho Asklepios"
+                       "Frans Hevel"
+                       "Kaneonuskatew Eiran"
+                       "Kfir Caj"
+                       "Nils Gotam"
+                       "Plato Yeshua"
+                       "Quentin Sören"
+                       "Rüstem Hebel"
+                       "Shad Ferdynand"
+                       "Simcha Yan"
+                       "Spiros Teofil"
+                       "Szymon Theutrich"]}
+       :created_at $})
+  ((user->client :rasta) :get 200 (format "meta/table/%d/query_metadata" (table->id :users))))
+
+
 ;; ## PUT /api/meta/table/:id
 (expect-eval-actual-first
     (match-$ (let [table (sel :one Table :id (table->id :users))]

test/metabase/driver/mongo/test_data.clj

@@ -15,7 +15,7 @@
             [metabase.test-data.data :as data]))
 
 (declare load-data
-         set-field-special-types!)
+         set-field-types!)
 
 ;; ## CONSTANTS
 
@@ -53,7 +53,7 @@
                   (log/info (color/cyan "Loading Mongo test data..."))
                   (load-data)
                   (driver/sync-database! db)
-                  (set-field-special-types!)
+                  (set-field-types!)
                   (log/info (color/cyan "Done."))
                   db))]
      (assert (and (map? db)
@@ -135,13 +135,17 @@
             (catch com.mongodb.MongoException$DuplicateKey _)))
         (log/info (color/cyan (format "Loaded data for collection '%s'." (name collection))))))))
 
-(defn- set-field-special-types! []
+(defn- set-field-types! []
   (doseq [[collection-name {fields :fields}] data/test-data]
-    (doseq [{:keys [special-type] :as field} fields]
-      (when special-type
+    (doseq [{:keys [special-type field-type] :as field} fields]
+      (when (or field-type special-type)
         (let [table-id (sel :one :id Table :name (name collection-name))
               _        (assert (integer? table-id))
               field-id (sel :one :id Field :table_id table-id :name (name (:name field)))
               _        (assert (integer? table-id))]
-          (log/info (format "SET SPECIAL TYPE %s.%s -> %s..." collection-name (:name field) special-type))
-          (upd Field field-id :special_type special-type))))))
+          (when special-type
+            (log/info (format "SET SPECIAL TYPE %s.%s -> %s..." collection-name (:name field) special-type))
+            (upd Field field-id :special_type special-type))
+          (when field-type
+            (log/info (format "SET FIELD TYPE %s.%s -> %s..." collection-name (:name field) field-type))
+            (upd Field field-id :field_type field-type)))))))

test/metabase/driver/mongo_test.clj

@@ -118,10 +118,10 @@
 
 ;; ### table->column-names
 (expect-when-testing-mongo
-    [#{:_id :name}
-     #{:_id :date :venue_id :user_id}
-     #{:_id :name :last_login}
-     #{:_id :name :longitude :latitude :price :category_id}]
+    [#{:_id :name}                                           ; categories
+     #{:_id :date :venue_id :user_id}                        ; checkins
+     #{:_id :name :last_login :password}                     ; users
+     #{:_id :name :longitude :latitude :price :category_id}] ; venues
   (->> table-names
        (map table-name->fake-table)
        (map table->column-names)))
@@ -149,10 +149,10 @@
 
 ;; ### active-column-names->type
 (expect-when-testing-mongo
-    [{"_id" :IntegerField, "name" :TextField}
-     {"_id" :IntegerField, "date" :DateField, "venue_id" :IntegerField, "user_id" :IntegerField}
-     {"_id" :IntegerField, "name" :TextField, "last_login" :DateField}
-     {"_id" :IntegerField, "name" :TextField, "longitude" :FloatField, "latitude" :FloatField, "price" :IntegerField, "category_id" :IntegerField}]
+    [{"_id" :IntegerField, "name" :TextField}                                                                                                       ; categories
+     {"_id" :IntegerField, "date" :DateField, "venue_id" :IntegerField, "user_id" :IntegerField}                                                    ; checkins
+     {"_id" :IntegerField, "password" :TextField, "name" :TextField, "last_login" :DateField}                                                       ; users
+     {"_id" :IntegerField, "name" :TextField, "longitude" :FloatField, "latitude" :FloatField, "price" :IntegerField, "category_id" :IntegerField}] ; venues
   (->> table-names
        (map table-name->fake-table)
        (mapv (partial i/active-column-names->type mongo/driver))))
@@ -169,29 +169,33 @@
 
 ;; Test that Tables got synced correctly, and row counts are correct
 (expect-when-testing-mongo
-    [{:rows 75, :active true, :name "categories"}
+    [{:rows 75,   :active true, :name "categories"}
      {:rows 1000, :active true, :name "checkins"}
-     {:rows 15, :active true, :name "users"}
-     {:rows 100, :active true, :name "venues"}]
+     {:rows 15,   :active true, :name "users"}
+     {:rows 100,  :active true, :name "venues"}]
   (sel :many :fields [Table :name :active :rows] :db_id @mongo-test-db-id (k/order :name)))
 
 ;; Test that Fields got synced correctly, and types are correct
 (expect-when-testing-mongo
-    [[{:special_type :id, :base_type :IntegerField, :name "_id"}
-      {:special_type :category, :base_type :DateField, :name "last_login"}
-      {:special_type :category, :base_type :TextField, :name "name"}]
-     [{:special_type :id, :base_type :IntegerField, :name "_id"}
-      {:special_type :category, :base_type :DateField, :name "last_login"}
-      {:special_type :category, :base_type :TextField, :name "name"}]
-     [{:special_type :id, :base_type :IntegerField, :name "_id"}
-      {:special_type :category, :base_type :DateField, :name "last_login"}
-      {:special_type :category, :base_type :TextField, :name "name"}]
-     [{:special_type :id, :base_type :IntegerField, :name "_id"}
-      {:special_type :category, :base_type :DateField, :name "last_login"}
-      {:special_type :category, :base_type :TextField, :name "name"}]]
+    [[{:special_type :id,        :base_type :IntegerField, :name "_id"}
+      {:special_type :name,      :base_type :TextField,    :name "name"}]
+     [{:special_type :id,        :base_type :IntegerField, :name "_id"}
+      {:special_type nil,        :base_type :DateField,    :name "date"}
+      {:special_type :category,  :base_type :IntegerField, :name "user_id"}
+      {:special_type nil,        :base_type :IntegerField, :name "venue_id"}]
+     [{:special_type :id,        :base_type :IntegerField, :name "_id"}
+      {:special_type :category,  :base_type :DateField,    :name "last_login"}
+      {:special_type :category,  :base_type :TextField,    :name "name"}
+      {:special_type :category,  :base_type :TextField,    :name "password"}]
+     [{:special_type :id,        :base_type :IntegerField, :name "_id"}
+      {:special_type :category,  :base_type :IntegerField, :name "category_id"}
+      {:special_type :latitude,  :base_type :FloatField,   :name "latitude"}
+      {:special_type :longitude, :base_type :FloatField,   :name "longitude"}
+      {:special_type :name,      :base_type :TextField,    :name "name"}
+      {:special_type :category,  :base_type :IntegerField, :name "price"}]]
   (let [table->fields (fn [table-name]
                         (sel :many :fields [Field :name :base_type :special_type]
                              :active true
-                             :table_id (table-name->id :users)
+                             :table_id (sel :one :id Table :db_id @mongo-test-db-id, :name (name table-name))
                              (k/order :name)))]
     (map table->fields table-names)))

test/metabase/driver/query_processor_test.clj

@@ -726,8 +726,8 @@
    :breakout     [(id :venues :price)]
    :order_by     [[["aggregation" 0] "descending"]]})
 
-;;; ### make sure that rows where preview_display = false don't get displayed
 
+;;; ### make sure that rows where preview_display = false don't get displayed
 (datasets/expect-with-all-datasets
  [(set (->columns "category_id" "name" "latitude" "id" "longitude" "price"))
   (set (->columns "category_id" "name" "latitude" "id" "longitude"))
@@ -746,3 +746,32 @@
         (get-col-names))
     (do (upd Field (id :venues :price) :preview_display true)
         (get-col-names))]))
+
+
+;;; ## :sensitive fields
+;;; Make sure :sensitive information fields are never returned by the QP
+(qp-expect-with-all-datasets
+ {:columns (->columns "id"
+                      "last_login"
+                      "name")
+  :cols [(users-col :id)
+         (users-col :last_login)
+         (users-col :name)]
+  :rows [[ 1 #inst "2014-04-01T08:30:00.000000000-00:00" "Plato Yeshua"]
+         [ 2 #inst "2014-12-05T15:15:00.000000000-00:00" "Felipinho Asklepios"]
+         [ 3 #inst "2014-11-06T16:15:00.000000000-00:00" "Kaneonuskatew Eiran"]
+         [ 4 #inst "2014-01-01T08:30:00.000000000-00:00" "Simcha Yan"]
+         [ 5 #inst "2014-10-03T17:30:00.000000000-00:00" "Quentin Sören"]
+         [ 6 #inst "2014-08-02T12:30:00.000000000-00:00" "Shad Ferdynand"]
+         [ 7 #inst "2014-08-02T09:30:00.000000000-00:00" "Conchúr Tihomir"]
+         [ 8 #inst "2014-02-01T10:15:00.000000000-00:00" "Szymon Theutrich"]
+         [ 9 #inst "2014-04-03T09:30:00.000000000-00:00" "Nils Gotam"]
+         [10 #inst "2014-07-03T19:30:00.000000000-00:00" "Frans Hevel"]
+         [11 #inst "2014-11-01T07:00:00.000000000-00:00" "Spiros Teofil"]
+         [12 #inst "2014-07-03T01:30:00.000000000-00:00" "Kfir Caj"]
+         [13 #inst "2014-08-01T10:30:00.000000000-00:00" "Dwight Gresham"]
+         [14 #inst "2014-10-03T13:45:00.000000000-00:00" "Broen Olujimi"]
+         [15 #inst "2014-08-01T12:45:00.000000000-00:00" "Rüstem Hebel"]]}
+ {:source_table (id :users)
+  :aggregation  ["rows"]
+  :order_by     [[(id :users :id) "ascending"]]})

test/metabase/models/field_values_test.clj

@@ -6,16 +6,25 @@
 
 (expect true
   (field-should-have-field-values? {:special_type :category
+                                    :field_type :info
+                                    :base_type :TextField}))
+
+(expect false
+  (field-should-have-field-values? {:special_type :category
+                                    :field_type :sensitive
                                     :base_type :TextField}))
 
 (expect false
   (field-should-have-field-values? {:special_type nil
+                                    :field_type :info
                                     :base_type :TextField}))
 
 (expect true
   (field-should-have-field-values? {:special_type "country"
+                                    :field_type :info
                                     :base_type :TextField}))
 
 (expect true
   (field-should-have-field-values? {:special_type nil
+                                    :field_type :info
                                     :base_type "BooleanField"}))

test/metabase/test_data.clj

@@ -28,6 +28,7 @@
 ;;    *  id
 ;;    *  name
 ;;    *  last_login
+;;    *  password (sensitive)
 ;; *  categories - 75 rows
 ;;    *  id
 ;;    *  name

test/metabase/test_data/data.clj

@@ -12,21 +12,21 @@
 
 ;; [name last_login]
 (defonce ^:private users
-  [["Plato Yeshua" (timestamp 2014 4 1 1 30)]
-   ["Felipinho Asklepios" (timestamp 2014 12 5 7 15)]
-   ["Kaneonuskatew Eiran" (timestamp 2014 11 6 8 15)]
-   ["Simcha Yan" (timestamp 2014 1 1 0 30)]
-   ["Quentin Sören" (timestamp 2014 10 3 10 30)]
-   ["Shad Ferdynand" (timestamp 2014 8 2 5 30)]
-   ["Conchúr Tihomir" (timestamp 2014 8 2 2 30)]
-   ["Szymon Theutrich" (timestamp 2014 2 1 2 15)]
-   ["Nils Gotam" (timestamp 2014 4 3 2 30)]
-   ["Frans Hevel" (timestamp 2014 7 3 12 30)]
-   ["Spiros Teofil" (timestamp 2014 11 1)]
-   ["Kfir Caj" (timestamp 2014 7 2 18 30)]
-   ["Dwight Gresham" (timestamp 2014 8 1 3 30)]
-   ["Broen Olujimi" (timestamp 2014 10 3 6 45)]
-   ["Rüstem Hebel" (timestamp 2014 8 1 5 45)]])
+  [["Plato Yeshua"        #inst "2014-04-01T08:30:00.000000000-00:00"]
+   ["Felipinho Asklepios" #inst "2014-12-05T15:15:00.000000000-00:00"]
+   ["Kaneonuskatew Eiran" #inst "2014-11-06T16:15:00.000000000-00:00"]
+   ["Simcha Yan"          #inst "2014-01-01T08:30:00.000000000-00:00"]
+   ["Quentin Sören"       #inst "2014-10-03T17:30:00.000000000-00:00"]
+   ["Shad Ferdynand"      #inst "2014-08-02T12:30:00.000000000-00:00"]
+   ["Conchúr Tihomir"     #inst "2014-08-02T09:30:00.000000000-00:00"]
+   ["Szymon Theutrich"    #inst "2014-02-01T10:15:00.000000000-00:00"]
+   ["Nils Gotam"          #inst "2014-04-03T09:30:00.000000000-00:00"]
+   ["Frans Hevel"         #inst "2014-07-03T19:30:00.000000000-00:00"]
+   ["Spiros Teofil"       #inst "2014-11-01T07:00:00.000000000-00:00"]
+   ["Kfir Caj"            #inst "2014-07-03T01:30:00.000000000-00:00"]
+   ["Dwight Gresham"      #inst "2014-08-01T10:30:00.000000000-00:00"]
+   ["Broen Olujimi"       #inst "2014-10-03T13:45:00.000000000-00:00"]
+   ["Rüstem Hebel"        #inst "2014-08-01T12:45:00.000000000-00:00"]])
 
 ;; name
 (defonce ^:private categories
@@ -1216,7 +1216,10 @@
   {:users {:fields [{:name :name
                      :type "VARCHAR(254)"}
                     {:name :last_login
-                     :type "TIMESTAMP"}]
+                     :type "TIMESTAMP"}
+                    {:name :password
+                     :type "VARCHAR(254)"
+                     :field-type :sensitive}]
            :rows users}
    :categories {:fields [{:name :name
                           :type "VARCHAR(254)"}]

test/metabase/test_data/load.clj

@@ -158,15 +158,23 @@
 (def ^:private table->id (u/runtime-resolved-fn 'metabase.test-data 'table->id))
 (def ^:private field->id (u/runtime-resolved-fn 'metabase.test-data 'field->id))
 
-(defn- set-special-type! [table-kw {field-kw :name special-type :special-type}]
+(defn- set-types! [table-kw {field-kw :name special-type :special-type, field-type :field-type}]
   {:post [(true? %)]}
   (if-not special-type true
           (do (log/info "SET SPECIAL TYPE" table-kw field-kw "->" special-type)
               (upd Field (field->id table-kw field-kw) :special_type (name special-type)))))
 
+(defn- set-types! [table-kw {field-kw :name, special-type :special-type, field-type :field-type}]
+  (when special-type
+    (log/info "SET SPECIAL TYPE" table-kw field-kw "->" special-type)
+    (upd Field (field->id table-kw field-kw) :special_type (name special-type)))
+  (when field-type
+    (log/info "SET FIELD TYPE" table-kw field-kw "->" field-type)
+    (upd Field (field->id table-kw field-kw) :field_type (name field-type))))
+
 (defn- add-metadata! []
   (dorun
    (map (fn [[table-kw {:keys [fields]}]]
-          (dorun (map (partial set-special-type! table-kw)
+          (dorun (map (partial set-types! table-kw)
                       fields)))
         data/test-data)))