introduce models for RawTable and RawColumn.

resources/migrations/032_add_physical_schema_tables.yaml

+databaseChangeLog:
+  - changeSet:
+      id: 32
+      author: agilliland
+      changes:
+        - createTable:
+            tableName: raw_table
+            columns:
+              - column:
+                  name: id
+                  type: int
+                  autoIncrement: true
+                  constraints:
+                    primaryKey: true
+                    nullable: false
+              - column:
+                  name: database_id
+                  type: int
+                  constraints:
+                    nullable: false
+                    references: metabase_database(id)
+                    foreignKeyName: fk_rawtable_ref_database
+                    deferrable: false
+                    initiallyDeferred: false
+              - column:
+                  name: active
+                  type: boolean
+                  constraints:
+                    nullable: false
+              - column:
+                  name: schema
+                  type: varchar(255)
+                  constraints:
+                    nullable: true
+              - column:
+                  name: name
+                  type: varchar(255)
+                  constraints:
+                    nullable: false
+              - column:
+                  name: details
+                  type: text
+                  constraints:
+                    nullable: false
+              - column:
+                  name: fks
+                  type: text
+                  constraints:
+                    nullable: true
+              - column:
+                  name: created_at
+                  type: DATETIME
+                  constraints:
+                    nullable: false
+              - column:
+                  name: updated_at
+                  type: DATETIME
+                  constraints:
+                    nullable: false
+        - createIndex:
+            tableName: raw_table
+            indexName: idx_rawtable_database_id
+            columns:
+              column:
+                name: database_id
+        - createTable:
+            tableName: raw_column
+            columns:
+              - column:
+                  name: id
+                  type: int
+                  autoIncrement: true
+                  constraints:
+                    primaryKey: true
+                    nullable: false
+              - column:
+                  name: raw_table_id
+                  type: int
+                  constraints:
+                    nullable: false
+                    references: raw_table(id)
+                    foreignKeyName: fk_rawcolumn_ref_rawtable
+                    deferrable: false
+                    initiallyDeferred: false
+              - column:
+                  name: active
+                  type: boolean
+                  constraints:
+                    nullable: false
+              - column:
+                  name: name
+                  type: varchar(255)
+                  constraints:
+                    nullable: false
+              - column:
+                  name: base_type
+                  type: varchar(32)
+                  constraints:
+                    nullable: false
+              - column:
+                  name: is_pk
+                  type: boolean
+                  constraints:
+                    nullable: false
+              - column:
+                  name: details
+                  type: text
+                  constraints:
+                    nullable: false
+              - column:
+                  name: created_at
+                  type: DATETIME
+                  constraints:
+                    nullable: false
+              - column:
+                  name: updated_at
+                  type: DATETIME
+                  constraints:
+                    nullable: false
+        - createIndex:
+            tableName: raw_column
+            indexName: idx_rawcolumn_raw_table_id
+            columns:
+              column:
+                name: raw_table_id
+        - addColumn:
+            tableName: metabase_table
+            columns:
+              - column:
+                  name: raw_table_id
+                  type: int
+                  constraints:
+                    nullable: true
+                    deferrable: false
+                    initiallyDeferred: false
+        - addColumn:
+            tableName: metabase_field
+            columns:
+              - column:
+                  name: raw_column_id
+                  type: int
+                  constraints:
+                    nullable: true
+                    deferrable: false
+                    initiallyDeferred: false
+        - modifySql:
+            dbms: postgresql
+            replace:
+              replace: WITHOUT
+              with: WITH

resources/migrations/liquibase.json

@@ -29,6 +29,7 @@
       {"include": {"file": "migrations/028_add_user_is_qbnewb.yaml"}},
       {"include": {"file": "migrations/029_add_pulse_channel_schedule_frame.yaml"}},
       {"include": {"file": "migrations/030_add_field_visibility_type.yaml"}},
-      {"include": {"file": "migrations/031_add_field_fk_target.yaml"}}
+      {"include": {"file": "migrations/031_add_field_fk_target.yaml"}},
+      {"include": {"file": "migrations/032_add_physical_schema_tables.yaml"}}
   ]
 }

src/metabase/models/database.clj

@@ -20,7 +20,8 @@
 
 (defn- pre-cascade-delete [{:keys [id]}]
   (cascade-delete 'Card  :database_id id)
-  (cascade-delete 'Table :db_id id))
+  (cascade-delete 'Table :db_id id)
+  (cascade-delete 'RawTable :database_id id))
 
 (defn ^:hydrate tables
   "Return the `Tables` associated with this `Database`."

src/metabase/models/raw_column.clj

+(ns metabase.models.raw-column
+  (:require [clojure.tools.logging :as log]
+            [metabase.db :as db]
+            [metabase.models.interface :as i]
+            [metabase.util :as u]))
+
+
+(i/defentity RawColumn :raw_column)
+
+(defn- pre-insert [table]
+  (let [defaults {:active  true
+                  :is_pk   false
+                  :details {}}]
+    (merge defaults table)))
+
+
+(u/strict-extend (class RawColumn)
+  i/IEntity (merge i/IEntityDefaults
+                   {:hydration-keys     (constantly [:columns])
+                    :types              (constantly {:base_type :keyword, :details :json})
+                    :timestamped?       (constantly true)
+                    :pre-insert         pre-insert}))
+
+
+;;; ## ---------------------------------------- PERSISTENCE FUNCTIONS ----------------------------------------
+
+
+(defn save-all-table-columns
+  "Save *all* `RawColumns` for a given RAW-TABLE."
+  [{:keys [id]} columns]
+  {:pre [(integer? id)
+         (coll? columns)
+         (every? map? columns)]}
+  (let [existing-columns (into {} (for [{:keys [name] :as column} (db/sel :many :fields [RawColumn :id :name] :raw_table_id id)]
+                                    {name column}))]
+
+    ;; deactivate any columns which were removed
+    (doseq [[column-name {column-id :id}] (sort-by :name existing-columns)]
+      (when-not (some #(= column-name (:name %)) columns)
+        (log/debug (u/format-color 'cyan "Marked column %s as inactive." column-name))
+        (db/upd RawColumn column-id :active false)))
+
+    ;; insert or update the remaining columns
+    (doseq [{column-name :name, :keys [base-type pk? special-type details]} (sort-by :name columns)]
+      (let [details (merge (or details {})
+                           (when special-type {:special-type special-type}))
+            is_pk   (true? pk?)]
+        (if-let [{column-id :id} (get existing-columns column-name)]
+          ;; column already exists, update it
+          (db/upd RawColumn column-id
+            :name      column-name
+            :base_type base-type
+            :is_pk     is_pk
+            :details   details
+            :active    true)
+          ;; must be a new column, insert it
+          (db/ins RawColumn
+            :raw_table_id  id
+            :name          column-name
+            :base_type     base-type
+            :is_pk         is_pk
+            :details       details
+            :active        true))))))

src/metabase/models/raw_table.clj

+(ns metabase.models.raw-table
+  (:require [korma.core :as k]
+            [metabase.db :as db]
+            [metabase.models.interface :as i]
+            [metabase.models.raw-column :refer [RawColumn], :as raw-column]
+            [metabase.util :as u]))
+
+
+(i/defentity RawTable :raw_table)
+
+(defn- pre-insert [table]
+  (let [defaults {:details {}}]
+    (merge defaults table)))
+
+(defn- pre-cascade-delete [{:keys [id]}]
+  (db/cascade-delete RawColumn :raw_table_id id))
+
+(u/strict-extend (class RawTable)
+  i/IEntity (merge i/IEntityDefaults
+                   {:types              (constantly {:details :json, :fks :json})
+                    :timestamped?       (constantly true)
+                    :pre-insert         pre-insert
+                    :pre-cascade-delete pre-cascade-delete}))
+
+
+;;; ## ---------------------------------------- PERSISTENCE FUNCTIONS ----------------------------------------
+
+
+(defn ^:hydrate columns
+  "Return the `RawColumns` belonging to RAW-TABLE."
+  [{:keys [id]}]
+  (db/sel :many RawColumn :raw_table_id id, (k/order :name :ASC)))
+
+
+(defn create-raw-table
+  "Create a new `RawTable`, includes saving all specified `:columns`."
+  [database-id {table-name :name, table-schema :schema, :keys [details columns fks]}]
+  {:pre [(integer? database-id)
+         (string? table-name)]}
+  (let [table (db/ins RawTable
+                :database_id  database-id
+                :schema       table-schema
+                :name         table-name
+                :details      (or details {})
+                :fks          fks
+                :active       true)]
+    ;; save columns
+    (raw-column/save-all-table-columns table columns)))
+
+(defn update-raw-table
+  "Update an existing `RawTable`, includes saving all specified `:columns`."
+  [{table-id :id, :as table} {table-name :name, table-schema :schema, :keys [details columns fks]}]
+  (db/upd RawTable table-id
+    :schema   table-schema
+    :name     table-name
+    :details  (or details {})
+    :fks      fks
+    :active   true)
+  ;; save columns
+  (raw-column/save-all-table-columns table columns))
+
+(defn disable-raw-tables
+  "Disable a list of `RawTable` ids, including all `RawColumns` associated with those tables."
+  [table-ids]
+  {:pre [(coll? table-ids)
+         (every? integer? table-ids)]}
+  (let [table-ids (filter identity table-ids)]
+    ;; disable the tables
+    (k/update RawTable
+      (k/where {:id [in table-ids]})
+      (k/set-fields {:active false}))
+    ;; whenever a table is disabled we need to disable all of its fields too
+    (k/update RawColumn
+      (k/where {:raw_table_id [in table-ids]})
+      (k/set-fields {:active false}))))

test/metabase/models/raw_column_test.clj

+(ns metabase.models.raw-column-test
+  (:require [expectations :refer :all]
+            [metabase.db :as db]
+            [metabase.models.database :as database]
+            [metabase.models.raw-table :as raw-table]
+            [metabase.models.raw-column :refer [RawColumn], :as raw-column]
+            [metabase.test.util :as tu]))
+
+(expect
+  [[]
+   [{:id           true,
+     :raw_table_id true,
+     :active       true,
+     :name         "beak_size",
+     :base_type    :IntegerField
+     :is_pk        true
+     :details      {:inches 7, :special-type "category"},
+     :created_at   true,
+     :updated_at   true}]
+   [{:id           true,
+     :raw_table_id true,
+     :active       true,
+     :name         "beak_size",
+     :base_type    :IntegerField
+     :is_pk        false
+     :details      {:inches 8},
+     :created_at   true,
+     :updated_at   true}
+    {:id           true,
+     :raw_table_id true,
+     :active       true,
+     :name         "num_feathers",
+     :base_type    :IntegerField
+     :is_pk        false
+     :details      {:count 10000},
+     :created_at   true,
+     :updated_at   true}]
+   [{:id           true,
+     :raw_table_id true,
+     :active       false,
+     :name         "beak_size",
+     :base_type    :IntegerField
+     :is_pk        false
+     :details      {:inches 8},
+     :created_at   true,
+     :updated_at   true}
+    {:id           true,
+     :raw_table_id true,
+     :active       true,
+     :name         "num_feathers",
+     :base_type    :IntegerField
+     :is_pk        false
+     :details      {:count 12000},
+     :created_at   true,
+     :updated_at   true}]
+   [{:id           true,
+     :raw_table_id true,
+     :active       true,
+     :name         "beak_size",
+     :base_type    :IntegerField
+     :is_pk        false
+     :details      {:inches 8},
+     :created_at   true,
+     :updated_at   true}
+    {:id           true,
+     :raw_table_id true,
+     :active       true,
+     :name         "num_feathers",
+     :base_type    :IntegerField
+     :is_pk        false
+     :details      {:count 12000},
+     :created_at   true,
+     :updated_at   true}]]
+  (tu/with-temp* [database/Database  [{database-id :id}]
+                  raw-table/RawTable [{raw-table-id :id, :as table} {:database_id database-id}]]
+    (let [get-columns #(->> (db/sel :many RawColumn :raw_table_id raw-table-id)
+                            (mapv tu/boolean-ids-and-timestamps))]
+      ;; original list should be empty
+      [(get-columns)
+       ;; now add a column
+       (do
+         (raw-column/save-all-table-columns table [{:name "beak_size", :base-type :IntegerField, :details {:inches 7}, :pk? true, :special-type "category"}])
+         (get-columns))
+       ;; now add another column and modify the first
+       (do
+         (raw-column/save-all-table-columns table [{:name "beak_size", :base-type :IntegerField, :details {:inches 8}}
+                                                   {:name "num_feathers", :base-type :IntegerField, :details {:count 10000}}])
+         (get-columns))
+       ;; now remove the first column
+       (do
+         (raw-column/save-all-table-columns table [{:name "num_feathers", :base-type :IntegerField, :details {:count 12000}}])
+         (get-columns))
+       ;; lastly, resurrect the first column (this ensures uniqueness by name)
+       (do
+         (raw-column/save-all-table-columns table [{:name "beak_size", :base-type :IntegerField, :details {:inches 8}}
+                                                    {:name "num_feathers", :base-type :IntegerField, :details {:count 12000}}])
+         (get-columns))])))

test/metabase/models/raw_table_test.clj

+(ns metabase.models.raw-table-test
+  (:require [expectations :refer :all]
+            [metabase.db :as db]
+            [metabase.models.database :as database]
+            [metabase.models.hydrate :as hydrate]
+            [metabase.models.raw-table :refer [RawTable], :as raw-table]
+            [metabase.models.raw-column :as raw-column]
+            [metabase.test.util :as tu]))
+
+(defn get-tables [database-id]
+  (->> (hydrate/hydrate (db/sel :many RawTable :database_id database-id) :columns)
+       (mapv tu/boolean-ids-and-timestamps)))
+
+;; create-raw-table
+(expect
+  [[]
+   [{:id          true
+     :database_id true
+     :active      true
+     :schema      nil
+     :name        "users"
+     :details     {:a "b"}
+     :columns     []
+     :fks         nil
+     :created_at  true
+     :updated_at  true}]
+   [{:id          true
+     :database_id true
+     :active      true
+     :schema      nil
+     :name        "users"
+     :details     {:a "b"}
+     :columns     []
+     :fks         nil
+     :created_at  true
+     :updated_at  true}
+    {:id          true
+     :database_id true
+     :active      true
+     :schema      "aviary"
+     :name        "toucanery"
+     :details     {:owner "Cam"}
+     :columns     [{:id           true
+                    :raw_table_id true
+                    :active       true
+                    :name         "beak_size"
+                    :base_type    :IntegerField
+                    :details      {:inches 7}
+                    :created_at   true
+                    :updated_at   true}]
+     :fks         [{:a "b"}]
+     :created_at  true
+     :updated_at  true}]]
+  (tu/with-temp* [database/Database [{database-id :id, :as db}]]
+    [(get-tables database-id)
+     ;; now add a table
+     (do
+       (raw-table/create-raw-table database-id {:schema nil,
+                                                :name "users",
+                                                :details {:a "b"}
+                                                :columns []})
+       (get-tables database-id))
+     ;; now add another table, this time with a couple columns and some fks
+     (do
+       (raw-table/create-raw-table database-id {:schema "aviary",
+                                                :name "toucanery",
+                                                :details {:owner "Cam"}
+                                                :columns [{:name "beak_size",
+                                                           :base_type :IntegerField,
+                                                           :details {:inches 7}}]
+                                                :fks     [{:a "b"}]})
+       (get-tables database-id))]))
+
+
+;; update-raw-table
+(expect
+  [[{:id          true
+     :database_id true
+     :active      true
+     :schema      "aviary"
+     :name        "toucanery"
+     :details     {:owner "Cam"}
+     :columns     []
+     :fks         [{:a "b"}]
+     :created_at  true
+     :updated_at  true}]
+   [{:id          true
+     :database_id true
+     :active      true
+     :schema      "aviary"
+     :name        "toucanery"
+     :details     {:owner "Cam", :sqft 10000}
+     :columns     [{:id           true
+                    :raw_table_id true
+                    :active       true
+                    :name         "beak_size"
+                    :base_type    :IntegerField
+                    :details      {:inches 7}
+                    :created_at   true
+                    :updated_at   true}]
+     :fks         nil
+     :created_at  true
+     :updated_at  true}]]
+  (tu/with-temp* [database/Database  [{database-id :id, :as db}]
+                  raw-table/RawTable [table {:database_id database-id
+                                             :schema      "aviary",
+                                             :name        "toucanery",
+                                             :details     {:owner "Cam"}
+                                             :fks         [{:a "b"}]}]]
+    [(get-tables database-id)
+     ;; now update the table
+     (do
+       (raw-table/update-raw-table table {:schema  "aviary",
+                                                :name    "toucanery",
+                                                :details {:owner "Cam", :sqft 10000}
+                                                :columns [{:name      "beak_size",
+                                                           :base_type :IntegerField,
+                                                           :details   {:inches 7}}]
+                                                :fks     nil})
+       (get-tables database-id))]))
+
+
+;; disable-raw-tables
+(expect
+  [[{:id          true
+     :database_id true
+     :active      true
+     :schema      "a"
+     :name        "1"
+     :details     {}
+     :columns     []
+     :fks         nil
+     :created_at  true
+     :updated_at  true}
+    {:id          true
+     :database_id true
+     :active      true
+     :schema      "a"
+     :name        "2"
+     :details     {}
+     :columns     [{:id           true
+                    :raw_table_id true
+                    :active       true
+                    :name         "beak_size"
+                    :base_type    :IntegerField
+                    :details      {}
+                    :created_at   true
+                    :updated_at   true}]
+     :fks         nil
+     :created_at  true
+     :updated_at  true}]
+   [{:id          true
+     :database_id true
+     :active      false
+     :schema      "a"
+     :name        "1"
+     :details     {}
+     :columns     []
+     :fks         nil
+     :created_at  true
+     :updated_at  true}
+    {:id          true
+     :database_id true
+     :active      false
+     :schema      "a"
+     :name        "2"
+     :details     {}
+     :columns     [{:id           true
+                    :raw_table_id true
+                    :active       false
+                    :name         "beak_size"
+                    :base_type    :IntegerField
+                    :details      {}
+                    :created_at   true
+                    :updated_at   true}]
+     :fks         nil
+     :created_at  true
+     :updated_at  true}]]
+  (tu/with-temp* [database/Database    [{database-id :id, :as db}]
+                  raw-table/RawTable   [t1 {:database_id database-id, :schema "a", :name "1"}]
+                  raw-table/RawTable   [t2 {:database_id database-id, :schema "a", :name "2"}]
+                  raw-column/RawColumn [c1 {:raw_table_id (:id t2), :name "beak_size", :base_type :IntegerField}]]
+    [(get-tables database-id)
+     (do
+       (raw-table/disable-raw-tables [(:id t1) (:id t2)])
+       (get-tables database-id))]))

test/metabase/test/util.clj

@@ -13,6 +13,8 @@
                              [metric :refer [Metric]]
                              [pulse :refer [Pulse]]
                              [pulse-channel :refer [PulseChannel]]
+                             [raw-column :refer [RawColumn]]
+                             [raw-table :refer [RawTable]]
                              [revision :refer [Revision]]
                              [segment :refer [Segment]]
                              [table :refer [Table]])))
@@ -84,6 +86,24 @@
   (->> (repeatedly 20 #(-> (rand-int 26) (+ (int \A)) char))
        (apply str)))
 
+
+(defn boolean-ids-and-timestamps
+  "Useful for unit test comparisons.  Converts map keys with 'id' or '_at' to booleans."
+  [m]
+  (let [f (fn [v]
+            (cond
+              (map? v) (boolean-ids-and-timestamps v)
+              (coll? v) (mapv boolean-ids-and-timestamps v)
+              :else v))]
+    (into {} (for [[k v] m]
+               (if (or (= :id k)
+                       (.endsWith (name k) "_id")
+                       (= :created_at k)
+                       (= :updated_at k))
+                 [k (not (nil? v))]
+                 [k (f v)])))))
+
+
 (defprotocol ^:private WithTempDefaults
   (^:private with-temp-defaults [this]))
 
@@ -117,6 +137,10 @@
                                                                                          :details       {}
                                                                                          :schedule_type :daily
                                                                                          :schedule_hour 15})})
+(u/strict-extend (class RawColumn)    WithTempDefaults {:with-temp-defaults (fn [_] {:active true
+                                                                                     :name   (random-name)})})
+(u/strict-extend (class RawTable)     WithTempDefaults {:with-temp-defaults (fn [_] {:active true
+                                                                                     :name   (random-name)})})
 (u/strict-extend (class Revision)     WithTempDefaults {:with-temp-defaults (fn [_] {:user_id      ((resolve 'metabase.test.data.users/user->id) :rasta)
                                                                                      :is_creation  false
                                                                                      :is_reversion false})})