Skip to content
Snippets Groups Projects
Commit 5f9a1af3 authored by Allen Gilliland's avatar Allen Gilliland
Browse files

add fk_target_column_id to raw_column table for better tracking of fks and... 

add fk_target_column_id to raw_column table for better tracking of fks and remove previous :fks attribute on raw_table.  move our sync related raw_* functions to a new namespace.
parent 51829c77
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
resources/migrations/032_add_physical_schema_tables.yaml
+ 10
− 6
View file @ 5f9a1af3
......@@ -42,11 +42,6 @@ databaseChangeLog:
type: text
constraints:
nullable: false
- column:
name: fks
type: text
constraints:
nullable: true
- column:
name: created_at
type: DATETIME
......@@ -79,7 +74,7 @@ databaseChangeLog:
constraints:
nullable: false
references: raw_table(id)
foreignKeyName: fk_rawcolumn_ref_rawtable
foreignKeyName: fk_rawcolumn_tableid_ref_rawtable
deferrable: false
initiallyDeferred: false
- column:
......@@ -102,6 +97,15 @@ databaseChangeLog:
type: boolean
constraints:
nullable: false
- column:
name: fk_target_column_id
type: int
constraints:
nullable: true
references: raw_column(id)
foreignKeyName: fk_rawcolumn_fktarget_ref_rawcolumn
deferrable: false
initiallyDeferred: false
- column:
name: details
type: text
......
src/metabase/models/raw_column.clj
+ 1
− 44
View file @ 5f9a1af3
(ns metabase.models.raw-column
(:require [clojure.tools.logging :as log]
[metabase.db :as db]
[metabase.models.interface :as i]
(:require [metabase.models.interface :as i]
[metabase.util :as u]))
......@@ -20,44 +18,3 @@
:types (constantly {:base_type :keyword, :details :json})
:timestamped? (constantly true)
:pre-insert pre-insert}))
;;; ## ---------------------------------------- PERSISTENCE FUNCTIONS ----------------------------------------
(defn save-all-table-columns
"Save *all* `RawColumns` for a given RAW-TABLE."
[{:keys [id]} columns]
{:pre [(integer? id)
(coll? columns)
(every? map? columns)]}
(let [existing-columns (into {} (for [{:keys [name] :as column} (db/sel :many :fields [RawColumn :id :name] :raw_table_id id)]
{name column}))]
;; deactivate any columns which were removed
(doseq [[column-name {column-id :id}] (sort-by :name existing-columns)]
(when-not (some #(= column-name (:name %)) columns)
(log/debug (u/format-color 'cyan "Marked column %s as inactive." column-name))
(db/upd RawColumn column-id :active false)))
;; insert or update the remaining columns
(doseq [{column-name :name, :keys [base-type pk? special-type details]} (sort-by :name columns)]
(let [details (merge (or details {})
(when special-type {:special-type special-type}))
is_pk (true? pk?)]
(if-let [{column-id :id} (get existing-columns column-name)]
;; column already exists, update it
(db/upd RawColumn column-id
:name column-name
:base_type base-type
:is_pk is_pk
:details details
:active true)
;; must be a new column, insert it
(db/ins RawColumn
:raw_table_id id
:name column-name
:base_type base-type
:is_pk is_pk
:details details
:active true))))))
src/metabase/models/raw_table.clj
+ 10
− 43
View file @ 5f9a1af3
......@@ -2,7 +2,7 @@
(:require [korma.core :as k]
[metabase.db :as db]
[metabase.models.interface :as i]
[metabase.models.raw-column :refer [RawColumn], :as raw-column]
[metabase.models.raw-column :refer [RawColumn]]
[metabase.util :as u]))
......@@ -17,7 +17,7 @@
(u/strict-extend (class RawTable)
i/IEntity (merge i/IEntityDefaults
{:types (constantly {:details :json, :fks :json})
{:types (constantly {:details :json})
:timestamped? (constantly true)
:pre-insert pre-insert
:pre-cascade-delete pre-cascade-delete}))
......@@ -31,45 +31,12 @@
[{:keys [id]}]
(db/sel :many RawColumn :raw_table_id id, (k/order :name :ASC)))
(defn active-tables
"Return the active `RawColumns` belonging to RAW-TABLE."
[database-id]
(db/sel :many RawTable :database_id database-id, :active true, (k/order :schema :ASC), (k/order :name :ASC)))
(defn create-raw-table
"Create a new `RawTable`, includes saving all specified `:columns`."
[database-id {table-name :name, table-schema :schema, :keys [details columns fks]}]
{:pre [(integer? database-id)
(string? table-name)]}
(let [table (db/ins RawTable
:database_id database-id
:schema table-schema
:name table-name
:details (or details {})
:fks fks
:active true)]
;; save columns
(raw-column/save-all-table-columns table columns)))
(defn update-raw-table
"Update an existing `RawTable`, includes saving all specified `:columns`."
[{table-id :id, :as table} {table-name :name, table-schema :schema, :keys [details columns fks]}]
(db/upd RawTable table-id
:schema table-schema
:name table-name
:details (or details {})
:fks fks
:active true)
;; save columns
(raw-column/save-all-table-columns table columns))
(defn disable-raw-tables
"Disable a list of `RawTable` ids, including all `RawColumns` associated with those tables."
[table-ids]
{:pre [(coll? table-ids)
(every? integer? table-ids)]}
(let [table-ids (filter identity table-ids)]
;; disable the tables
(k/update RawTable
(k/where {:id [in table-ids]})
(k/set-fields {:active false}))
;; whenever a table is disabled we need to disable all of its fields too
(k/update RawColumn
(k/where {:raw_table_id [in table-ids]})
(k/set-fields {:active false}))))
(defn active-columns
"Return the active `RawColumns` belonging to RAW-TABLE."
[{:keys [id]}]
(db/sel :many RawColumn :raw_table_id id, :active true, (k/order :name :ASC)))
test/metabase/models/raw_column_test.clj deleted 100644 → 0
+ 0
− 97
View file @ 51829c77
(ns metabase.models.raw-column-test
(:require [expectations :refer :all]
[metabase.db :as db]
[metabase.models.database :as database]
[metabase.models.raw-table :as raw-table]
[metabase.models.raw-column :refer [RawColumn], :as raw-column]
[metabase.test.util :as tu]))
(expect
[[]
[{:id true,
:raw_table_id true,
:active true,
:name "beak_size",
:base_type :IntegerField
:is_pk true
:details {:inches 7, :special-type "category"},
:created_at true,
:updated_at true}]
[{:id true,
:raw_table_id true,
:active true,
:name "beak_size",
:base_type :IntegerField
:is_pk false
:details {:inches 8},
:created_at true,
:updated_at true}
{:id true,
:raw_table_id true,
:active true,
:name "num_feathers",
:base_type :IntegerField
:is_pk false
:details {:count 10000},
:created_at true,
:updated_at true}]
[{:id true,
:raw_table_id true,
:active false,
:name "beak_size",
:base_type :IntegerField
:is_pk false
:details {:inches 8},
:created_at true,
:updated_at true}
{:id true,
:raw_table_id true,
:active true,
:name "num_feathers",
:base_type :IntegerField
:is_pk false
:details {:count 12000},
:created_at true,
:updated_at true}]
[{:id true,
:raw_table_id true,
:active true,
:name "beak_size",
:base_type :IntegerField
:is_pk false
:details {:inches 8},
:created_at true,
:updated_at true}
{:id true,
:raw_table_id true,
:active true,
:name "num_feathers",
:base_type :IntegerField
:is_pk false
:details {:count 12000},
:created_at true,
:updated_at true}]]
(tu/with-temp* [database/Database [{database-id :id}]
raw-table/RawTable [{raw-table-id :id, :as table} {:database_id database-id}]]
(let [get-columns #(->> (db/sel :many RawColumn :raw_table_id raw-table-id)
(mapv tu/boolean-ids-and-timestamps))]
;; original list should be empty
[(get-columns)
;; now add a column
(do
(raw-column/save-all-table-columns table [{:name "beak_size", :base-type :IntegerField, :details {:inches 7}, :pk? true, :special-type "category"}])
(get-columns))
;; now add another column and modify the first
(do
(raw-column/save-all-table-columns table [{:name "beak_size", :base-type :IntegerField, :details {:inches 8}}
{:name "num_feathers", :base-type :IntegerField, :details {:count 10000}}])
(get-columns))
;; now remove the first column
(do
(raw-column/save-all-table-columns table [{:name "num_feathers", :base-type :IntegerField, :details {:count 12000}}])
(get-columns))
;; lastly, resurrect the first column (this ensures uniqueness by name)
(do
(raw-column/save-all-table-columns table [{:name "beak_size", :base-type :IntegerField, :details {:inches 8}}
{:name "num_feathers", :base-type :IntegerField, :details {:count 12000}}])
(get-columns))])))
test/metabase/models/raw_table_test.clj deleted 100644 → 0
+ 0
− 186
View file @ 51829c77
(ns metabase.models.raw-table-test
(:require [expectations :refer :all]
[metabase.db :as db]
[metabase.models.database :as database]
[metabase.models.hydrate :as hydrate]
[metabase.models.raw-table :refer [RawTable], :as raw-table]
[metabase.models.raw-column :as raw-column]
[metabase.test.util :as tu]))
(defn get-tables [database-id]
(->> (hydrate/hydrate (db/sel :many RawTable :database_id database-id) :columns)
(mapv tu/boolean-ids-and-timestamps)))
;; create-raw-table
(expect
[[]
[{:id true
:database_id true
:active true
:schema nil
:name "users"
:details {:a "b"}
:columns []
:fks nil
:created_at true
:updated_at true}]
[{:id true
:database_id true
:active true
:schema nil
:name "users"
:details {:a "b"}
:columns []
:fks nil
:created_at true
:updated_at true}
{:id true
:database_id true
:active true
:schema "aviary"
:name "toucanery"
:details {:owner "Cam"}
:columns [{:id true
:raw_table_id true
:active true
:name "beak_size"
:base_type :IntegerField
:details {:inches 7}
:created_at true
:updated_at true}]
:fks [{:a "b"}]
:created_at true
:updated_at true}]]
(tu/with-temp* [database/Database [{database-id :id, :as db}]]
[(get-tables database-id)
;; now add a table
(do
(raw-table/create-raw-table database-id {:schema nil,
:name "users",
:details {:a "b"}
:columns []})
(get-tables database-id))
;; now add another table, this time with a couple columns and some fks
(do
(raw-table/create-raw-table database-id {:schema "aviary",
:name "toucanery",
:details {:owner "Cam"}
:columns [{:name "beak_size",
:base_type :IntegerField,
:details {:inches 7}}]
:fks [{:a "b"}]})
(get-tables database-id))]))
;; update-raw-table
(expect
[[{:id true
:database_id true
:active true
:schema "aviary"
:name "toucanery"
:details {:owner "Cam"}
:columns []
:fks [{:a "b"}]
:created_at true
:updated_at true}]
[{:id true
:database_id true
:active true
:schema "aviary"
:name "toucanery"
:details {:owner "Cam", :sqft 10000}
:columns [{:id true
:raw_table_id true
:active true
:name "beak_size"
:base_type :IntegerField
:details {:inches 7}
:created_at true
:updated_at true}]
:fks nil
:created_at true
:updated_at true}]]
(tu/with-temp* [database/Database [{database-id :id, :as db}]
raw-table/RawTable [table {:database_id database-id
:schema "aviary",
:name "toucanery",
:details {:owner "Cam"}
:fks [{:a "b"}]}]]
[(get-tables database-id)
;; now update the table
(do
(raw-table/update-raw-table table {:schema "aviary",
:name "toucanery",
:details {:owner "Cam", :sqft 10000}
:columns [{:name "beak_size",
:base_type :IntegerField,
:details {:inches 7}}]
:fks nil})
(get-tables database-id))]))
;; disable-raw-tables
(expect
[[{:id true
:database_id true
:active true
:schema "a"
:name "1"
:details {}
:columns []
:fks nil
:created_at true
:updated_at true}
{:id true
:database_id true
:active true
:schema "a"
:name "2"
:details {}
:columns [{:id true
:raw_table_id true
:active true
:name "beak_size"
:base_type :IntegerField
:details {}
:created_at true
:updated_at true}]
:fks nil
:created_at true
:updated_at true}]
[{:id true
:database_id true
:active false
:schema "a"
:name "1"
:details {}
:columns []
:fks nil
:created_at true
:updated_at true}
{:id true
:database_id true
:active false
:schema "a"
:name "2"
:details {}
:columns [{:id true
:raw_table_id true
:active false
:name "beak_size"
:base_type :IntegerField
:details {}
:created_at true
:updated_at true}]
:fks nil
:created_at true
:updated_at true}]]
(tu/with-temp* [database/Database [{database-id :id, :as db}]
raw-table/RawTable [t1 {:database_id database-id, :schema "a", :name "1"}]
raw-table/RawTable [t2 {:database_id database-id, :schema "a", :name "2"}]
raw-column/RawColumn [c1 {:raw_table_id (:id t2), :name "beak_size", :base_type :IntegerField}]]
[(get-tables database-id)
(do
(raw-table/disable-raw-tables [(:id t1) (:id t2)])
(get-tables database-id))]))
test/metabase/test/util.clj
+ 2
− 1
View file @ 5f9a1af3
......@@ -138,7 +138,8 @@
:schedule_type :daily
:schedule_hour 15})})
(u/strict-extend (class RawColumn) WithTempDefaults {:with-temp-defaults (fn [_] {:active true
:name (random-name)})})
:name (random-name)
:base_type :IntegerField})})
(u/strict-extend (class RawTable) WithTempDefaults {:with-temp-defaults (fn [_] {:active true
:name (random-name)})})
(u/strict-extend (class Revision) WithTempDefaults {:with-temp-defaults (fn [_] {:user_id ((resolve 'metabase.test.data.users/user->id) :rasta)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment