Merge pull request #5 from metabase/ag-liquibase

Liquibase for migrations??

project.clj

@@ -12,12 +12,16 @@
                  [org.clojure/data.json "0.2.5"]            ; JSON parsing / generation
                  [org.clojure/tools.logging "0.3.1"]        ; logging framework
                  [org.clojure/tools.macro "0.1.2"]          ; tools for writing macros
+                 [clj-time "0.5.1"]                         ; library for dealing with date/time
                  [com.cemerick/friend "0.2.1"]              ; auth library
                  [com.h2database/h2 "1.3.170"]              ; embedded SQL database
                  [compojure "1.3.1"]                        ; HTTP Routing library built on Ring
                  [environ "0.5.0"]                          ; easy environment management
                  [expectations "2.0.12"]                    ; unit tests
                  [korma "0.4.0"]                            ; SQL lib
+                 [org.clojure/java.jdbc "0.3.6"]            ; basic jdbc access from clojure
+                 [org.liquibase/liquibase-core "3.3.2"]     ; migration management (Java lib)
+                 [org.yaml/snakeyaml "1.14"]                ; YAML parser (required by liquibase)
                  [log4j/log4j "1.2.17"
                   :exclusions [javax.mail/mail
                                javax.jms/jms
@@ -34,6 +38,7 @@
             [lein-marginalia "LATEST"]                      ; generate documentation with 'lein marg'
             [lein-ring "0.8.10"]                            ; start the HTTP server with 'lein ring server'
             ]
+  :java-source-paths ["src/java"]
   :main ^:skip-aot metabase.core
   :target-path "target/%s"
   :ring {:handler metabase.core/app}

resources/migrations/liquibase.json

+{
+  "databaseChangeLog": [
+    {"include": {"file": "migrations/liquibase_initial.json"}}
+  ]
+}
\ No newline at end of file

resources/migrations/liquibase_initial.json

+{
+  "databaseChangeLog": [
+    {
+      "changeSet": {
+        "id": "1",
+        "author": "agilliland",
+        "changes": [
+          {
+            "createTable": {
+              "tableName": "core_organization",
+              "columns": [
+                {
+                  "column": {
+                    "name": "id",
+                    "type": "int",
+                    "autoIncrement": true,
+                    "constraints": {
+                      "primaryKey": true,
+                      "nullable": false
+                    }
+                  }
+                },
+                {
+                  "column": {
+                    "name": "slug",
+                    "type": "varchar(254)",
+                    "constraints": {
+                      "unique": true,
+                      "nullable": false
+                    }
+                  }
+                },
+                {
+                  "column": {
+                    "name": "name",
+                    "type": "varchar(254)",
+                    "constraints": {"nullable": false}
+                  }
+                },
+                {
+                  "column": {
+                    "name": "description",
+                    "type": "text"
+                  }
+                },
+                {
+                  "column": {
+                    "name": "logo_url",
+                    "type": "varchar(254)"
+                  }
+                },
+                {
+                  "column": {
+                    "name": "inherits",
+                    "type": "boolean",
+                    "constraints": {"nullable": false}
+                  }
+                }
+              ]
+            }
+          },
+          {
+            "createTable": {
+              "tableName": "core_user",
+              "columns": [
+                {
+                  "column": {
+                    "name": "id",
+                    "type": "int",
+                    "autoIncrement": true,
+                    "constraints": {
+                      "primaryKey": true,
+                      "nullable": false
+                    }
+                  }
+                },
+                {
+                  "column": {
+                    "name": "email",
+                    "type": "varchar(254)",
+                    "constraints": {
+                      "unique": true,
+                      "nullable": false
+                    }
+                  }
+                },
+                {
+                  "column": {
+                    "name": "first_name",
+                    "type": "varchar(254)",
+                    "constraints": {"nullable": false}
+                  }
+                },
+                {
+                  "column": {
+                    "name": "last_name",
+                    "type": "varchar(254)",
+                    "constraints": {"nullable": false}
+                  }
+                },
+                {
+                  "column": {
+                    "name": "password",
+                    "type": "varchar(254)",
+                    "constraints": {"nullable": false}
+                  }
+                },
+                {
+                  "column": {
+                    "name": "date_joined",
+                    "type": "DATETIME",
+                    "constraints": {"nullable": false}
+                  }
+                },
+                {
+                  "column": {
+                    "name": "last_login",
+                    "type": "DATETIME",
+                    "constraints": {"nullable": true}
+                  }
+                },
+                {
+                  "column": {
+                    "name": "is_staff",
+                    "type": "boolean",
+                    "constraints": {"nullable": false}
+                  }
+                },
+                {
+                  "column": {
+                    "name": "is_superuser",
+                    "type": "boolean",
+                    "constraints": {"nullable": false}
+                  }
+                },
+                {
+                  "column": {
+                    "name": "is_active",
+                    "type": "boolean",
+                    "constraints": {"nullable": false}
+                  }
+                }
+              ]
+            }
+          },
+          {
+            "createTable": {
+              "tableName": "core_userorgperm",
+              "columns": [
+                {
+                  "column": {
+                    "name": "id",
+                    "type": "int",
+                    "autoIncrement": true,
+                    "constraints": {
+                      "primaryKey": true,
+                      "nullable": false
+                    }
+                  }
+                },
+                {
+                  "column": {
+                    "name": "admin",
+                    "type": "boolean",
+                    "constraints": {"nullable": false}
+                  }
+                },
+                {
+                  "column": {
+                    "name": "user_id",
+                    "type": "int",
+                    "constraints": {
+                      "nullable": false,
+                      "references": "core_user(id)",
+                      "foreignKeyName": "fk_user_id"
+                    }
+                  }
+                },
+                {
+                  "column": {
+                    "name": "organization_id",
+                    "type": "int",
+                    "constraints": {
+                      "nullable": false,
+                      "references": "core_organization(id)",
+                      "foreignKeyName": "fk_organization_id"
+                    }
+                  }
+                }
+              ]
+            }
+          },
+          {
+            "createIndex": {
+              "tableName": "core_userorgperm",
+              "indexName": "idx_unique_user_organization",
+              "unique": true,
+              "columns": [
+                {
+                  "column": {
+                    "name": "user_id",
+                    "type": "int"
+                  }
+                },
+                {
+                  "column": {
+                    "name": "organization_id",
+                    "type": "int"
+                  }
+                }
+              ]
+            }
+          },
+          {
+            "modifySql": {
+              "dbms": "postgresql",
+              "replace": {
+                "replace": "WITHOUT",
+                "with": "WITH"
+              }
+            }
+          }
+        ]
+      }
+    }
+  ]
+}
\ No newline at end of file

src/java/com/metabase/corvus/migrations/LiquibaseMigrations.java

+package com.metabase.corvus.migrations;
+
+import java.io.FileWriter;
+import java.sql.SQLException;
+import liquibase.Liquibase;
+import liquibase.database.DatabaseFactory;
+import liquibase.database.Database;
+import liquibase.database.jvm.JdbcConnection;
+import liquibase.exception.DatabaseException;
+import liquibase.resource.ClassLoaderResourceAccessor;
+
+
+public class LiquibaseMigrations {
+
+  private static final String LIQUIBASE_CHANGELOG = "migrations/liquibase.json";
+
+
+  public static final void setupDatabase(java.sql.Connection dbConnection) throws Exception {
+    try {
+      Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(dbConnection));
+      Liquibase liquibase = new Liquibase(LIQUIBASE_CHANGELOG, new ClassLoaderResourceAccessor(), database);
+      liquibase.update("");
+    } catch (Exception e) {
+      throw new DatabaseException(e);
+    } finally {
+      if (dbConnection != null) {
+        try {
+          dbConnection.rollback();
+          dbConnection.close();
+        } catch (SQLException e){
+          //nothing to do
+        }
+      }
+    }
+  }
+
+  public static final void teardownDatabase(java.sql.Connection dbConnection) throws Exception {
+    try {
+      Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(dbConnection));
+      Liquibase liquibase = new Liquibase(LIQUIBASE_CHANGELOG, new ClassLoaderResourceAccessor(), database);
+      liquibase.rollback(10000, "");
+    } catch (Exception e) {
+      throw new DatabaseException(e);
+    } finally {
+      if (dbConnection != null) {
+        try {
+          dbConnection.rollback();
+          dbConnection.close();
+        } catch (SQLException e){
+          //nothing to do
+        }
+      }
+    }
+  }
+
+  public static final void genSqlDatabase(java.sql.Connection dbConnection) throws Exception {
+    try {
+      Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(dbConnection));
+      Liquibase liquibase = new Liquibase(LIQUIBASE_CHANGELOG, new ClassLoaderResourceAccessor(), database);
+      liquibase.update("", new FileWriter("/tmp/script.sql"));
+    } catch (Exception e) {
+      throw new DatabaseException(e);
+    } finally {
+      if (dbConnection != null) {
+        try {
+          dbConnection.rollback();
+          dbConnection.close();
+        } catch (SQLException e){
+          //nothing to do
+        }
+      }
+    }
+  }
+
+}

src/metabase/core.clj

 (ns metabase.core
   (:gen-class)
   (:require [clojure.tools.logging :as log]
+            [clojure.java.jdbc :as jdbc]
             [compojure.core :refer [context defroutes GET]]
             [compojure.route :as route]
             [ring.middleware.json :refer [wrap-json-response]]))
 
+(defn liquibase-sql []
+  (let [conn (jdbc/get-connection {:subprotocol "postgresql"
+                                   :subname "//localhost:15432/corvus_test"
+                                   :user "corvus"
+                                   :password "corvus"})]
+    (com.metabase.corvus.migrations.LiquibaseMigrations/genSqlDatabase conn)))
+
 (defn -main
   "I don't do a whole lot ... yet."
   [& args]
-  (println "Hello, World!")
-  (log/info "testing logging"))
+  (liquibase-sql))
 
 (defn first-element [sequence default]
   (or (first sequence) default))

src/metabase/db.clj

 (ns metabase.db
   (:require [clojure.tools.logging :as log]
+            [clojure.string :as str]
             [environ.core :refer [env]]
             [korma.core :refer :all]
             [korma.db :refer :all]
@@ -9,9 +10,11 @@
 (defn get-db-file
   "Check config/environment to determine the path to the h2 db file we want to use"
   []
-  (or (env :database-file) (get app-defaults :database-file)))
+  (str "file:" (or (env :database-file) (get app-defaults :database-file))))
 
 
 (defdb db (do
             (log/info (str "Using H2 database file: " (get-db-file)))
-            (h2 {:db (get-db-file)})))
+            (h2 {:db (get-db-file)
+                 :naming {:keys str/lower-case
+                          :fields str/upper-case}})))

src/metabase/models/org.clj

+(ns metabase.models.org
+  (:use korma.core
+        ))
+
+(defentity Org
+  (table :core_organization))

src/metabase/models/org_perm.clj

+(ns metabase.models.org-perm
+  (:use korma.core))
+
+(defentity OrgPerm
+  (table :core_userorgperm))

src/metabase/models/user.clj

+(ns metabase.models.user
+  (:use korma.core
+        [metabase.models.org-perm :only (OrgPerm)]))
+
+(defentity User
+  (table :core_user)
+  (has-many OrgPerm {:fk :user_id}))

test/metabase/models/org_perm_test.clj

+(ns metabase.models.org-perm-test
+  (:require [clojure.tools.logging :as log]
+            [metabase.test-util :as util]
+            [metabase.db :refer :all]
+            [metabase.config :refer [app-defaults]]
+            [metabase.models.org-perm :refer [OrgPerm]]
+            [metabase.models.org :refer [Org]]
+            [metabase.models.user :refer [User]]
+            [korma.core :refer :all]
+            [midje.sweet :refer :all]
+            [clj-time.core :as time]
+            [clj-time.coerce :as tc]))
+
+
+(defn insert-org []
+  (let [result (insert Org (values {:name "testtest"
+                                    :slug "test"
+                                    :inherits false}))]
+    (or (:generated_key result) ((keyword "scope_identity()") result) -1)))
+
+(defn insert-user []
+  (let [result (insert User (values {:email "testtest"
+                                     :first_name "test"
+                                     :last_name "test"
+                                     :password "test"
+                                     :date_joined (java.sql.Timestamp. (tc/to-long (time/now)))
+                                     :is_active true
+                                     :is_staff true
+                                     :is_superuser false}))]
+    (or (:generated_key result) ((keyword "scope_identity()") result) -1)))
+
+(defn count-perms []
+  (get-in (first (select OrgPerm (aggregate (count :*) :cnt))) [:cnt]))
+
+
+(facts "about OrgPerm model"
+  (with-state-changes [(before :facts (util/liquibase-up))
+                       (after :facts (util/liquibase-down))]
+    (fact "starts with 0 entries"
+      (count-perms) => 0)
+    (fact "can insert new entries"
+      (let [org-id (insert-org)
+            user-id (insert-user)
+            result (insert OrgPerm (values {:admin false
+                                            :organization_id org-id
+                                            :user_id user-id}))]
+        (nil? result) => false
+        (> (or (:generated_key result) ((keyword "scope_identity()") result) -1) 0) => true)
+      (count-perms) => 1)
+    ))

test/metabase/models/org_test.clj

+(ns metabase.models.org-test
+  (:require [clojure.tools.logging :as log]
+            [metabase.test-util :as util]
+            [metabase.db :refer :all]
+            [metabase.config :refer [app-defaults]]
+            [metabase.models.org :refer [Org]]
+            [korma.core :refer :all]
+            [midje.sweet :refer :all]))
+
+
+(defn count-orgs []
+  (get-in (first (select Org (aggregate (count :*) :cnt))) [:cnt]))
+
+
+(facts "about Org model"
+  (with-state-changes [(before :facts (util/liquibase-up))
+                       (after :facts (util/liquibase-down))]
+    (fact "starts with 0 entries"
+      (count-orgs) => 0)
+    (fact "can insert new entries"
+      (let [result (insert Org (values {:name "test"
+                                        :slug "test"
+                                        :inherits false}))]
+        (nil? result) => false
+        (println "orgId" (or (:generated_key result) ((keyword "scope_identity()") result) -1))
+        (> (or (:generated_key result) ((keyword "scope_identity()") result) -1) 0) => true)
+      (count-orgs) => 1)
+    ))

test/metabase/models/user_test.clj

+(ns metabase.models.user-test
+  (:require [clojure.tools.logging :as log]
+            [metabase.test-util :as util]
+            [metabase.db :refer :all]
+            [metabase.config :refer [app-defaults]]
+            [metabase.models.user :refer [User]]
+            [korma.core :refer :all]
+            [midje.sweet :refer :all]
+            [clj-time.core :as time]
+            [clj-time.coerce :as tc]))
+
+
+(defn count-users []
+  (get-in (first (select User (aggregate (count :*) :cnt))) [:cnt]))
+
+
+(facts "about User model"
+  (with-state-changes [(before :facts (util/liquibase-up))
+                       (after :facts (util/liquibase-down))]
+    (fact "starts with 0 entries"
+      (count-users) => 0)
+    (fact "can insert new entries"
+      (let [result (insert User (values {:email "test"
+                                         :first_name "test"
+                                         :last_name "test"
+                                         :password "test"
+                                         :date_joined (java.sql.Timestamp. (tc/to-long (time/now)))
+                                         :is_active true
+                                         :is_staff true
+                                         :is_superuser false}))]
+        (nil? result) => false
+        (> (or (:generated_key result) ((keyword "scope_identity()") result) -1) 0) => true)
+      (count-users) => 1)
+    ))

test/metabase/test_util.clj

+(ns metabase.test-util
+  (:require [clojure.tools.logging :as log]
+            [clojure.java.jdbc :as jdbc]
+            [metabase.config :refer [app-defaults]]
+            [metabase.db :refer [get-db-file]]))
+
+
+(defn liquibase-up
+  "Run Liquibase migrations update"
+  []
+  (let [conn (jdbc/get-connection {:subprotocol "h2"
+                                   :subname (get-db-file)})]
+    (com.metabase.corvus.migrations.LiquibaseMigrations/setupDatabase conn)))
+
+(defn liquibase-down
+  "Run Liquibase migrations rollback"
+  []
+  (let [conn (jdbc/get-connection {:subprotocol "h2"
+                                   :subname (get-db-file)})]
+    (com.metabase.corvus.migrations.LiquibaseMigrations/teardownDatabase conn)))
\ No newline at end of file