[Apps] Prototype backend app scaffolding (#25314)

* [Apps] Prototype backend app scaffolding

WIP

Given a set of table-ids, we try to build out an app. Ideally the
produced scaffold is done on the frontend and passed in to

1. avoid code duplication
2. allow the front end to maintain ownership of visualization_settings
3. avoid regressions on frontend code changes

However, this currently builds the scaffold on the backend due to FE dev
bandwidth. In theory, it is done in such a way that it both matches the
FE as closely as possible as well as becomes easy to change this code to
accept a scaffold rather than generating one itself.

It uses `scaffold-target` to map cards and dashboards in the scaffold
with the inserted ids.

Things that are still being worked out elsewhere:
1. The shape of nav-items
2. The shape of implicit action buttons

* Sort namespace

* Update and fix scaffold based on demo

* Address review comments

Deduplicate table-ids and make sure they are valid.

i18n page suffixes.

Check that tables have exactly one primary key column.

Check that card scaffold has a `scaffold-target`

Remove redudant check in scaffold-target replacement that the map lookup took care of.

src/metabase/api/app.clj

 (ns metabase.api.app
   (:require
+    [clojure.walk :as walk]
     [compojure.core :refer [POST PUT]]
+    [medley.core :as m]
+    [metabase.api.card :as api.card]
     [metabase.api.collection :as api.collection]
     [metabase.api.common :as api]
-    [metabase.models :refer [App Collection]]
+    [metabase.models :refer [App Collection Dashboard Table]]
     [metabase.models.collection :as collection]
+    [metabase.models.dashboard :as dashboard]
+    [metabase.util.i18n :as i18n]
     [metabase.util.schema :as su]
     [schema.core :as s]
     [toucan.db :as db]
@@ -13,11 +18,21 @@
 (defn- hydrate-details [apps]
   (hydrate apps [:collection :can_write]))
 
+(defn- create-app! [{:keys [collection] :as app}]
+  (db/transaction
+   (let [coll-params (select-keys collection [:name :color :description :namespace :authority_level])
+         collection-instance (api.collection/create-collection! coll-params)
+         app-params (-> app
+                        (select-keys [:dashboard_id :options :nav_items])
+                        (assoc :collection_id (:id collection-instance)))
+         app (db/insert! App app-params)]
+     (hydrate-details app))))
+
 (api/defendpoint POST "/"
   "Endpoint to create an app"
-  [:as {{:keys [collection dashboard_id options nav_items]
+  [:as {{:keys [dashboard_id options nav_items]
          {:keys [name color description namespace authority_level]} :collection
-         :as body} :body}]
+         :as app} :body}]
   {dashboard_id    (s/maybe su/IntGreaterThanOrEqualToZero)
    options         (s/maybe su/Map)
    nav_items       (s/maybe [(s/maybe su/Map)])
@@ -26,14 +41,7 @@
    description     (s/maybe su/NonBlankString)
    namespace       (s/maybe su/NonBlankString)
    authority_level collection/AuthorityLevel}
-  (db/transaction
-   (let [coll-params (select-keys collection [:name :color :description :namespace :authority_level])
-         collection-instance (api.collection/create-collection! coll-params)
-         app-params (-> body
-                        (select-keys [:dashboard_id :options :nav_items])
-                        (assoc :collection_id (:id collection-instance)))
-         app (db/insert! App app-params)]
-     (hydrate-details app))))
+  (create-app! app))
 
 (api/defendpoint PUT "/:app-id"
   "Endpoint to change an app"
@@ -68,4 +76,151 @@
   [id]
   (hydrate-details (api/read-check App id)))
 
+(defn- replace-scaffold-targets
+  [structure scaffold-target->target-id]
+  (walk/postwalk
+    (fn [node]
+      (if (and (vector? node) (= "scaffold-target-id" (first node)))
+        (if-let [target-id (get scaffold-target->target-id node)]
+          target-id
+          node)
+        node))
+    structure))
+
+(defn- generate-scaffold
+  [app-name table-ids]
+  (let [table-ids (distinct table-ids)
+        tables (hydrate (db/select Table :id [:in table-ids]) :fields)
+        _ (when (not= (count table-ids) (count tables))
+            (throw (ex-info (i18n/tru "Some tables could not be found. Given: {0} Found: {1}"
+                                      (pr-str table-ids)
+                                      (pr-str (map :id tables)))
+                            {:status-code 400})))
+        table-id->table (m/index-by :id tables)
+        page-type-display {"list" {:name (i18n/tru "List")
+                                   :display "list"}
+                           "detail" {:name (i18n/tru "Detail")
+                                     :display "object"}}]
+    {:app {:collection {:name app-name :color "#FFA500"}
+           :dashboard_id ["scaffold-target-id" "page" (:id (first tables)) "list"]
+           :nav_items (for [table-id table-ids
+                            page-type ["list" "detail"]]
+                        (cond-> {:page_id ["scaffold-target-id" "page" table-id page-type]}
+                          (= page-type "detail") (assoc :indent 1 :hidden true)))}
+     :cards (for [table-id table-ids
+                  :let [table (get table-id->table table-id)]
+                  page-type ["list" "detail"]]
+              {:scaffold-target ["card" table-id page-type]
+               :name (format "Query %s %s"
+                             (or (:display_name table) (:name table))
+                             (get-in page-type-display [page-type :name]))
+               :display (get-in page-type-display [page-type :display])
+               :visualization_settings (cond-> {}
+                                         (= page-type "list") (assoc "actions.bulk_enabled" false))
+               :dataset_query {:type "query"
+                               :database (:db_id table)
+                               :query {:source_table table-id}}})
+     :pages (for [table-id table-ids
+                  :let [table (get table-id->table table-id)
+                        pks (filter (comp #(= :type/PK %) :semantic_type) (:fields table))
+                        _ (when (not= 1 (count pks))
+                            (throw (ex-info (i18n/tru "Table must have a single primary key: {0}" (:name table))
+                                            {:status-code 400})))
+                        pk-field-id (:id (first pks))]
+                  page-type ["list" "detail"]]
+              (cond->
+               {:name (format "%s %s"
+                              (or (:display_name table) (:name table))
+                              (get-in page-type-display [page-type :name]))
+                :scaffold-target ["page" table-id page-type]
+                :ordered_cards (if (= "list" page-type)
+                                 [{:size_y 6 :size_x 18 :row 1 :col 0
+                                   :card_id ["scaffold-target-id" "card" table-id page-type]
+                                   :visualization_settings {"click_behavior"
+                                                            {"type" "link"
+                                                             "linkType" "dashboard"
+                                                             "parameterMapping" {(str "scaffold_" table-id) {"source" {"type" "column",
+                                                                                                                       "id" "ID",
+                                                                                                                       "name" "ID"},
+                                                                                                             "target" {"type" "parameter",
+                                                                                                                       "id" (str "scaffold_" table-id)},
+                                                                                                             "id" "scaffold_91"}}
+                                                             "targetId" ["scaffold-target-id" "page" table-id "detail"]}}}
+                                  {:size_y 1 :size_x 2 :row 0 :col 16
+                                   :visualization_settings {"virtual_card" {"display" "action-button"}
+                                                            "button.label" "New",
+                                                            "click_behavior" {"type" "action" "actionType" "insert" "tableId" table-id}}}]
+                                 [{:size_y 6 :size_x 18 :row 1 :col 0
+                                   :parameter_mappings [{"parameter_id" (str "scaffold_" table-id)
+                                                         "card_id" ["scaffold-target-id" "card" table-id "detail"]
+                                                         "target" ["dimension", ["field", pk-field-id, nil]]}]
+                                   :card_id ["scaffold-target-id" "card" table-id "detail"]
+                                   :scaffold-target ["dashcard" table-id]}
+                                  {:size_y 1 :size_x 2 :row 0 :col 16
+                                   :visualization_settings {"virtual_card" {"display" "action-button"}
+                                                            "button.label" "Delete",
+                                                            "button.variant" "danger"
+                                                            "click_behavior" {"type" "action" "actionType" "delete" "objectDetailDashCardId" ["scaffold-target-id" "dashcard" table-id]}}}
+                                  {:size_y 1 :size_x 2 :row 0 :col 14
+                                   :visualization_settings {"virtual_card" {"display" "action-button"}
+                                                            "button.label" "Edit",
+                                                            "click_behavior" {"type" "action" "actionType" "update" "objectDetailDashCardId" ["scaffold-target-id" "dashcard" table-id]}}}])}
+                (= "detail" page-type) (assoc :parameters [{:name "ID",
+                                                            :slug "id",
+                                                            :id (str "scaffold_" table-id),
+                                                            :type "id",
+                                                            :hidden true
+                                                            :sectionId "id"}])))}))
+
+(api/defendpoint POST "/scaffold"
+  "Endpoint to scaffold a fully working data-app"
+  [:as {{:keys [table-ids app-name]} :body}]
+  (db/transaction
+    (let [{:keys [app pages cards] :as scaffold} (generate-scaffold app-name table-ids)
+          ;; Create a blank app with just the collection info, we will update the rest later after we replace scaffold-target-id
+          {app-id :id {collection-id :id} :collection} (create-app! (select-keys app [:collection]))
+          ;; We create the cards so we can replace scaffold-target-id elsewhere
+          scaffold-target->id (reduce
+                                (fn [accum {:keys [scaffold-target] :as card}]
+                                  (when-not scaffold-target
+                                    (throw (ex-info (i18n/tru "A scaffold-target was not provided for Card: {0}" (:name card))
+                                                    {:status-code 400})))
+                                  (let [card (api.card/create-card! (-> card
+                                                                        (assoc :collection_id collection-id)
+                                                                        (dissoc :scaffold-target)))]
+                                    (assoc accum (into ["scaffold-target-id"] scaffold-target) (:id card))))
+                                {}
+                                cards)
+          ;; We create the dashboards (without dashcards) so we can replace scaffold-target-id elsewhere
+          scaffold-target->id (reduce
+                                (fn [accum {:keys [scaffold-target] :as page}]
+                                  (when-not scaffold-target
+                                    (throw (ex-info (i18n/tru "A scaffold-target was not provided for Page: {0}" (:name page))
+                                                    {:status-code 400})))
+                                  (let [blank-page (-> page
+                                                       (assoc :collection_id collection-id
+                                                              :is_app_page true
+                                                              :creator_id api/*current-user-id*)
+                                                       (dissoc :ordered_cards :scaffold-target))
+                                        dashboard (db/insert! Dashboard blank-page)]
+                                    (assoc accum (into ["scaffold-target-id"] scaffold-target) (:id dashboard))))
+                                scaffold-target->id
+                                pages)
+          ;; now replace targets with actual ids
+          {:keys [app pages]} (replace-scaffold-targets scaffold scaffold-target->id)]
+      (db/update! App app-id (select-keys app [:dashboard_id :options :nav_items]))
+      (doseq [{:keys [ordered_cards scaffold-target]} pages
+              :let [dashboard-id (get scaffold-target->id (into ["scaffold-target-id"] scaffold-target))]]
+        ;; if dashcards need to refer to each other with scaffold-target they must be in the right order
+        (reduce (fn [accum {:keys [scaffold-target] :as dashcard}]
+                  (let [{dashcard-id :id} (dashboard/add-dashcard!
+                                            dashboard-id
+                                            (:card_id dashcard)
+                                            (replace-scaffold-targets dashcard accum))]
+                    (cond-> accum
+                      scaffold-target (assoc (into ["scaffold-target-id"] scaffold-target) dashcard-id))))
+                {}
+                ordered_cards))
+      (hydrate-details (db/select-one App :id app-id)))))
+
 (api/define-routes)

src/metabase/api/card.clj

@@ -322,7 +322,7 @@ saved later when it is ready."
                   (log/info (trs "Not updating metadata asynchronously for card {0} because query has changed"
                                  id)))))))))
 
-(defn- create-card!
+(defn create-card!
   "Create a new Card. Metadata will be fetched off thread. If the metadata takes longer than [[metadata-sync-wait-ms]]
   the card will be saved without metadata and it will be saved to the card in the future when it is ready."
   [{:keys [dataset_query result_metadata dataset parameters parameter_mappings], :as card-data}]

test/metabase/api/app_test.clj

 (ns metabase.api.app-test
   (:require
     [clojure.test :refer [deftest is testing]]
-    [metabase.models :refer [App Collection Dashboard]]
+    [medley.core :as m]
+    [metabase.models :refer [App Card Collection Dashboard]]
     [metabase.models.permissions :as perms]
     [metabase.models.permissions-group :as perms-group]
-    [metabase.test :as mt]))
+    [metabase.test :as mt]
+    [metabase.test.data :as data]
+    [toucan.db :as db]
+    [toucan.hydrate :refer [hydrate]]))
 
 (deftest create-test
   (mt/with-model-cleanup [Collection]
@@ -131,3 +135,33 @@
         (testing "that app detail properly checks permissions"
           (is (= "You don't have permissions to do that."
                  (mt/user-http-request :rasta :get 403 (str "app/" app-id)))))))))
+
+(deftest scaffold-test
+  (mt/with-model-cleanup [Card Dashboard Collection]
+    (testing "Golden path"
+      (let [app (mt/user-http-request
+                  :crowberto :post 200 "app/scaffold"
+                  {:table-ids [(data/id :venues)]
+                   :app-name (str "My test app " (gensym))})
+            pages (m/index-by :name (hydrate (db/select Dashboard :collection_id (:collection_id app)) :ordered_cards))
+            list-page (get pages "Venues List")
+            detail-page (get pages "Venues Detail")]
+        (is (partial= {:nav_items [{:page_id (:id list-page)}
+                                   {:page_id (:id detail-page) :hidden true :indent 1}]
+                       :dashboard_id (:id list-page)}
+                      app))
+        (is (partial= {:ordered_cards [{:visualization_settings {:click_behavior
+                                                                 {:type "link",
+                                                                  :linkType "dashboard",
+                                                                  :targetId (:id detail-page)}}}
+                                       {}]}
+                      list-page))))
+    (testing "Bad or duplicate tables"
+      (is (= (format "Some tables could not be found. Given: (%s %s) Found: (%s)"
+                     (data/id :venues)
+                     Integer/MAX_VALUE
+                     (data/id :venues))
+             (mt/user-http-request
+               :crowberto :post 400 "app/scaffold"
+               {:table-ids [(data/id :venues) (data/id :venues) Integer/MAX_VALUE]
+                :app-name (str "My test app " (gensym))}))))))