Skip to content
Snippets Groups Projects
Unverified Commit dd58aa16 authored by Cam Saul's avatar Cam Saul Committed by GitHub
Browse files

MLv2: Finish porting query description generation (#29507) 

* Improved implementation

* Amazing code cleanup

* Cleanup

* More tests
parent 9fada844
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 129 additions and 360 deletions
.clj-kondo/config.edn
+ 0
− 1
View file @ dd58aa16
......@@ -243,7 +243,6 @@
metabase.api.preview-embed api.preview-embed
metabase.api.public api.public
metabase.api.pulse api.pulse
metabase.api.query-description api.qd
metabase.api.revision api.revision
metabase.api.search api.search
metabase.api.segment api.segment
......
bin/kondo.sh
+ 3
− 0
View file @ dd58aa16
......@@ -8,6 +8,9 @@ set -euxo pipefail
clj-kondo --copy-configs --dependencies --lint "$(clojure -A:dev -Spath)" --skip-lint --parallel
rm -rf .clj-kondo/metosin/malli-types-clj/
rm -rf .clj-kondo/.cache
# Run Kondo against all of our Clojure files in the various directories they might live.
find modules/drivers shared enterprise/backend \
-maxdepth 2 \
......
frontend/src/metabase-lib/queries/utils/description.js deleted 100644 → 0
+ 0
− 156
View file @ 9fada844
import React from "react";
import { t } from "ttag";
import _ from "underscore";
import inflection from "inflection";
import { FilterClause, MetricClause } from "./description.styled.tsx";
// -----------------------------------------------------------------------------
// These functions use the `query_description` field returned by the Segment and
// Metric APIs. They're meant for cases where you do not have the full database
// metadata available, and the server side will generate a data structure
// containing all the applicable data for formatting a user-friendly description
// of a query.
//
// TODO: This is almost 100% duplicated with code in Question.ts, find a way to
// consolidate them
// -----------------------------------------------------------------------------
function formatTableDescription({ table }, options = {}) {
return [inflection.pluralize(table)];
}
function formatAggregationDescription({ aggregation }, options = {}) {
if (!aggregation || !aggregation.length) {
return [];
}
return conjunctList(
aggregation.map(agg => {
switch (agg["type"]) {
case "aggregation":
return [agg["arg"]];
case "metric":
return [
options.jsx ? (
<MetricClause>{agg["arg"]}</MetricClause>
) : (
agg["arg"]
),
];
case "rows":
return [t`Raw data`];
case "count":
return [t`Count`];
case "cum-count":
return [t`Cumulative count`];
case "avg":
return [t`Average of `, agg["arg"]];
case "median":
return [t`Median of `, agg["arg"]];
case "distinct":
return [t`Distinct values of `, agg["arg"]];
case "stddev":
return [t`Standard deviation of `, agg["arg"]];
case "sum":
return [t`Sum of `, agg["arg"]];
case "cum-sum":
return [t`Cumulative sum of `, agg["arg"]];
case "max":
return [t`Maximum of `, agg["arg"]];
case "min":
return [t`Minimum of `, agg["arg"]];
default: {
console.warn(
"Unexpected aggregation type in formatAggregationDescription: ",
agg["type"],
);
return null;
}
}
}),
);
}
function formatFilterDescription({ filter }, options = {}) {
if (!filter || !filter.length) {
return [];
}
return [
t`Filtered by `,
joinList(
filter.map(f => {
if (f["segment"] != null) {
return options.jsx ? (
<FilterClause>{f["segment"]}</FilterClause>
) : (
f["segment"]
);
} else if (f["field"] != null) {
return f["field"];
}
}),
", ",
),
];
}
export function formatQueryDescription(parts, options = {}) {
if (!parts) {
return "";
}
options = {
jsx: false,
sections: ["table", "aggregation", "filter"],
...options,
};
const sectionFns = {
table: formatTableDescription,
aggregation: formatAggregationDescription,
filter: formatFilterDescription,
};
// these array gymnastics are needed to support JSX formatting
const sections = (options.sections || [])
.map(section =>
_.flatten(sectionFns[section](parts, options)).filter(s => !!s),
)
.filter(s => s && s.length > 0);
const description = _.flatten(joinList(sections, ", "));
if (options.jsx) {
return <span>{description}</span>;
} else {
return description.join("");
}
}
function joinList(list, joiner) {
return _.flatten(
list.map((l, i) => (i === list.length - 1 ? [l] : [l, joiner])),
true,
);
}
function conjunctList(list, conjunction) {
switch (list.length) {
case 0:
return null;
case 1:
return list[0];
case 2:
return [list[0], " ", conjunction, " ", list[1]];
default:
return [
list.slice(0, -1).join(", "),
", ",
conjunction,
" ",
list[list.length - 1],
];
}
}
frontend/src/metabase-lib/queries/utils/description.styled.tsx deleted 100644 → 0
+ 0
− 12
View file @ 9fada844
import styled from "@emotion/styled";
import { color } from "metabase/lib/colors";
export const MetricClause = styled.span`
color: ${color("summarize")};
font-weight: 700;
`;
export const FilterClause = styled.span`
color: ${color("filter")};
font-weight: 700;
`;
frontend/src/metabase/admin/datamodel/components/MetricItem.jsx
+ 1
− 7
View file @ dd58aa16
......@@ -2,7 +2,6 @@ import React, { Component } from "react";
import PropTypes from "prop-types";
import Icon from "metabase/components/Icon";
import { formatQueryDescription } from "metabase-lib/queries/utils/description";
import ObjectActionSelect from "./ObjectActionSelect";
export default class MetricItem extends Component {
......@@ -14,11 +13,6 @@ export default class MetricItem extends Component {
render() {
const { metric, onRetire } = this.props;
const description = formatQueryDescription(metric.query_description, {
sections: ["table", "aggregation", "filter"],
jsx: true,
});
return (
<tr>
<td className="px1 py1 text-wrap">
......@@ -27,7 +21,7 @@ export default class MetricItem extends Component {
<span className="text-dark text-bold">{metric.name}</span>
</span>
</td>
<td className="px1 py1 text-wrap">{description}</td>
<td className="px1 py1 text-wrap">{metric.definition_description}</td>
<td className="px1 py1 text-centered">
<ObjectActionSelect
object={metric}
......
frontend/src/metabase/admin/datamodel/components/SegmentItem.jsx
+ 1
− 7
View file @ dd58aa16
......@@ -2,7 +2,6 @@ import React, { Component } from "react";
import PropTypes from "prop-types";
import Icon from "metabase/components/Icon";
import { formatQueryDescription } from "metabase-lib/queries/utils/description";
import ObjectActionSelect from "./ObjectActionSelect";
export default class SegmentItem extends Component {
......@@ -14,11 +13,6 @@ export default class SegmentItem extends Component {
render() {
const { segment, onRetire } = this.props;
const description = formatQueryDescription(segment.query_description, {
sections: ["filter"],
jsx: true,
});
return (
<tr className="mt1 mb3">
<td className="px1 py1 text-wrap">
......@@ -31,7 +25,7 @@ export default class SegmentItem extends Component {
<span className="text-dark text-bold">{segment.name}</span>
</span>
</td>
<td className="px1 py1 text-wrap">{description}</td>
<td className="px1 py1 text-wrap">{segment.definition_description}</td>
<td className="px1 py1 text-centered">
<ObjectActionSelect
object={segment}
......
shared/src/metabase/mbql/util.cljc
+ 12
− 0
View file @ dd58aa16
......@@ -733,6 +733,18 @@
(when (keyword? k)
(namespace k)))))))
(defn referenced-field-ids
"Find all the `:field` references with integer IDs in `coll`, which can be a full MBQL query, a snippet of MBQL, or a
sequence of those things; return a set of Field IDs. Includes Fields referenced indirectly via `:source-field`.
Returns `nil` if no IDs are found."
[coll]
(not-empty
(into #{}
(comp cat (filter some?))
(mbql.match/match coll
[:field (id :guard integer?) opts]
[id (:source-field opts)]))))
#?(:clj
(p/import-vars
[mbql.match
......
src/metabase/api/metric.clj
+ 7
− 17
View file @ dd58aa16
......@@ -4,7 +4,6 @@
[clojure.data :as data]
[compojure.core :refer [DELETE GET POST PUT]]
[metabase.api.common :as api]
[metabase.api.query-description :as api.qd]
[metabase.events :as events]
[metabase.mbql.normalize :as mbql.normalize]
[metabase.models :refer [Metric MetricImportantField Table]]
......@@ -14,6 +13,7 @@
[metabase.util :as u]
[metabase.util.i18n :refer [trs]]
[metabase.util.log :as log]
[metabase.util.malli.schema :as ms]
[metabase.util.schema :as su]
[schema.core :as s]
[toucan.db :as db]
......@@ -44,20 +44,11 @@
(-> (api/read-check (t2/select-one Metric :id id))
(hydrate :creator)))
(defn- add-query-descriptions
[metrics] {:pre [(coll? metrics)]}
(when (some? metrics)
(for [metric metrics]
(let [table (t2/select-one Table :id (:table_id metric))]
(assoc metric
:query_description
(api.qd/generate-query-description table (:definition metric)))))))
#_{:clj-kondo/ignore [:deprecated-var]}
(api/defendpoint-schema GET "/:id"
(api/defendpoint GET "/:id"
"Fetch `Metric` with ID."
[id]
(first (add-query-descriptions [(hydrated-metric id)])))
{id ms/PositiveInt}
(hydrated-metric id))
(defn- add-db-ids
"Add `:database_id` fields to `metrics` by looking them up from their `:table_id`."
......@@ -67,15 +58,14 @@
(for [metric metrics]
(assoc metric :database_id (table-id->db-id (:table_id metric)))))))
#_{:clj-kondo/ignore [:deprecated-var]}
(api/defendpoint-schema GET "/"
(api/defendpoint GET "/"
"Fetch *all* `Metrics`."
[]
(as-> (t2/select Metric, :archived false, {:order-by [:%lower.name]}) metrics
(hydrate metrics :creator)
(hydrate metrics :creator :definition_description)
(add-db-ids metrics)
(filter mi/can-read? metrics)
(add-query-descriptions metrics)))
metrics))
(defn- write-check-and-update-metric!
"Check whether current user has write permissions, then update Metric with values in `body`. Publishes appropriate
......
src/metabase/api/query_description.clj deleted 100644 → 0
+ 0
− 128
View file @ 9fada844
(ns metabase.api.query-description
"Functions for generating human friendly query descriptions"
(:require
[clojure.string :as str]
[metabase.mbql.predicates :as mbql.preds]
[metabase.mbql.util :as mbql.u]
[metabase.models.field :refer [Field]]
[metabase.models.metric :refer [Metric]]
[metabase.models.segment :refer [Segment]]
[metabase.util.i18n :refer [deferred-tru]]
[metabase.util.log :as log]
[toucan2.core :as t2]))
(defn- get-table-description
[metadata _query]
{:table (:display_name metadata)})
(defn- field-clause->display-name [clause]
(mbql.u/match-one clause
[:field (id :guard integer?) _]
(t2/select-one-fn :display_name Field :id id)
[:field (field-name :guard string?) _]
field-name))
(defn- get-aggregation-details
[metadata query]
(letfn [(field-name [match] (or (when (mbql.preds/Field? match)
(field-clause->display-name match))
(flatten (get-aggregation-details metadata match))))]
(when-let [agg-matches (mbql.u/match query
[:aggregation-options _ (options :guard :display-name)]
{:type :aggregation :arg (:display-name options)}
[:aggregation-options ag _]
#_:clj-kondo/ignore
(recur ag)
[(operator :guard #{:+ :- :/ :*}) & args]
(interpose (name operator) (map field-name args))
[:metric (arg :guard integer?)]
{:type :metric
:arg (let [metric-name (t2/select-one-fn :name Metric :id arg)]
(if-not (str/blank? metric-name)
metric-name
(deferred-tru "[Unknown Metric]")))}
[:rows] {:type :rows}
[:count] {:type :count}
[:cum-count] {:type :cum-count}
[:avg arg] {:type :avg :arg (field-name arg)}
[:distinct arg] {:type :distinct :arg (field-name arg)}
[:stddev arg] {:type :stddev :arg (field-name arg)}
[:sum arg] {:type :sum :arg (field-name arg)}
[:cum-sum arg] {:type :cum-sum :arg (field-name arg)}
[:max arg] {:type :max :arg (field-name arg)}
[:min arg] {:type :min :arg (field-name arg)})]
agg-matches)))
(defn- get-aggregation-description
[metadata query]
(when-let [details (get-aggregation-details metadata query)]
{:aggregation details}))
(defn- get-breakout-description
[_metadata query]
(when-let [breakouts (seq (:breakout query))]
{:breakout (map #(t2/select-one-fn :display_name Field :id %) breakouts)}))
(defn- get-filter-clause-description
[_metadata filt]
(let [typ (first filt)]
(condp = typ
:field {:field (field-clause->display-name filt)}
:segment {:segment (let [segment (t2/select-one Segment :id (second filt))]
(if segment
(:name segment)
(deferred-tru "[Unknown Segment]")))}
nil)))
(defn- get-filter-description
[metadata query]
(when-let [filters (:filter query)]
{:filter (map #(get-filter-clause-description metadata %)
(mbql.u/match filters #{:field :segment} &match))}))
(defn- get-order-by-description
[_metadata query]
(when (:order-by query)
{:order-by (map (fn [[direction field]]
{:field (field-clause->display-name field)
:direction direction})
(mbql.u/match query #{:asc :desc} &match))}))
(defn- get-limit-description
[_metadata query]
(when-let [limit (:limit query)]
{:limit limit}))
(def ^:private query-descriptor-functions
[get-table-description
get-aggregation-description
get-breakout-description
get-filter-description
get-order-by-description
get-limit-description])
(defn generate-query-description
"Analyze a query and return a data structure with the parts broken down for display
in the UI.
Ex:
{
:table \"Orders\"
:filters [\"Created At\", \"Product ID\"]
:order-by [{\"Created At\" :asc}]
}
This data structure allows the UI to format the strings appropriately (including JSX)"
[metadata query]
(try
(apply merge
(map (fn [f] (f metadata query))
query-descriptor-functions))
(catch Exception e
(log/warn e "Error generating query description")
{})))
src/metabase/api/segment.clj
+ 6
− 20
View file @ dd58aa16
......@@ -3,17 +3,16 @@
(:require
[compojure.core :refer [DELETE GET POST PUT]]
[metabase.api.common :as api]
[metabase.api.query-description :as api.qd]
[metabase.events :as events]
[metabase.mbql.normalize :as mbql.normalize]
[metabase.models.interface :as mi]
[metabase.models.revision :as revision]
[metabase.models.segment :as segment :refer [Segment]]
[metabase.models.table :as table :refer [Table]]
[metabase.related :as related]
[metabase.util :as u]
[metabase.util.i18n :refer [trs]]
[metabase.util.log :as log]
[metabase.util.malli.schema :as ms]
[metabase.util.schema :as su]
[schema.core :as s]
[toucan.hydrate :refer [hydrate]]
......@@ -43,29 +42,18 @@
(-> (api/read-check (t2/select-one Segment :id id))
(hydrate :creator)))
(defn- add-query-descriptions
[segments] {:pre [(coll? segments)]}
(when (some? segments)
(for [segment segments]
(let [table (t2/select-one Table :id (:table_id segment))]
(assoc segment
:query_description
(api.qd/generate-query-description table (:definition segment)))))))
#_{:clj-kondo/ignore [:deprecated-var]}
(api/defendpoint-schema GET "/:id"
(api/defendpoint GET "/:id"
"Fetch `Segment` with ID."
[id]
(first (add-query-descriptions [(hydrated-segment id)])))
{id ms/PositiveInt}
(hydrated-segment id))
#_{:clj-kondo/ignore [:deprecated-var]}
(api/defendpoint-schema GET "/"
(api/defendpoint GET "/"
"Fetch *all* `Segments`."
[]
(as-> (t2/select Segment, :archived false, {:order-by [[:%lower.name :asc]]}) segments
(filter mi/can-read? segments)
(hydrate segments :creator)
(add-query-descriptions segments)))
(hydrate segments :creator :definition_description)))
(defn- write-check-and-update-segment!
"Check whether current user has write permissions, then update Segment with values in `body`. Publishes appropriate
......@@ -114,7 +102,6 @@
(write-check-and-update-segment! id {:archived true, :revision_message revision_message})
api/generic-204-no-content)
#_{:clj-kondo/ignore [:deprecated-var]}
(api/defendpoint-schema GET "/:id/revisions"
"Fetch `Revisions` for `Segment` with ID."
......@@ -122,7 +109,6 @@
(api/read-check Segment id)
(revision/revisions+details Segment id))
#_{:clj-kondo/ignore [:deprecated-var]}
(api/defendpoint-schema POST "/:id/revert"
"Revert a `Segement` to a prior `Revision`."
......
src/metabase/lib/aggregation.cljc
+ 1
− 1
View file @ dd58aa16
......@@ -24,7 +24,7 @@
:stage-number stage-number})))
(nth aggregations index)))
(defmethod lib.metadata.calculation/describe-top-level-key :aggregation
(defmethod lib.metadata.calculation/describe-top-level-key-method :aggregation
[query stage-number _k]
(when-let [aggregations (not-empty (:aggregation (lib.util/query-stage query stage-number)))]
(lib.util/join-strings-with-conjunction
......
src/metabase/lib/breakout.cljc
+ 1
− 1
View file @ dd58aa16
......@@ -9,7 +9,7 @@
[metabase.shared.util.i18n :as i18n]
[metabase.util.malli :as mu]))
(defmethod lib.metadata.calculation/describe-top-level-key :breakout
(defmethod lib.metadata.calculation/describe-top-level-key-method :breakout
[query stage-number _k]
(when-let [breakouts (not-empty (:breakout (lib.util/query-stage query stage-number)))]
(i18n/tru "Grouped by {0}"
......
src/metabase/lib/convert.cljc
+ 9
− 0
View file @ dd58aa16
......@@ -68,6 +68,15 @@
(let [[tag opts & args] (->pMBQL aggregation)]
(into [tag (merge opts options)] args)))
(defn legacy-query-from-inner-query
"Convert a legacy 'inner query' to a full legacy 'outer query' so you can pass it to stuff
like [[metabase.mbql.normalize/normalize]], and then probably to [[->pMBQL]]."
[database-id inner-query]
(merge {:database database-id, :type :query}
(if (:native inner-query)
{:native (set/rename-keys inner-query {:native :query})}
{:query inner-query})))
(defmulti ->legacy-MBQL
"Coerce something to legacy MBQL (the version of MBQL understood by the query processor and Metabase Lib v1) if it's
not already legacy MBQL."
......
src/metabase/lib/core.cljc
+ 3
− 0
View file @ dd58aa16
......@@ -16,6 +16,7 @@
[metabase.lib.metric :as lib.metric]
[metabase.lib.order-by :as lib.order-by]
[metabase.lib.query :as lib.query]
[metabase.lib.segment :as lib.segment]
[metabase.lib.stage :as lib.stage]
[metabase.lib.table :as lib.table]
[metabase.lib.temporal-bucket :as lib.temporal-bucket]
......@@ -33,6 +34,7 @@
lib.metric/keep-me
lib.order-by/keep-me
lib.query/keep-me
lib.segment/keep-me
lib.stage/keep-me
lib.table/keep-me
lib.temporal-bucket/keep-me)
......@@ -131,6 +133,7 @@
[lib.metadata.calculation
column-name
describe-query
describe-top-level-key
display-name
suggested-name]
[lib.order-by
......
src/metabase/lib/field.cljc
+ 6
− 4
View file @ dd58aa16
......@@ -105,10 +105,12 @@
(defmethod lib.metadata.calculation/display-name-method :field
[query stage-number [_field {:keys [join-alias temporal-unit], :as _opts} _id-or-name, :as field-clause]]
(let [field-metadata (cond-> (resolve-field-metadata query stage-number field-clause)
join-alias (assoc :source_alias join-alias)
temporal-unit (assoc :unit temporal-unit))]
(lib.metadata.calculation/display-name query stage-number field-metadata)))
(if-let [field-metadata (cond-> (resolve-field-metadata query stage-number field-clause)
join-alias (assoc :source_alias join-alias)
temporal-unit (assoc :unit temporal-unit))]
(lib.metadata.calculation/display-name query stage-number field-metadata)
;; mostly for the benefit of JS, which does not enforce the Malli schemas.
(i18n/tru "[Unknown Field]")))
(defmulti ^:private ->field
{:arglists '([query stage-number field])}
......
src/metabase/lib/filter.cljc
+ 1
− 1
View file @ dd58aa16
......@@ -16,7 +16,7 @@
(comment metabase.lib.schema/keep-me)
(defmethod lib.metadata.calculation/describe-top-level-key :filter
(defmethod lib.metadata.calculation/describe-top-level-key-method :filter
[query stage-number _key]
(when-let [filter-clause (:filter (lib.util/query-stage query stage-number))]
(i18n/tru "Filtered by {0}" (lib.metadata.calculation/display-name query stage-number filter-clause))))
......
src/metabase/lib/js/metadata.cljs
+ 2
− 2
View file @ dd58aa16
......@@ -87,7 +87,7 @@
(fn [object]
(try
(let [parsed (assoc (obj->clj xform object) :lib/type lib-type-name)]
(log/infof "Parsed metadata %s %s\n%s" object-type (:id parsed) (u/pprint-to-str parsed))
(log/debugf "Parsed metadata %s %s\n%s" object-type (:id parsed) (u/pprint-to-str parsed))
parsed)
(catch js/Error e
(log/errorf e "Error parsing %s %s: %s" object-type (pr-str object) (ex-message e))
......@@ -304,7 +304,7 @@
"Use a `metabase-lib/metadata/Metadata` as a [[metabase.lib.metadata.protocols/MetadataProvider]]."
[database-id metadata]
(let [metadata (parse-metadata metadata)]
(log/info "Created metadata provider for metadata")
(log/debug "Created metadata provider for metadata")
(reify lib.metadata.protocols/MetadataProvider
(database [_this] (database metadata database-id))
(table [_this table-id] (table metadata table-id))
......
src/metabase/lib/limit.cljc
+ 1
− 1
View file @ dd58aa16
......@@ -7,7 +7,7 @@
[metabase.shared.util.i18n :as i18n]
[metabase.util.malli :as mu]))
(defmethod lib.metadata.calculation/describe-top-level-key :limit
(defmethod lib.metadata.calculation/describe-top-level-key-method :limit
[query stage-number _k]
(when-let [limit (:limit (lib.util/query-stage query stage-number))]
(str limit \space (i18n/trun "row" "rows" limit))))
......
src/metabase/lib/metadata.cljc
+ 4
− 2
View file @ dd58aa16
......@@ -44,7 +44,8 @@
[:map
[:lib/type [:= :metadata/field]] ; TODO -- should this be changed to `:metadata/column`?
[:id {:optional true} ::lib.schema.id/field]
[:name ::lib.schema.common/non-blank-string]])
[:name ::lib.schema.common/non-blank-string]
[:display_name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]])
(def ^:private CardMetadata
[:map
......@@ -69,7 +70,8 @@
[:lib/type [:= :metadata/table]]
[:id ::lib.schema.id/table]
[:name ::lib.schema.common/non-blank-string]
[:schema {:optional true} [:maybe ::lib.schema.common/non-blank-string]]
[:display_name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]
[:schema {:optional true} [:maybe ::lib.schema.common/non-blank-string]]
;; This is now optional! If the [[MetadataProvider]] provides it, great, but if not we can always make the
;; subsequent request to fetch fields separately.
[:fields {:optional true} [:maybe [:sequential ColumnMetadata]]]
......
src/metabase/lib/metadata/cached_provider.cljc 0 → 100644
+ 71
− 0
View file @ dd58aa16
(ns metabase.lib.metadata.cached-provider
(:require
[clojure.set :as set]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
[metabase.util.log :as log]
#?@(:clj ([pretty.core :as pretty]))))
(defn- get-in-cache [cache ks]
(when-some [cached-value (get-in @cache ks)]
(when-not (= cached-value ::nil)
cached-value)))
(defn- store-in-cache! [cache ks value]
(let [value (if (some? value) value ::nil)]
(swap! cache assoc-in ks value)
value))
(defn- get-in-cache-or-fetch [cache ks fetch-thunk]
(if-some [cached-value (get-in @cache ks)]
(when-not (= cached-value ::nil)
cached-value)
(store-in-cache! cache ks (fetch-thunk))))
(defn- bulk-metadata [cache uncached-provider metadata-type ids]
(when (seq ids)
(log/debugf "Getting %s metadata with IDs %s" metadata-type (pr-str (sort ids)))
(let [existing-ids (set (keys (get @cache metadata-type)))
missing-ids (set/difference (set ids) existing-ids)]
(log/debugf "Already fetched %s: %s" metadata-type (pr-str (sort (set/intersection (set ids) existing-ids))))
(when (seq missing-ids)
(log/debugf "Need to fetch %s: %s" metadata-type (pr-str (sort missing-ids)))
;; TODO -- we should probably store `::nil` markers for things we tried to fetch that didn't exist
(doseq [instance (lib.metadata.protocols/bulk-metadata uncached-provider metadata-type missing-ids)]
(store-in-cache! cache [metadata-type (:id instance)] instance))))
(for [id ids]
(get-in-cache cache [metadata-type id]))))
(deftype CachedProxyMetadataProvider [cache metadata-provider]
lib.metadata.protocols/MetadataProvider
(database [_this] (get-in-cache-or-fetch cache [:metadata/database] #(lib.metadata.protocols/database metadata-provider)))
(table [_this table-id] (get-in-cache-or-fetch cache [:metadata/table table-id] #(lib.metadata.protocols/table metadata-provider table-id)))
(field [_this field-id] (get-in-cache-or-fetch cache [:metadata/field field-id] #(lib.metadata.protocols/field metadata-provider field-id)))
(card [_this card-id] (get-in-cache-or-fetch cache [:metadata/card card-id] #(lib.metadata.protocols/card metadata-provider card-id)))
(metric [_this metric-id] (get-in-cache-or-fetch cache [:metadata/metric metric-id] #(lib.metadata.protocols/metric metadata-provider metric-id)))
(segment [_this segment-id] (get-in-cache-or-fetch cache [:metadata/segment segment-id] #(lib.metadata.protocols/segment metadata-provider segment-id)))
(tables [_this] (get-in-cache-or-fetch cache [::database-tables] #(lib.metadata.protocols/tables metadata-provider)))
(fields [_this table-id] (get-in-cache-or-fetch cache [::table-fields table-id] #(lib.metadata.protocols/fields metadata-provider table-id)))
lib.metadata.protocols/CachedMetadataProvider
(cached-database [_this] (get-in-cache cache [:metadata/database]))
(cached-metadata [_this metadata-type id] (get-in-cache cache [metadata-type id]))
(store-database! [_this database-metadata] (store-in-cache! cache [:metadata/database] (assoc database-metadata :lib/type :metadata/database)))
(store-metadata! [_this metadata-type id metadata] (store-in-cache! cache [metadata-type id] (assoc metadata :lib/type metadata-type)))
;; these only work if the underlying metadata provider is also a [[BulkMetadataProvider]].
lib.metadata.protocols/BulkMetadataProvider
(bulk-metadata [_this metadata-type ids]
(bulk-metadata cache metadata-provider metadata-type ids))
#?@(:clj
[pretty.core/PrettyPrintable
(pretty [_this]
(list `->CachedProxyMetadataProvider cache metadata-provider))]))
(defn cached-metadata-provider
"Wrap `metadata-provider` with an implementation that automatically caches results.
If the metadata provider implements [[lib.metadata.protocols/BulkMetadataProvider]],
then [[lib.metadata.protocols/bulk-metadata]] will work as expected; it can be done for side-effects as well."
^CachedProxyMetadataProvider [metadata-provider]
(->CachedProxyMetadataProvider (atom {}) metadata-provider))
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment