Merge pull request #4490 from metabase/caching

In-DB Caching

frontend/src/metabase/admin/settings/components/SettingsSetting.jsx

@@ -3,6 +3,7 @@ import React, { Component, PropTypes } from "react";
 import SettingHeader from "./SettingHeader.jsx";
 
 import SettingInput from "./widgets/SettingInput.jsx";
+import SettingNumber from "./widgets/SettingNumber.jsx";
 import SettingPassword from "./widgets/SettingPassword.jsx";
 import SettingRadio from "./widgets/SettingRadio.jsx";
 import SettingToggle from "./widgets/SettingToggle.jsx";
@@ -10,6 +11,7 @@ import SettingSelect from "./widgets/SettingSelect.jsx";
 
 const SETTING_WIDGET_MAP = {
     "string":   SettingInput,
+    "number":   SettingNumber,
     "password": SettingPassword,
     "select":   SettingSelect,
     "radio":    SettingRadio,

frontend/src/metabase/admin/settings/components/widgets/SettingNumber.jsx

+import React from "react";
+
+import SettingInput from "./SettingInput";
+
+const SettingNumber = ({ type = "number", ...props }) =>
+    <SettingInput {...props} type="number" />
+
+export default SettingNumber;

frontend/src/metabase/admin/settings/selectors.js

@@ -86,7 +86,7 @@ const SECTIONS = [
                 key: "email-smtp-port",
                 display_name: "SMTP Port",
                 placeholder: "587",
-                type: "string",
+                type: "number",
                 required: true,
                 validations: [["integer", "That's not a valid port number"]]
             },
@@ -236,6 +236,34 @@ const SECTIONS = [
                 getHidden: (settings) => !settings["enable-embedding"]
             }
         ]
+    },
+    {
+        name: "Caching",
+        settings: [
+            {
+                key: "enable-query-caching",
+                display_name: "Enable Caching",
+                type: "boolean"
+            },
+            {
+                key: "query-caching-min-ttl",
+                display_name: "Minimum Query Duration",
+                type: "number",
+                getHidden: (settings) => !settings["enable-query-caching"]
+            },
+            {
+                key: "query-caching-ttl-ratio",
+                display_name: "Cache Time-To-Live (TTL)",
+                type: "number",
+                getHidden: (settings) => !settings["enable-query-caching"]
+            },
+            {
+                key: "query-caching-max-kb",
+                display_name: "Max Cache Entry Size",
+                type: "number",
+                getHidden: (settings) => !settings["enable-query-caching"]
+            }
+        ]
     }
 ];
 for (const section of SECTIONS) {

frontend/src/metabase/components/ShrinkableList.jsx

+/* @flow */
+
+import React, { Component, PropTypes } from "react";
+import ReactDOM from "react-dom";
+
+import ExplicitSize from "metabase/components/ExplicitSize";
+
+type Props = {
+    className?: string,
+    items: any[],
+    renderItem: (item: any) => any,
+    renderItemSmall: (item: any) => any
+};
+
+type State = {
+    isShrunk: ?boolean
+};
+
+@ExplicitSize
+export default class ShrinkableList extends Component<*, Props, State> {
+    state: State = {
+        isShrunk: null
+    }
+
+    componentWillReceiveProps() {
+        this.setState({
+            isShrunk: null
+        })
+    }
+
+    componentDidMount() {
+        this.componentDidUpdate();
+    }
+
+    componentDidUpdate() {
+        const container = ReactDOM.findDOMNode(this)
+        const { isShrunk } = this.state;
+        if (container && isShrunk === null) {
+            this.setState({
+                isShrunk: container.scrollWidth !== container.offsetWidth
+            })
+        }
+    }
+
+    render() {
+        const { items, className, renderItemSmall, renderItem } = this.props;
+        const { isShrunk } = this.state;
+        return (
+            <div className={className}>
+                { items.map(item =>
+                    isShrunk ?
+                        renderItemSmall(item)
+                    :
+                        renderItem(item)
+                )}
+            </div>
+        );
+    }
+}

frontend/src/metabase/components/Tooltip.jsx

@@ -14,7 +14,7 @@ export default class Tooltip extends Component {
     }
 
     static propTypes = {
-        tooltip: PropTypes.node.isRequired,
+        tooltip: PropTypes.node,
         children: PropTypes.element.isRequired,
         isEnabled: PropTypes.bool,
         verticalAttachments: PropTypes.array,

frontend/src/metabase/css/core/flex.css

@@ -32,6 +32,10 @@
     justify-content: space-between;
 }
 
+.justify-end {
+    justify-content: flex-end;
+}
+
 .align-start {
     align-items: flex-start;
 }

frontend/src/metabase/css/query_builder.css

@@ -518,12 +518,13 @@
     z-index: 1;
     opacity: 1;
     box-shadow: 0 1px 2px rgba(0, 0, 0, .22);
-    transition: margin-top 0.5s, opacity 0.5s;
+    transition: transform 0.5s, opacity 0.5s;
     min-width: 8em;
+    position: relative;
 }
 
 .RunButton.RunButton--hidden {
-    margin-top: -110px;
+    transform: translateY(-65px);
     opacity: 0;
 }
 

frontend/src/metabase/qb/components/TimeseriesFilterWidget.jsx

@@ -34,7 +34,7 @@ type Props = {
     card: CardObject,
     tableMetadata: TableMetadata,
     setDatasetQuery: (datasetQuery: DatasetQuery) => void,
-    runQueryFn: () => void
+    runQuery: () => void
 };
 
 type State = {
@@ -90,7 +90,7 @@ export default class TimeseriesFilterWidget extends Component<*, Props, State> {
             card,
             tableMetadata,
             setDatasetQuery,
-            runQueryFn
+            runQuery
         } = this.props;
         const { filter, filterIndex, currentFilter } = this.state;
         let currentDescription;
@@ -148,7 +148,7 @@ export default class TimeseriesFilterWidget extends Component<*, Props, State> {
                                     ...card.dataset_query,
                                     query
                                 });
-                                runQueryFn();
+                                runQuery();
                             }
                             if (this._popover) {
                                 this._popover.close();

frontend/src/metabase/qb/components/TimeseriesGroupingWidget.jsx

@@ -20,14 +20,14 @@ import type {
 type Props = {
     card: CardObject,
     setDatasetQuery: (datasetQuery: DatasetQuery) => void,
-    runQueryFn: () => void
+    runQuery: () => void
 };
 
 export default class TimeseriesGroupingWidget extends Component<*, Props, *> {
     _popover: ?any;
 
     render() {
-        const { card, setDatasetQuery, runQueryFn } = this.props;
+        const { card, setDatasetQuery, runQuery } = this.props;
         if (Card.isStructured(card)) {
             const query = Card.getQuery(card);
             const breakouts = query && Query.getBreakouts(query);
@@ -62,7 +62,7 @@ export default class TimeseriesGroupingWidget extends Component<*, Props, *> {
                                     ...card.dataset_query,
                                     query
                                 });
-                                runQueryFn();
+                                runQuery();
                                 if (this._popover) {
                                     this._popover.close();
                                 }

frontend/src/metabase/qb/components/modes/TimeseriesMode.jsx

@@ -29,7 +29,7 @@ type Props = {
     lastRunCard: CardObject,
     tableMetadata: TableMetadata,
     setDatasetQuery: (datasetQuery: DatasetQuery) => void,
-    runQueryFn: () => void
+    runQuery: () => void
 };
 
 export const TimeseriesModeFooter = (props: Props) => {

frontend/src/metabase/query_builder/actions.js

@@ -223,7 +223,7 @@ export const initializeQB = createThunkAction(INITIALIZE_QB, (location, params)
         if (card && card.dataset_query && (Query.canRun(card.dataset_query.query) || card.dataset_query.type === "native")) {
             // NOTE: timeout to allow Parameters widget to set parameterValues
             setTimeout(() =>
-                dispatch(runQuery(card, false))
+                dispatch(runQuery(card, { shouldUpdateUrl: false }))
             , 0);
         }
 
@@ -285,7 +285,7 @@ export const cancelEditing = createThunkAction(CANCEL_EDITING, () => {
         dispatch(loadMetadataForCard(card));
 
         // we do this to force the indication of the fact that the card should not be considered dirty when the url is updated
-        dispatch(runQuery(card, false));
+        dispatch(runQuery(card, { shouldUpdateUrl: false }));
         dispatch(updateUrl(card, { dirty: false }));
 
         MetabaseAnalytics.trackEvent("QueryBuilder", "Edit Cancel");
@@ -466,7 +466,7 @@ export const reloadCard = createThunkAction(RELOAD_CARD, () => {
         dispatch(loadMetadataForCard(card));
 
         // we do this to force the indication of the fact that the card should not be considered dirty when the url is updated
-        dispatch(runQuery(card, false));
+        dispatch(runQuery(card, { shouldUpdateUrl: false }));
         dispatch(updateUrl(card, { dirty: false }));
 
         return card;
@@ -482,7 +482,7 @@ export const setCardAndRun = createThunkAction(SET_CARD_AND_RUN, (runCard, shoul
 
         dispatch(loadMetadataForCard(card));
 
-        dispatch(runQuery(card, shouldUpdateUrl));
+        dispatch(runQuery(card, { shouldUpdateUrl: shouldUpdateUrl }));
 
         return card;
     };
@@ -850,7 +850,11 @@ export const removeQueryExpression = createQueryAction(
 
 // runQuery
 export const RUN_QUERY = "metabase/qb/RUN_QUERY";
-export const runQuery = createThunkAction(RUN_QUERY, (card, shouldUpdateUrl = true, parameterValues, dirty) => {
+export const runQuery = createThunkAction(RUN_QUERY, (card, {
+    shouldUpdateUrl = true,
+    ignoreCache = false, // currently only implemented for saved cards
+    parameterValues
+} = {}) => {
     return async (dispatch, getState) => {
         const state = getState();
         const parameters = getParameters(state);
@@ -880,8 +884,12 @@ export const runQuery = createThunkAction(RUN_QUERY, (card, shouldUpdateUrl = tr
         const datasetQuery = applyParameters(card, parameters, parameterValues);
 
         // use the CardApi.query if the query is saved and not dirty so users with view but not create permissions can see it.
-        if (card.id && !cardIsDirty) {
-            CardApi.query({ cardId: card.id, parameters: datasetQuery.parameters }, { cancelled: cancelQueryDeferred.promise }).then(onQuerySuccess, onQueryError);
+        if (card.id != null && !cardIsDirty) {
+            CardApi.query({
+                cardId: card.id,
+                parameters: datasetQuery.parameters,
+                ignore_cache: ignoreCache
+            }, { cancelled: cancelQueryDeferred.promise }).then(onQuerySuccess, onQueryError);
         } else {
             MetabaseApi.dataset(datasetQuery, { cancelled: cancelQueryDeferred.promise }).then(onQuerySuccess, onQueryError);
         }
@@ -1108,8 +1116,6 @@ export const toggleDataReferenceFn = toggleDataReference;
 export const onBeginEditing = beginEditing;
 export const onCancelEditing = cancelEditing;
 export const setQueryModeFn = setQueryMode;
-export const runQueryFn = runQuery;
-export const cancelQueryFn = cancelQuery;
 export const setDatabaseFn = setQueryDatabase;
 export const setSourceTableFn = setQuerySourceTable;
 export const setDisplayFn = setCardVisualization;

frontend/src/metabase/query_builder/components/NativeQueryEditor.jsx

@@ -90,7 +90,7 @@ export default class NativeQueryEditor extends Component {
         nativeDatabases: PropTypes.array.isRequired,
         datasetQuery: PropTypes.object.isRequired,
         setDatasetQuery: PropTypes.func.isRequired,
-        runQueryFn: PropTypes.func.isRequired,
+        runQuery: PropTypes.func.isRequired,
         setDatabaseFn: PropTypes.func.isRequired,
         autocompleteResultsFn: PropTypes.func.isRequired,
         isOpen: PropTypes.bool,
@@ -163,10 +163,10 @@ export default class NativeQueryEditor extends Component {
                 const selectedText = this._editor.getSelectedText();
                 if (selectedText) {
                     const temporaryCard = assocIn(card, ["dataset_query", "native", "query"], selectedText);
-                    this.props.runQueryFn(temporaryCard, false, null, true);
+                    this.props.runQuery(temporaryCard, { shouldUpdateUrl: false });
                 }
             } else {
-                this.props.runQueryFn();
+                this.props.runQuery();
             }
         }
     }

frontend/src/metabase/query_builder/components/QueryVisualization.jsx

@@ -2,6 +2,10 @@ import React, { Component, PropTypes } from "react";
 import { Link } from "react-router";
 
 import LoadingSpinner from 'metabase/components/LoadingSpinner.jsx';
+import Tooltip from "metabase/components/Tooltip";
+import Icon from "metabase/components/Icon";
+import ShrinkableList from "metabase/components/ShrinkableList";
+
 import RunButton from './RunButton.jsx';
 import VisualizationSettings from './VisualizationSettings.jsx';
 
@@ -12,13 +16,16 @@ import Warnings from "./Warnings.jsx";
 import QueryDownloadWidget from "./QueryDownloadWidget.jsx";
 import QuestionEmbedWidget from "../containers/QuestionEmbedWidget";
 
-import { formatNumber, inflect } from "metabase/lib/formatting";
+import { formatNumber, inflect, duration } from "metabase/lib/formatting";
 import Utils from "metabase/lib/utils";
 import MetabaseSettings from "metabase/lib/settings";
 import * as Urls from "metabase/lib/urls";
 
 import cx from "classnames";
 import _ from "underscore";
+import moment from "moment";
+
+const REFRESH_TOOLTIP_THRESHOLD = 30 * 1000; // 30 seconds
 
 export default class QueryVisualization extends Component {
     constructor(props, context) {
@@ -40,8 +47,8 @@ export default class QueryVisualization extends Component {
         cellClickedFn: PropTypes.func,
         isRunning: PropTypes.bool.isRequired,
         isRunnable: PropTypes.bool.isRequired,
-        runQueryFn: PropTypes.func.isRequired,
-        cancelQueryFn: PropTypes.func
+        runQuery: PropTypes.func.isRequired,
+        cancelQuery: PropTypes.func
     };
 
     static defaultProps = {
@@ -63,44 +70,85 @@ export default class QueryVisualization extends Component {
         }
     }
 
-    queryIsDirty() {
-        // a query is considered dirty if ANY part of it has been changed
-        return (
-            !Utils.equals(this.props.card.dataset_query, this.state.lastRunDatasetQuery) ||
-            !Utils.equals(this.props.parameterValues, this.state.lastRunParameterValues)
-        );
-    }
-
     isChartDisplay(display) {
         return (display !== "table" && display !== "scalar");
     }
 
+    runQuery = () => {
+        this.props.runQuery(null, { ignoreCache: true });
+    }
+
     renderHeader() {
-        const { isObjectDetail, isRunnable, isRunning, isAdmin, card, result, runQueryFn, cancelQueryFn } = this.props;
-        const isDirty = this.queryIsDirty();
+        const { isObjectDetail, isRunnable, isRunning, isResultDirty, isAdmin, card, result, cancelQuery } = this.props;
         const isSaved = card.id != null;
+
+        let runButtonTooltip;
+        if (!isResultDirty && result && result.cached && result.average_execution_time > REFRESH_TOOLTIP_THRESHOLD) {
+            runButtonTooltip = `This question will take approximately ${duration(result.average_execution_time)} to refresh`;
+        }
+
+        const messages = [];
+        if (result && result.cached) {
+            messages.push({
+                icon: "clock",
+                message: (
+                    <div>
+                        Updated {moment(result.updated_at).fromNow()}
+                    </div>
+                )
+            })
+        }
+        if (result && result.data && !isObjectDetail && card.display === "table") {
+            messages.push({
+                icon: "table2",
+                message: (
+                    <div>
+                        { result.data.rows_truncated != null ? ("Showing first ") : ("Showing ")}
+                        <strong>{formatNumber(result.row_count)}</strong>
+                        { " " + inflect("row", result.data.rows.length) }
+                    </div>
+                )
+            })
+        }
+
         const isPublicLinksEnabled = MetabaseSettings.get("public_sharing");
         const isEmbeddingEnabled = MetabaseSettings.get("embedding");
         return (
-            <div className="relative flex flex-no-shrink mt3 mb1" style={{ minHeight: "2em" }}>
-                <span className="relative z4">
+            <div className="relative flex align-center flex-no-shrink mt2 mb1" style={{ minHeight: "2em" }}>
+                <div className="z4 flex-full">
                   { !isObjectDetail && <VisualizationSettings ref="settings" {...this.props} /> }
-                </span>
-                <div className="absolute flex layout-centered left right z3">
-                    <RunButton
-                        isRunnable={isRunnable}
-                        isDirty={isDirty}
-                        isRunning={isRunning}
-                        onRun={runQueryFn}
-                        onCancel={cancelQueryFn}
-                    />
                 </div>
-                <div className="absolute right z4 flex align-center" style={{ lineHeight: 0 /* needed to align icons :-/ */ }}>
-                    { !isDirty && this.renderCount() }
+                <div className="z3">
+                    <Tooltip tooltip={runButtonTooltip}>
+                        <RunButton
+                            isRunnable={isRunnable}
+                            isDirty={isResultDirty}
+                            isRunning={isRunning}
+                            onRun={this.runQuery}
+                            onCancel={cancelQuery}
+                        />
+                    </Tooltip>
+                </div>
+                <div className="z4 flex-full flex align-center justify-end" style={{ lineHeight: 0 /* needed to align icons :-/ */ }}>
+                    <ShrinkableList
+                        className="flex"
+                        items={messages}
+                        renderItem={(item) =>
+                            <div className="flex-no-shrink flex align-center mx2 h5 text-grey-4">
+                                <Icon className="mr1" name={item.icon} size={12} />
+                                {item.message}
+                            </div>
+                        }
+                        renderItemSmall={(item) =>
+                            <Tooltip tooltip={<div className="p1">{item.message}</div>}>
+                                <Icon className="mx1" name={item.icon} size={16} />
+                            </Tooltip>
+                        }
+                    />
                     { !isObjectDetail &&
-                        <Warnings warnings={this.state.warnings} className="mx2" size={18} />
+                        <Warnings warnings={this.state.warnings} className="mx1" size={18} />
                     }
-                    { !isDirty && result && !result.error ?
+                    { !isResultDirty && result && !result.error ?
                         <QueryDownloadWidget
                             className="mx1"
                             card={card}
@@ -121,19 +169,6 @@ export default class QueryVisualization extends Component {
         );
     }
 
-    renderCount() {
-        let { result, isObjectDetail, card } = this.props;
-        if (result && result.data && !isObjectDetail && card.display === "table") {
-            return (
-                <div>
-                    { result.data.rows_truncated != null ? ("Showing first ") : ("Showing ")}
-                    <b>{formatNumber(result.row_count)}</b>
-                    { " " + inflect("row", result.data.rows.length) }.
-                </div>
-            );
-        }
-    }
-
     render() {
         const { className, card, databases, isObjectDetail, isRunning, result } = this.props
         let viz;

frontend/src/metabase/query_builder/components/dataref/DataReference.jsx

@@ -26,7 +26,7 @@ export default class DataReference extends Component {
     static propTypes = {
         query: PropTypes.object.isRequired,
         onClose: PropTypes.func.isRequired,
-        runQueryFn: PropTypes.func.isRequired,
+        runQuery: PropTypes.func.isRequired,
         setDatasetQuery: PropTypes.func.isRequired,
         setDatabaseFn: PropTypes.func.isRequired,
         setSourceTableFn: PropTypes.func.isRequired,

frontend/src/metabase/query_builder/components/dataref/FieldPane.jsx

@@ -28,7 +28,7 @@ export default class FieldPane extends Component {
         field: PropTypes.object.isRequired,
         datasetQuery: PropTypes.object,
         loadTableAndForeignKeysFn: PropTypes.func.isRequired,
-        runQueryFn: PropTypes.func.isRequired,
+        runQuery: PropTypes.func.isRequired,
         setDatasetQuery: PropTypes.func.isRequired,
         setCardAndRun: PropTypes.func.isRequired
     };
@@ -63,7 +63,7 @@ export default class FieldPane extends Component {
         }
         Query.addBreakout(datasetQuery.query, this.props.field.id);
         this.props.setDatasetQuery(datasetQuery);
-        this.props.runQueryFn();
+        this.props.runQuery();
     }
 
     newCard() {

frontend/src/metabase/query_builder/components/dataref/MetricPane.jsx

@@ -26,7 +26,7 @@ export default class MetricPane extends Component {
         metric: PropTypes.object.isRequired,
         query: PropTypes.object,
         loadTableAndForeignKeysFn: PropTypes.func.isRequired,
-        runQueryFn: PropTypes.func.isRequired,
+        runQuery: PropTypes.func.isRequired,
         setDatasetQuery: PropTypes.func.isRequired,
         setCardAndRun: PropTypes.func.isRequired
     };

frontend/src/metabase/query_builder/components/dataref/SegmentPane.jsx

@@ -27,7 +27,7 @@ export default class SegmentPane extends Component {
         segment: PropTypes.object.isRequired,
         datasetQuery: PropTypes.object,
         loadTableAndForeignKeysFn: PropTypes.func.isRequired,
-        runQueryFn: PropTypes.func.isRequired,
+        runQuery: PropTypes.func.isRequired,
         setDatasetQuery: PropTypes.func.isRequired,
         setCardAndRun: PropTypes.func.isRequired
     };
@@ -53,7 +53,7 @@ export default class SegmentPane extends Component {
         }
         Query.addFilter(datasetQuery.query, ["SEGMENT", this.props.segment.id]);
         this.props.setDatasetQuery(datasetQuery);
-        this.props.runQueryFn();
+        this.props.runQuery();
     }
 
     newCard() {

project.clj

@@ -50,6 +50,7 @@
                  [com.mattbertolini/liquibase-slf4j "2.0.0"]          ; Java Migrations lib
                  [com.mchange/c3p0 "0.9.5.2"]                         ; connection pooling library
                  [com.novemberain/monger "3.1.0"]                     ; MongoDB Driver
+                 [com.taoensso/nippy "2.13.0"]                        ; Fast serialization (i.e., GZIP) library for Clojure
                  [compojure "1.5.2"]                                  ; HTTP Routing library built on Ring
                  [crypto-random "1.2.0"]                              ; library for generating cryptographically secure random bytes and strings
                  [environ "1.1.0"]                                    ; easy environment management

reset_password/metabase/reset_password/core.clj

@@ -14,10 +14,10 @@
 (defn -main
   [email-address]
   (mdb/setup-db!)
-  (println (format "Resetting password for %s..." email-address))
+  (printf "Resetting password for %s...\n" email-address)
   (try
-    (println (format "OK [[[%s]]]" (set-reset-token! email-address)))
+    (printf "OK [[[%s]]]\n" (set-reset-token! email-address))
     (System/exit 0)
     (catch Throwable e
-      (println (format "FAIL [[[%s]]]" (.getMessage e)))
+      (printf "FAIL [[[%s]]]\n" (.getMessage e))
       (System/exit -1))))

resources/migrations/052_add_query_cache_table.yaml

+databaseChangeLog:
+  - property:
+        name: blob.type
+        value: blob
+        dbms: mysql,h2
+  - property:
+      name: blob.type
+      value: bytea
+      dbms: postgresql
+  - changeSet:
+      id: 52
+      author: camsaul
+      changes:
+        - createTable:
+            tableName: query_cache
+            remarks: 'Cached results of queries are stored here when using the DB-based query cache.'
+            columns:
+              - column:
+                  name: query_hash
+                  type: binary(32)
+                  remarks: 'The hash of the query dictionary. (This is a 256-bit SHA3 hash of the query dict).'
+                  constraints:
+                    primaryKey: true
+                    nullable: false
+              - column:
+                  name: updated_at
+                  type: datetime
+                  remarks: 'The timestamp of when these query results were last refreshed.'
+                  constraints:
+                    nullable: false
+              - column:
+                  name: results
+                  type: ${blob.type}
+                  remarks: 'Cached, compressed results of running the query with the given hash.'
+                  constraints:
+                    nullable: false
+        - createIndex:
+            tableName: query_cache
+            indexName: idx_query_cache_updated_at
+            columns:
+              column:
+                name: updated_at
+        - addColumn:
+            tableName: report_card
+            columns:
+              - column:
+                  name: cache_ttl
+                  type: int
+                  remarks: 'The maximum time, in seconds, to return cached results for this Card rather than running a new query.'