diff --git a/frontend/src/metabase-lib/lib/Question.js b/frontend/src/metabase-lib/lib/Question.js
index e9169b0e3e4335c3ed9362de1ac9037188f5f0ae..f3c73e9b6d97e1068f09a63b4f2ac1da9d03b953 100644
--- a/frontend/src/metabase-lib/lib/Question.js
+++ b/frontend/src/metabase-lib/lib/Question.js
@@ -99,7 +99,7 @@ export default class Question {
tableId?: TableId,
metadata: Metadata,
parameterValues?: ParameterValues
- }
+ } = {}
) {
// $FlowFixMe
const card: Card = {
diff --git a/frontend/src/metabase/components/TermWithDefinition.jsx b/frontend/src/metabase/components/TermWithDefinition.jsx
new file mode 100644
index 0000000000000000000000000000000000000000..bca65ee4e184219cfb44632521ae38b9bfb5298b
--- /dev/null
+++ b/frontend/src/metabase/components/TermWithDefinition.jsx
@@ -0,0 +1,16 @@
+import React from 'react'
+import cxs from "cxs";
+import Tooltip from "metabase/components/Tooltip";
+
+const termStyles = cxs({
+ textDecoration: "none",
+ borderBottom: '1px dotted #DCE1E4'
+})
+export const TermWithDefinition = ({ children, definition, link }) =>
+ <Tooltip tooltip={definition}>
+ { link
+ ? <a href={link} className={termStyles} target="_blank">{ children }</a>
+ : <span className={termStyles}>{ children }</span>
+ }
+ </Tooltip>
+
diff --git a/frontend/src/metabase/icon_paths.js b/frontend/src/metabase/icon_paths.js
index f32d67f73aaeb697bbeacd75ccf19a410a63cfe5..c0970a906f3c302c66f2a497df0902d58956f283 100644
--- a/frontend/src/metabase/icon_paths.js
+++ b/frontend/src/metabase/icon_paths.js
@@ -129,6 +129,7 @@ export var ICON_PATHS = {
},
info: 'M16 0 A16 16 0 0 1 16 32 A16 16 0 0 1 16 0 M19 15 L13 15 L13 26 L19 26 z M16 6 A3 3 0 0 0 16 12 A3 3 0 0 0 16 6',
infooutlined: 'M16 29c7.18 0 13-5.82 13-13S23.18 3 16 3 3 8.82 3 16s5.82 13 13 13zm0 3C7.163 32 0 24.837 0 16S7.163 0 16 0s16 7.163 16 16-7.163 16-16 16zm1.697-20h-4.185v14h4.185V12zm.432-3.834c0-.342-.067-.661-.203-.958a2.527 2.527 0 0 0-1.37-1.31 2.613 2.613 0 0 0-.992-.188c-.342 0-.661.062-.959.189a2.529 2.529 0 0 0-1.33 1.309c-.13.297-.195.616-.195.958 0 .334.065.646.196.939.13.292.31.549.54.77.23.22.492.395.79.526.297.13.616.196.958.196.351 0 .682-.066.992-.196.31-.13.583-.306.817-.527a2.47 2.47 0 0 0 .553-.77c.136-.292.203-.604.203-.938z',
+ insight: 'M12.6325203 19.3674797 0 16 12.6325203 12.6325203 16 0 19.3674797 12.6325203 32 16 19.3674797 19.3674797 16 32z',
int: {
path: 'M15.141,15.512 L14.294,20 L13.051,20 C12.8309989,20 12.6403341,19.9120009 12.479,19.736 C12.3176659,19.5599991 12.237,19.343668 12.237,19.087 C12.237,19.0503332 12.2388333,19.0155002 12.2425,18.9825 C12.2461667,18.9494998 12.2516666,18.9146668 12.259,18.878 L12.908,15.512 L10.653,15.512 L10.015,19.01 C9.94899967,19.3620018 9.79866784,19.6149992 9.564,19.769 C9.32933216,19.9230008 9.06900143,20 8.783,20 L7.584,20 L8.42,15.512 L7.155,15.512 C6.92033216,15.512 6.74066729,15.4551672 6.616,15.3415 C6.49133271,15.2278328 6.429,15.0390013 6.429,14.775 C6.429,14.6723328 6.43999989,14.5550007 6.462,14.423 L6.605,13.554 L8.695,13.554 L9.267,10.518 L6.913,10.518 L7.122,9.385 C7.17333359,9.10633194 7.28699912,8.89916734 7.463,8.7635 C7.63900088,8.62783266 7.92499802,8.56 8.321,8.56 L9.542,8.56 L10.224,5.018 C10.282667,4.7246652 10.4183323,4.49733414 10.631,4.336 C10.8436677,4.17466586 11.0929986,4.094 11.379,4.094 L12.611,4.094 L11.775,8.56 L14.019,8.56 L14.866,4.094 L16.076,4.094 C16.3326679,4.094 16.5416659,4.1673326 16.703,4.314 C16.8643341,4.4606674 16.945,4.64766553 16.945,4.875 C16.945,4.9483337 16.9413334,5.00333315 16.934,5.04 L16.252,8.56 L18.485,8.56 L18.276,9.693 C18.2246664,9.97166806 18.1091676,10.1788327 17.9295,10.3145 C17.7498324,10.4501673 17.4656686,10.518 17.077,10.518 L15.977,10.518 L15.416,13.554 L16.978,13.554 C17.2126678,13.554 17.3904994,13.6108328 17.5115,13.7245 C17.6325006,13.8381672 17.693,14.0306653 17.693,14.302 C17.693,14.4046672 17.6820001,14.5219993 17.66,14.654 L17.528,15.512 L15.141,15.512 Z M10.928,13.554 L13.183,13.554 L13.744,10.518 L11.5,10.518 L10.928,13.554 Z',
attrs: { viewBox: '0 0 24 24' }
diff --git a/frontend/src/metabase/lib/formatting.js b/frontend/src/metabase/lib/formatting.js
index 245a18ecf467855b31051ee5cc25155863dd36c0..895de42af5ee9c797cc8e08847e7289a4fcf281f 100644
--- a/frontend/src/metabase/lib/formatting.js
+++ b/frontend/src/metabase/lib/formatting.js
@@ -361,6 +361,11 @@ export function slugify(name: string) {
return name && name.toLowerCase().replace(/[^a-z0-9_]/g, "_");
}
+// Adds commas and spaces between array items and replaces the comma between two last items with "and"
+export function formatListOfItems(items: []): string {
+ return [items.slice(0, -1).join(', '), items.slice(-1)[0]].join(items.length < 2 ? '' : ' and ');
+}
+
export function assignUserColors(userIds: number[], currentUserId: number, colorClasses: string[] = ['bg-brand', 'bg-purple', 'bg-error', 'bg-green', 'bg-gold', 'bg-grey-2']) {
let assignments = {};
diff --git a/frontend/src/metabase/xray/components/InsightCard.jsx b/frontend/src/metabase/xray/components/InsightCard.jsx
new file mode 100644
index 0000000000000000000000000000000000000000..15a95418623bf0d2bb1c36b7bcfaa5ee4db5d1aa
--- /dev/null
+++ b/frontend/src/metabase/xray/components/InsightCard.jsx
@@ -0,0 +1,332 @@
+import React, { Component } from 'react'
+import { formatTimeWithUnit } from "metabase/lib/formatting";
+import Icon from "metabase/components/Icon";
+import { Link } from "react-router";
+import Question from "metabase-lib/lib/Question";
+import { TermWithDefinition } from "metabase/components/TermWithDefinition";
+import { t, jt } from 'c-3po'
+
+const InsightText = ({ children }) =>
+ <p className="text-paragraph">
+ {children}
+ </p>
+
+const Feedback = ({ insightType }) =>
+ <div className="flex align-center px1">
+ {t`Was this helpful?`}
+ <div className="ml-auto text-bold">
+ <a className="text-brand-hover" data-metabase-event={`InsightFeedback;${insightType};Yes`}>
+ {t`Yes`}
+ </a>
+ <a className="text-brand-hover ml1" data-metabase-event={`InsightFeedback;${insightType};No`}>
+ {t`No`}
+ </a>
+ </div>
+ </div>
+
+export class NormalRangeInsight extends Component {
+ static insightType = "normal-range"
+ static title = t`Normal range of values`
+ static icon = "insight"
+
+ render() {
+ const { lower, upper, features: { model } } = this.props
+ return (
+ <InsightText>
+ { jt`Most of the values for ${ model.display_name || model.name } are between ${<b>{ lower }</b>} and ${<b>{ upper }</b>}.` }
+ </InsightText>
+ )
+ }
+}
+
+export class NilsInsight extends Component {
+ static insightType = "nils"
+ static title = t`Missing data`
+ static icon = "warning"
+
+ render() {
+ const { quality, filter, features: { table } } = this.props
+
+ const viewAllRowsUrl = table && Question.create()
+ .query()
+ // imitate the required hydrated metadata format
+ .setTable({ ...table, database: { id: table.db_id }})
+ .addFilter(filter)
+ .question()
+ .getUrl()
+
+ // construct the question with filter
+ return (
+ <InsightText>
+ {t`You have ${ quality } missing (null) values in your data`}.
+ <span> </span>
+ { table && <span><Link to={viewAllRowsUrl}>View all rows</Link> with missing value.</span> }
+ </InsightText>
+ )
+ }
+}
+
+export class ZerosInsight extends Component {
+ static insightType = "zeros"
+ static title = t`Zeros in your data`
+ static icon = "warning"
+
+ render() {
+ const { quality, filter, features: { table } } = this.props
+
+ const viewAllRowsUrl = table && Question.create()
+ .query()
+ // imitate the required hydrated metadata format
+ .setTable({ ...table, database: { id: table.db_id }})
+ .addFilter(filter)
+ .question()
+ .getUrl()
+
+ // construct the question with filter
+ return (
+ <InsightText>
+ { t`You have ${ quality } zeros in your data. They may be standins for missing data or indicate some other abnormality.` }
+ <span> </span>
+ { table && <span><Link to={viewAllRowsUrl}>View all rows</Link> with zeros.</span> }
+ </InsightText>
+ )
+ }
+}
+
+const noisinessDefinition = t`Noisy data is highly variable jumping all over the place with changes carrying relatively little information.`
+const noisinessLink = "https://en.wikipedia.org/wiki/Noisy_data"
+
+export class NoisinessInsight extends Component {
+ static insightType = "noisiness"
+ static title = t`Noisy data`
+ static icon = "warning"
+
+ render() {
+ const { quality, "recommended-resolution": resolution } = this.props
+
+ return (
+ <InsightText>
+ Your data is { quality }
+ <span> </span>
+ <TermWithDefinition definition={noisinessDefinition} link={noisinessLink}>
+ noisy
+ </TermWithDefinition>.
+ { resolution && ` You might consider looking at it by ${resolution}.` }
+ </InsightText>
+ )
+ }
+}
+
+const autocorrelationDefinition = t`Measure of how much (changes in) previous values predict future values.`
+const autocorrelationLink = "https://en.wikipedia.org/wiki/Autocorrelation"
+
+export class AutocorrelationInsight extends Component {
+ static insightType = "autocorrelation"
+ static title = t`Autocorrelation`
+ static icon = "insight"
+
+ render() {
+ const { quality, lag } = this.props
+
+ return (
+ <InsightText>
+ Your data has a { quality } <TermWithDefinition definition={autocorrelationDefinition} link={autocorrelationLink}>
+ autocorrelation
+ </TermWithDefinition> at lag { lag }.
+ </InsightText>
+ )
+ }
+}
+
+const variationTrendDefinition = t`How variance in your data is changing over time.`
+const varianceLink = "https://en.wikipedia.org/wiki/Variance"
+
+export class VariationTrendInsight extends Component {
+ static insightType = "variation-trend"
+ static title = t`Trending variation`
+ static icon = "insight"
+
+ render() {
+ const { mode } = this.props
+
+ return (
+ <InsightText>
+ Looks like this data has grown { mode }ly <TermWithDefinition definition={variationTrendDefinition} link={varianceLink}>
+ varied</TermWithDefinition> over time.
+ </InsightText>
+ )
+ }
+}
+
+export class SeasonalityInsight extends Component {
+ static insightType = "seasonality"
+ static title = t`Seasonality`
+ static icon = "insight"
+
+ render() {
+ const { quality } = this.props
+
+ return (
+ <InsightText>
+ { jt`Your data has a ${ quality } seasonal compoment.` }
+ </InsightText>
+ )
+ }
+}
+
+const multimodalDefinition = t`Data distribution with multiple peaks (modes).`
+const multimodalLink = "https://en.wikipedia.org/wiki/Multimodal_distribution"
+
+export class MultimodalInsight extends Component {
+ static insightType = "multimodal"
+ static title = t`Multimodal`
+ static icon = "warning"
+
+ render() {
+ return (
+ <InsightText>
+ Your data looks to be <TermWithDefinition definition={multimodalDefinition} link={multimodalLink}>
+ multimodal
+ </TermWithDefinition>. This is often the case when different segments of data are mixed together.
+ </InsightText>
+ )
+ }
+}
+
+export class OutliersInsight extends Component {
+ static insightType = "outliers"
+ static title = t`Outliers`
+ static icon = "warning"
+
+ render() {
+ const { filter, features: { table } } = this.props
+
+ const viewAllRowsUrl = table && Question.create()
+ .query()
+ // imitate the required hydrated metadata format
+ .setTable({ ...table, database: { id: table.db_id }})
+ .addFilter(filter)
+ .question()
+ .getUrl()
+
+ // construct the question with filter
+ return (
+ <InsightText>
+ You have some outliers.
+ <span> </span>
+ { table && <span><Link to={viewAllRowsUrl}>View all rows</Link> with outliers.</span> }
+ </InsightText>
+ )
+ }
+}
+
+export class StructuralBreaksInsight extends Component {
+ static insightType = "structural-breaks"
+ static title = t`Structural breaks`
+ static icon = "insight"
+
+ render() {
+ const { breaks, features: { resolution } } = this.props
+
+ const breakPoints = breaks.map( (point, idx) => {
+ point = formatTimeWithUnit(point, resolution);
+
+ if (idx == breaks.length - 1 && breaks.length > 1) {
+ return (
+ <span>, and { point }</span>
+ )
+ } else {
+ return (
+ <span>{ idx > 0 && <span>, </span>}{ point }</span>
+ )
+ }
+ })
+
+ return (
+ <InsightText>
+ It looks like your data has
+ { breaks.length > 1 && <span> structural breaks </span>}
+ { breaks.length == 1 && <span> a structural break </span>}
+ at { breakPoints }.
+ </InsightText>
+ )
+ }
+}
+
+const stationaryDefinition = t`Mean does not change with time.`
+const stationaryLink = "https://en.wikipedia.org/wiki/Stationary_process"
+
+export class StationaryInsight extends Component {
+ static insightType = "stationary"
+ static title = t`Stationary data`
+ static icon = "insight"
+
+ render() {
+ return (
+ <InsightText>
+ Your data looks to be <TermWithDefinition definition={stationaryDefinition} link={stationaryLink}>
+ stationary</TermWithDefinition>.
+ </InsightText>
+ )
+ }
+}
+
+export class TrendInsight extends Component {
+ static insightType = "trend"
+ static title = t`Trend`
+ static icon = "insight"
+
+ render() {
+ const { mode, shape } = this.props
+
+ return(
+ <InsightText>
+ { jt`Your data seems to be ${ mode } ${ shape }.` }
+ </InsightText>
+ )
+ }
+}
+
+const INSIGHT_COMPONENTS = [
+ // any field
+ NilsInsight,
+ // numeric fields
+ NormalRangeInsight,
+ ZerosInsight,
+ MultimodalInsight,
+ OutliersInsight,
+ // timeseries
+ NoisinessInsight,
+ VariationTrendInsight,
+ AutocorrelationInsight,
+ SeasonalityInsight,
+ StructuralBreaksInsight,
+ StationaryInsight,
+ TrendInsight,
+]
+
+export const InsightCard = ({type, props, features}) => {
+ const Insight = INSIGHT_COMPONENTS.find((component) => component.insightType === type)
+
+ return (
+ <div>
+ <div className="bg-white bordered rounded shadowed p3" style={{ height: 180 }}>
+ <header className="flex align-center">
+ <Icon
+ name={Insight.icon}
+ size={24}
+ className="mr1"
+ style={{ color: '#93a1ab' }}
+ />
+ <span className="text-bold text-uppercase">{Insight.title}</span>
+ </header>
+ <div style={{ lineHeight: '1.4em' }}>
+ <Insight {...props} features={features} />
+ </div>
+ </div>
+ <div className="mt1">
+ <Feedback insightType={type} />
+ </div>
+ </div>
+ )
+}
diff --git a/frontend/src/metabase/xray/components/Insights.jsx b/frontend/src/metabase/xray/components/Insights.jsx
new file mode 100644
index 0000000000000000000000000000000000000000..5dbf9f5ceacf1616a7f7dbb73d935344387d573a
--- /dev/null
+++ b/frontend/src/metabase/xray/components/Insights.jsx
@@ -0,0 +1,27 @@
+import React, { Component } from 'react'
+import { InsightCard } from "metabase/xray/components/InsightCard";
+
+export class Insights extends Component {
+ props: {
+ features: any,
+ }
+
+ render() {
+ const { features } = this.props;
+
+ const parametrizedInsights = Object.entries(features["insights"])
+ // temporary hacks as we have two formats
+ .filter(([key, value]) => !key.includes(2))
+ .map(([key, value]) => [key.replace("1",""), value])
+
+ return (
+ <ol className="Grid Grid--gutters Grid--1of4">
+ { parametrizedInsights.map(([type, props], index) =>
+ <div className="Grid-cell">
+ <InsightCard key={index} type={type} props={props} features={features} />
+ </div>
+ )}
+ </ol>
+ )
+ }
+}
diff --git a/frontend/src/metabase/xray/components/StatGroup.jsx b/frontend/src/metabase/xray/components/StatGroup.jsx
index acdf7b9cff30792b2e46a3d179a6756dcba73692..76ef0c8d0f37a615fe19e9473be827b26f76973f 100644
--- a/frontend/src/metabase/xray/components/StatGroup.jsx
+++ b/frontend/src/metabase/xray/components/StatGroup.jsx
@@ -12,14 +12,14 @@ const StatGroup = ({ heading, xray, stats, showDescriptions }) =>
<div className="bordered rounded shadowed bg-white">
<ol className="Grid Grid--1of4">
{ stats.map(stat =>
- !!xray[stat] && xray[stat].value && (
+ (!!xray[stat] && xray[stat].value) ? (
<li className="Grid-cell p1 px2 md-p2 md-px3 lg-p3 lg-px4 border-right border-bottom" key={stat}>
<SimpleStat
stat={xray[stat]}
showDescription={showDescriptions}
/>
</li>
- )
+ ) : null
)}
</ol>
</div>
diff --git a/frontend/src/metabase/xray/containers/CardXRay.jsx b/frontend/src/metabase/xray/containers/CardXRay.jsx
index e2c7469a12ffb0507e2bff548036210f5aead7b0..5083579ee505316b23f6206e97dd358b2dff4bf3 100644
--- a/frontend/src/metabase/xray/containers/CardXRay.jsx
+++ b/frontend/src/metabase/xray/containers/CardXRay.jsx
@@ -22,6 +22,7 @@ import Visualization from 'metabase/visualizations/components/Visualization'
import { XRayPageWrapper, Heading } from 'metabase/xray/components/XRayLayout'
import Periodicity from 'metabase/xray/components/Periodicity'
import LoadingAnimation from 'metabase/xray/components/LoadingAnimation'
+import { Insights } from "metabase/xray/components/Insights";
const mapStateToProps = state => ({
xray: getXray(state),
@@ -96,6 +97,12 @@ class CardXRay extends Component {
<div className="mt4 mb2">
<h1 className="my3">{xray.features.model.name} X-ray</h1>
</div>
+ { xray.features["insights"] &&
+ <div className="mt4">
+ <Heading heading="Takeaways" />
+ <Insights features={xray.features} />
+ </div>
+ }
<Heading heading="Growth rate" />
<div className="bg-white bordered rounded shadowed">
<div className="Grid Grid--1of4 border-bottom">
diff --git a/frontend/src/metabase/xray/containers/FieldXray.jsx b/frontend/src/metabase/xray/containers/FieldXray.jsx
index 597fa9aef8067ae1dddb312a8f52506af0e50163..1c110f23745e0bdef03b6ced28d830eb0ba18c64 100644
--- a/frontend/src/metabase/xray/containers/FieldXray.jsx
+++ b/frontend/src/metabase/xray/containers/FieldXray.jsx
@@ -33,6 +33,7 @@ import LoadingAnimation from 'metabase/xray/components/LoadingAnimation'
import type { Field } from 'metabase/meta/types/Field'
import type { Table } from 'metabase/meta/types/Table'
+import { Insights } from "metabase/xray/components/Insights";
type Props = {
fetchFieldXray: () => void,
@@ -44,7 +45,8 @@ type Props = {
table: Table,
histogram: {
value: {}
- }
+ },
+ insights: []
},
params: {
cost: string,
@@ -135,6 +137,12 @@ class FieldXRay extends Component {
</p>
</div>
</div>
+ { features["insights"] &&
+ <div className="mt4">
+ <Heading heading="Takeaways" />
+ <Insights features={features} />
+ </div>
+ }
<div className="mt4">
<Heading heading="Distribution" />
<div className="bg-white bordered shadowed">
diff --git a/frontend/test/metabase-lib/Question.integ.spec.js b/frontend/test/metabase-lib/Question.integ.spec.js
index d787bf8983fc68147ac36b9f662680481b386efb..63262360645426c12b0e6904f0807d359f65c042 100644
--- a/frontend/test/metabase-lib/Question.integ.spec.js
+++ b/frontend/test/metabase-lib/Question.integ.spec.js
@@ -46,7 +46,7 @@ describe("Question", () => {
question._parameterValues = { [templateTagId]: "5" };
const results2 = await question.getResults({ ignoreCache: true });
expect(results2[0]).toBeDefined();
- expect(results2[0].data.rows[0][0]).toEqual(18.1);
+ expect(results2[0].data.rows[0][0]).toEqual(116.35497575401975);
});
it("should return correct result with an optional template tag clause", async () => {
@@ -79,7 +79,7 @@ describe("Question", () => {
question._parameterValues = { [templateTagId]: "5" };
const results2 = await question.getResults({ ignoreCache: true });
expect(results2[0]).toBeDefined();
- expect(results2[0].data.rows[0][0]).toEqual(18.1);
+ expect(results2[0].data.rows[0][0]).toEqual(116.35497575401975);
});
});
});
diff --git a/frontend/test/parameters/parameters.integ.spec.js b/frontend/test/parameters/parameters.integ.spec.js
index 5b957b042eca903ebafe6025a360e82545e7cadf..06877cc9a4b7c60919c0ad2b80621879b539ce52 100644
--- a/frontend/test/parameters/parameters.integ.spec.js
+++ b/frontend/test/parameters/parameters.integ.spec.js
@@ -60,8 +60,8 @@ const getRelativeUrlWithoutHash = (url) =>
url.replace(/#.*$/, "").replace(/http:\/\/.*?\//, "/")
const COUNT_ALL = "200";
-const COUNT_DOOHICKEY = "56";
-const COUNT_GADGET = "43";
+const COUNT_DOOHICKEY = "51";
+const COUNT_GADGET = "47";
describe("parameters", () => {
beforeAll(async () =>
diff --git a/frontend/test/query_builder/query_builder.integ.spec.js b/frontend/test/query_builder/query_builder.integ.spec.js
index 1a56d2063547b569216c1e940835502974ef63de..f246577b428d815a938f3ea0aef356ec51385fd4 100644
--- a/frontend/test/query_builder/query_builder.integ.spec.js
+++ b/frontend/test/query_builder/query_builder.integ.spec.js
@@ -451,7 +451,7 @@ describe("QueryBuilder", () => {
await store.waitForActions([QUERY_COMPLETED]);
// We can use the visible row count as we have a low number of result rows
- expect(qb.find(".ShownRowCount").text()).toBe("Showing 6 rows");
+ expect(qb.find(".ShownRowCount").text()).toBe("Showing 14 rows");
// Get the binning
const results = getQueryResults(store.getState())[0]
@@ -478,7 +478,7 @@ describe("QueryBuilder", () => {
click(qb.find(RunButton));
await store.waitForActions([QUERY_COMPLETED]);
- expect(qb.find(".ShownRowCount").text()).toBe("Showing 95 rows");
+ expect(qb.find(".ShownRowCount").text()).toBe("Showing 253 rows");
const results = getQueryResults(store.getState())[0]
const breakoutBinningInfo = results.data.cols[0].binning_info;
expect(breakoutBinningInfo.binning_strategy).toBe("num-bins");
@@ -602,10 +602,10 @@ describe("QueryBuilder", () => {
const firstRowCells = table.find("tbody tr").first().find("td");
expect(firstRowCells.length).toBe(2);
- expect(firstRowCells.first().text()).toBe("12 – 14");
+ expect(firstRowCells.first().text()).toBe("4 – 6");
const countCell = firstRowCells.last();
- expect(countCell.text()).toBe("387");
+ expect(countCell.text()).toBe("2");
click(countCell.children().first());
// Drill-through is delayed in handleVisualizationClick of Visualization.jsx by 100ms
@@ -646,7 +646,7 @@ describe("QueryBuilder", () => {
expect(firstRowCells.first().text()).toBe("AA");
const countCell = firstRowCells.last();
- expect(countCell.text()).toBe("417");
+ expect(countCell.text()).toBe("233");
click(countCell.children().first());
// Drill-through is delayed in handleVisualizationClick of Visualization.jsx by 100ms
@@ -690,7 +690,7 @@ describe("QueryBuilder", () => {
expect(firstRowCells.first().text()).toBe("90° S – 80° S");
const countCell = firstRowCells.last();
- expect(countCell.text()).toBe("1,079");
+ expect(countCell.text()).toBe("701");
click(countCell.children().first());
// Drill-through is delayed in handleVisualizationClick of Visualization.jsx by 100ms
@@ -796,7 +796,7 @@ describe("QueryBuilder", () => {
expect(firstRowCells.length).toBe(6);
- expect(firstRowCells.at(4).text()).toBe("Enjoyable");
+ expect(firstRowCells.at(4).text()).toBe("Perfecto");
})
});
@@ -816,7 +816,7 @@ describe("QueryBuilder", () => {
expect(firstRowCells.length).toBe(6);
- expect(firstRowCells.at(3).text()).toBe("Ergonomic Leather Pants");
+ expect(firstRowCells.at(3).text()).toBe("Awesome Wooden Pants");
})
});
diff --git a/frontend/test/xray/xray.integ.spec.js b/frontend/test/xray/xray.integ.spec.js
index aa462e7ed4801f6b6d2b27d397708939416c8727..1cc5330233c5b2d82b062a2b8e4ff03b6b753114 100644
--- a/frontend/test/xray/xray.integ.spec.js
+++ b/frontend/test/xray/xray.integ.spec.js
@@ -18,8 +18,10 @@ import { delay } from "metabase/lib/promise";
import {
FETCH_CARD_XRAY,
FETCH_FIELD_XRAY,
- FETCH_SEGMENT_XRAY, FETCH_SHARED_TYPE_COMPARISON_XRAY,
- FETCH_TABLE_XRAY, FETCH_TWO_TYPES_COMPARISON_XRAY
+ FETCH_SEGMENT_XRAY,
+ FETCH_SHARED_TYPE_COMPARISON_XRAY,
+ FETCH_TABLE_XRAY,
+ FETCH_TWO_TYPES_COMPARISON_XRAY
} from "metabase/xray/xray";
import FieldXray from "metabase/xray/containers/FieldXray";
@@ -50,6 +52,12 @@ import { ComparisonDropdown } from "metabase/xray/components/ComparisonDropdown"
import { TestPopover } from "metabase/components/Popover";
import ItemLink from "metabase/xray/components/ItemLink";
import { TableLikeComparisonXRay } from "metabase/xray/containers/TableLikeComparison";
+import {
+ InsightCard,
+ NoisinessInsight,
+ NormalRangeInsight,
+ AutocorrelationInsight
+} from "metabase/xray/components/InsightCard";
describe("xray integration tests", () => {
let segmentId = null;
@@ -137,9 +145,9 @@ describe("xray integration tests", () => {
})
describe("field x-rays", async () => {
- it("should render the field x-ray page without errors", async () => {
+ it("should render the field x-ray page with expected insights", async () => {
const store = await createTestStore()
- store.pushPath(`/xray/field/1/approximate`);
+ store.pushPath(`/xray/field/2/approximate`);
const app = mount(store.getAppContainer());
await store.waitForActions([FETCH_FIELD_XRAY], { timeout: 20000 })
@@ -148,6 +156,8 @@ describe("xray integration tests", () => {
expect(fieldXRay.length).toBe(1)
expect(fieldXRay.find(CostSelect).length).toBe(1)
+ expect(app.find(InsightCard).length > 0).toBe(true)
+ expect(app.find(NormalRangeInsight).length).toBe(1)
})
})
@@ -292,12 +302,17 @@ describe("xray integration tests", () => {
click(xrayOptionIcon);
- await store.waitForActions([FETCH_CARD_XRAY], {timeout: 5000})
+ await store.waitForActions([FETCH_CARD_XRAY], {timeout: 20000})
expect(store.getPath()).toBe(`/xray/card/${timeBreakoutQuestion.id()}/extended`)
const cardXRay = app.find(CardXRay)
expect(cardXRay.length).toBe(1)
expect(cardXRay.text()).toMatch(/Time breakout question/);
+
+ // Should contain the expected insights
+ expect(app.find(InsightCard).length > 0).toBe(true)
+ expect(app.find(NoisinessInsight).length).toBe(1)
+ expect(app.find(AutocorrelationInsight).length).toBe(1)
})
it("let you see segment xray for a question containing a segment", async () => {
@@ -312,7 +327,7 @@ describe("xray integration tests", () => {
const xrayOptionIcon = actionsWidget.find('.Icon.Icon-beaker')
click(xrayOptionIcon);
- await store.waitForActions([FETCH_SEGMENT_XRAY], { timeout: 5000 })
+ await store.waitForActions([FETCH_SEGMENT_XRAY], { timeout: 20000 })
expect(store.getPath()).toBe(`/xray/segment/${segmentId}/approximate`)
const segmentXRay = app.find(SegmentXRay)
@@ -332,7 +347,7 @@ describe("xray integration tests", () => {
app = mount(store.getAppContainer())
- await store.waitForActions([LOAD_CURRENT_USER, INITIALIZE_SETTINGS])
+ await store.waitForActions([LOAD_CURRENT_USER, INITIALIZE_SETTINGS], { timeout: 20000 })
const xraySettings = app.find(SettingsXrayForm)
const xrayToggle = xraySettings.find(Toggle)
diff --git a/project.clj b/project.clj
index 4d29769f92c1aeaca7a46afa87a777efba136bc4..46e5884a8c7ea925af02c4a74a0e6517044f05b7 100644
--- a/project.clj
+++ b/project.clj
@@ -63,7 +63,7 @@
[hiccup "1.0.5"] ; HTML templating
[honeysql "0.8.2"] ; Transform Clojure data structures to SQL
[io.crate/crate-jdbc "2.1.6"] ; Crate JDBC driver
- [kixi/stats "0.3.9" ; Various statistic measures implemented as transducers
+ [kixi/stats "0.3.10" ; Various statistic measures implemented as transducers
:exclusions [org.clojure/test.check ; test.check and AVL trees are used in kixi.stats.random. Remove exlusion if using.
org.clojure/data.avl]]
[log4j/log4j "1.2.17" ; logging framework
@@ -74,7 +74,10 @@
[medley "0.8.4"] ; lightweight lib of useful functions
[metabase/throttle "1.0.1"] ; Tools for throttling access to API endpoints and other code pathways
[mysql/mysql-connector-java "5.1.39"] ; !!! Don't upgrade to 6.0+ yet -- that's Java 8 only !!!
- [net.cgrand/xforms "0.13.0"] ; Additional transducers
+ [jdistlib "0.5.1" ; Distribution statistic tests
+ :exclusions [com.github.wendykierp/JTransforms]]
+ [net.cgrand/xforms "0.13.0" ; Additional transducers
+ :exclusions [org.clojure/clojurescript]]
[net.sf.cssbox/cssbox "4.12" ; HTML / CSS rendering
:exclusions [org.slf4j/slf4j-api]]
[com.clearspring.analytics/stream "2.9.5" ; Various sketching algorithms
@@ -163,8 +166,7 @@
:jvm-opts ["-Dclojure.compiler.elide-meta=[:doc :added :file :line]" ; strip out metadata for faster load / smaller uberjar size
"-Dmanifold.disable-jvm8-primitives=true"]} ; disable Manifold Java 8 primitives (see https://github.com/ztellman/manifold#java-8-extensions)
;; generate sample dataset with `lein generate-sample-dataset`
- :generate-sample-dataset {:dependencies [[faker "0.2.2"] ; Fake data generator -- port of Perl/Ruby library
- [incanter/incanter-core "1.9.1"]] ; Satistical functions like normal distibutions}})
+ :generate-sample-dataset {:dependencies [[faker "0.2.2"]] ; Fake data generator -- port of Perl/Ruby library
:source-paths ["sample_dataset"]
:main ^:skip-aot metabase.sample-dataset.generate}
;; Profile Metabase start time with `lein profile`
diff --git a/resources/sample-dataset.db.mv.db b/resources/sample-dataset.db.mv.db
index 55eebd4a85a8f105d3f9312016676ad1eff6780b..5ca82600bfcfb98e5958a360d2bdd9771c49009d 100644
Binary files a/resources/sample-dataset.db.mv.db and b/resources/sample-dataset.db.mv.db differ
diff --git a/sample_dataset/metabase/sample_dataset/generate.clj b/sample_dataset/metabase/sample_dataset/generate.clj
index d7cfe9cbe996de64d622ff15a3bbce814999c292..d51dd122abcb8ec9472e1af37607a1f90efc8246 100644
--- a/sample_dataset/metabase/sample_dataset/generate.clj
+++ b/sample_dataset/metabase/sample_dataset/generate.clj
@@ -6,13 +6,14 @@
[jdbc :as jdbc]]
[clojure.math.numeric-tower :as math]
[clojure.string :as s]
+ [jdistlib.core :as dist]
[faker
[address :as address]
[company :as company]
[internet :as internet]
[lorem :as lorem]
[name :as name]]
- [incanter.distributions :as dist]
+ [medley.core :as m]
[metabase.db.spec :as dbspec]
[metabase.util :as u])
(:import java.util.Date))
@@ -21,7 +22,7 @@
(str (System/getProperty "user.dir") "/resources/sample-dataset.db"))
(defn- normal-distribution-rand [mean median]
- (dist/draw (dist/normal-distribution mean median)))
+ (dist/sample (dist/normal mean median)))
(defn- normal-distribution-rand-int [mean median]
(math/round (normal-distribution-rand mean median)))
@@ -60,18 +61,31 @@
:latitude (random-latitude)
:longitude (random-longitude)
:source (rand-nth ["Google" "Twitter" "Facebook" "Organic" "Affiliate"])
- :created_at (random-date-between (u/relative-date :year -1) (u/relative-date :year 1))}))
+ :created_at (random-date-between (u/relative-date :year -2) (u/relative-date :year 1))}))
;;; ## PRODUCTS
(defn- random-company-name []
(first (company/names)))
+(defn- rejection-sample
+ "Sample from distribution `dist` until `pred` is truthy for the sampled value.
+ https://en.wikipedia.org/wiki/Rejection_sampling"
+ [pred dist]
+ (let [x (dist/sample dist)]
+ (if (pred x)
+ x
+ (rejection-sample pred dist))))
+
(defn- random-price [min max]
- (let [range (- max min)]
- (-> (rand-int (* range 100))
- (/ 100.0)
- (+ min))))
+ (let [range (- max min)
+ mean1 (+ min (* range 0.3))
+ mean2 (+ min (* range 0.75))
+ variance (/ range 8)]
+ ; Sample from a multi modal distribution (mix of two normal distributions
+ ; with means `mean1` and `mean2` and variance `variance`).
+ (rejection-sample #(<= min % max) (rand-nth [(dist/normal mean1 variance)
+ (dist/normal mean2 variance)]))))
(def ^:private ^:const product-names
{:adjective '[Small, Ergonomic, Rustic, Intelligent, Gorgeous, Incredible, Fantastic, Practical, Sleek, Awesome, Enormous, Mediocre, Synergistic, Heavy-Duty, Lightweight, Aerodynamic, Durable]
@@ -110,7 +124,7 @@
:category (rand-nth ["Widget" "Gizmo" "Gadget" "Doohickey"])
:vendor (random-company-name)
:price (random-price 12 100)
- :created_at (random-date-between (u/relative-date :year -1) (u/relative-date :year 1))})
+ :created_at (random-date-between (u/relative-date :year -2) (u/relative-date :year 1))})
;;; ## ORDERS
@@ -194,20 +208,34 @@
(doto (Date.)
(.setTime (apply min (map #(.getTime ^Date %) dates)))))
+(defn- with-probability
+ "Return `(f)` with probability `p`, else return nil."
+ [p f]
+ (when (> p (rand))
+ (f)))
+
(defn random-order [{:keys [state], :as ^Person person} {:keys [price], :as product}]
{:pre [(string? state)
(number? price)]
:post [(map? %)]}
- (let [tax-rate (state->tax-rate state)
- _ (assert tax-rate
- (format "No tax rate found for state '%s'." state))
- tax (u/round-to-decimals 2 (* price tax-rate))]
+ (let [tax-rate (state->tax-rate state)
+ _ (assert tax-rate
+ (format "No tax rate found for state '%s'." state))
+ created-at (random-date-between (min-date (:created_at person)
+ (:created_at product))
+ (u/relative-date :year 2))
+ price (if (> (.getTime created-at) (.getTime (Date. 118 0 1)))
+ (* 1.5 price)
+ price)
+ tax (u/round-to-decimals 2 (* price tax-rate))]
{:user_id (:id person)
:product_id (:id product)
:subtotal price
:tax tax
+ :quantity (random-price 1 5)
+ :discount (with-probability 0.1 #(random-price 0 10))
:total (+ price tax)
- :created_at (random-date-between (min-date (:created_at person) (:created_at product)) (u/relative-date :year 1))}))
+ :created_at created-at}))
;;; ## REVIEWS
@@ -221,7 +249,7 @@
4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
5 5 5 5 5 5 5 5 5 5 5 5 5])
:body (first (lorem/paragraphs))
- :created_at (random-date-between (:created_at product) (u/relative-date :year 1))})
+ :created_at (random-date-between (:created_at product) (u/relative-date :year 2))})
(defn- create-randoms [n f]
(vec (map-indexed (fn [id obj]
@@ -248,6 +276,58 @@
person
(assoc person :orders (vec (repeatedly num-orders #(random-order person (rand-nth products))))))))
+(defn- add-autocorrelation
+ "Add autocorrelation with lag `lag` to field `k` by adding the value from `lag`
+ steps back (and dividing by 2 to retain roughly the same value range).
+ https://en.wikipedia.org/wiki/Autocorrelation"
+ ([k xs] (add-autocorrelation 1 k xs))
+ ([lag k xs]
+ (map (fn [prev next]
+ (update next k #(/ (+ % (k prev)) 2)))
+ xs
+ (drop lag xs))))
+
+(defn- add-increasing-variance
+ "Gradually increase variance of field `k` by scaling it an (on average)
+ increasingly larger random noise.
+ https://en.wikipedia.org/wiki/Variance"
+ [k xs]
+ (let [n (count xs)]
+ (map-indexed (fn [i x]
+ ; Limit the noise to [0.1, 2.1].
+ (update x k * (+ 1 (* (/ i n) (- (* 2 (rand)) 0.9)))))
+ xs)))
+
+(defn- add-seasonality
+ "Add seasonal component to field `k`. Seasonal variation (a multiplicative
+ factor) is described with map `seasonality-map` indexed into by `season-fn`
+ (eg. month of year of field created_at)."
+ [season-fn k seasonality-map xs]
+ (for [x xs]
+ (update x k * (seasonality-map (season-fn x)))))
+
+(defn- add-outliers
+ "Add `n` outliers (times `scale` value spikes) to field `k`. `n` can be either
+ percentage or count, determined by `mode`."
+ ([mode n k xs] (add-outliers mode n 10 k xs))
+ ([mode n scale k xs]
+ (if (= mode :share)
+ (for [x xs]
+ (if (< (rand) n)
+ (update x k * scale)
+ x))
+ (let [candidate-idxs (keep (fn [[idx x]]
+ (when (k x)
+ idx))
+ (m/indexed xs))
+ ; Note: since we are sampling with replacement there is a small chance
+ ; single index gets chosen multiple times.
+ outlier-idx? (set (repeatedly n #(rand-nth candidate-idxs)))]
+ (for [[idx x] (m/indexed xs)]
+ (if (outlier-idx? idx)
+ (update x k * scale)
+ x))))))
+
(defn create-random-data [& {:keys [people products]
:or {people 2500 products 200}}]
{:post [(map? %)
@@ -261,7 +341,26 @@
{:people (mapv #(dissoc % :orders) people)
:products (mapv #(dissoc % :reviews) products)
:reviews (vec (mapcat :reviews products))
- :orders (vec (mapcat :orders people))}))
+ :orders (->> people
+ (mapcat :orders)
+ (add-autocorrelation :quantity)
+ (add-outliers :share 0.01 :quantity)
+ (add-outliers :count 5 :discount)
+ (add-increasing-variance :total)
+ (add-seasonality #(.getMonth ^java.util.Date (:created_at %))
+ :quantity {0 0.6
+ 1 0.5
+ 2 0.3
+ 3 0.9
+ 4 1.3
+ 5 1.9
+ 6 1.5
+ 7 2.1
+ 8 1.5
+ 9 1.7
+ 10 0.9
+ 11 0.6})
+ vec)}))
;;; # LOADING THE DATA
@@ -274,7 +373,7 @@
(format "CREATE TABLE \"%s\" (\"ID\" BIGINT AUTO_INCREMENT, %s, PRIMARY KEY (\"ID\"));"
(s/upper-case (name table-name))
(apply str (->> (for [[field type] (seq field->type)]
- (format "\"%s\" %s NOT NULL" (s/upper-case (name field)) type))
+ (format "\"%s\" %s" (s/upper-case (name field)) type))
(interpose ", ")))))
(def ^:private ^:const tables
@@ -302,7 +401,9 @@
:subtotal "FLOAT"
:tax "FLOAT"
:total "FLOAT"
- :created_at "DATETIME"}
+ :discount "FLOAT"
+ :created_at "DATETIME"
+ :quantity "INTEGER"}
:reviews {:product_id "INTEGER"
:reviewer "VARCHAR(255)"
:rating "SMALLINT"
@@ -330,7 +431,9 @@
"on some products are not included here, but instead are accounted for in the subtotal.")}
:total {:description "The total billed amount."}
:user_id {:description (str "The id of the user who made this order. Note that in some cases where an order was created on behalf "
- "of a customer who phoned the order in, this might be the employee who handled the request.")}}}
+ "of a customer who phoned the order in, this might be the employee who handled the request.")}
+ :quantity {:description "Number of products bought."}
+ :discount {:description "Discount amount."}}}
:people {:description "This is a user account. Note that employees and customer support staff will have accounts."
:columns {:address {:description "The street address of the account’s billing address"}
:birth_date {:description "The date of birth of the user"}
diff --git a/src/metabase/feature_extraction/comparison.clj b/src/metabase/feature_extraction/comparison.clj
index b44b3479acaacfaad8ac9a9897736bdcc8aa3779..b40e3b2b3d17efd7040c0a63ca344dd8cf9d0db6 100644
--- a/src/metabase/feature_extraction/comparison.clj
+++ b/src/metabase/feature_extraction/comparison.clj
@@ -1,48 +1,16 @@
(ns metabase.feature-extraction.comparison
"Feature vector similarity comparison."
(:require [bigml.histogram.core :as h.impl]
+ [clojure.math.numeric-tower :as num]
[clojure.set :as set]
- [kixi.stats
- [core :as stats]
- [math :as math]]
+ [kixi.stats.core :as stats]
[metabase.feature-extraction
[feature-extractors :as fe]
- [histogram :as h]]
+ [histogram :as h]
+ [math :as math]]
[redux.core :as redux])
(:import com.bigml.histogram.Histogram))
-(def magnitude
- "Transducer that claclulates magnitude (Euclidean norm) of given vector.
- https://en.wikipedia.org/wiki/Euclidean_distance"
- (redux/post-complete (redux/pre-step + math/sq) math/sqrt))
-
-(defn cosine-distance
- "Cosine distance between vectors `a` and `b`.
- https://en.wikipedia.org/wiki/Cosine_similarity"
- [a b]
- (transduce identity
- (redux/post-complete
- (redux/fuse {:magnitude-a (redux/pre-step magnitude first)
- :magnitude-b (redux/pre-step magnitude second)
- :product (redux/pre-step + (partial apply *))})
- (fn [{:keys [magnitude-a magnitude-b product]}]
- (some->> (fe/safe-divide product magnitude-a magnitude-b)
- (- 1))))
- (map (comp (partial map double) vector) a b)))
-
-(defn head-tails-breaks
- "Pick out the cluster of N largest elements.
- https://en.wikipedia.org/wiki/Head/tail_Breaks"
- ([keyfn xs] (head-tails-breaks 0.6 keyfn xs))
- ([threshold keyfn xs]
- (let [mean (transduce (map keyfn) stats/mean xs)
- head (filter (comp (partial < mean) keyfn) xs)]
- (cond
- (empty? head) xs
- (>= threshold (/ (count head)
- (count xs))) (recur threshold keyfn head)
- :else head))))
-
(defmulti
^{:doc "Difference between two features.
Confined to [0, 1] with 0 being same, and 1 orthogonal."
@@ -56,8 +24,8 @@
(zero? (max a b)) 1
:else (let [a (double a)
b (double b)]
- (/ (math/abs (- a b))
- 2 (max (math/abs a) (math/abs b)))))})
+ (/ (num/abs (- a b))
+ 2 (max (num/abs a) (num/abs b)))))})
(defmethod difference [Boolean Boolean]
[a b]
@@ -94,8 +62,8 @@
(map vector a b))]
{:correlation corr
:covariance cov
- :significant? (some-> corr math/abs (> 0.3))
- :difference (or (cosine-distance a b) 0.5)
+ :significant? (some-> corr num/abs (> 0.3))
+ :difference (or (math/cosine-distance a b) 0.5)
:deltas (map (fn [t a b]
[t (- a b)])
t a b)}))
@@ -112,35 +80,6 @@
[a b]
{:difference nil})
-(defn chi-squared-distance
- "Chi-squared distane between empirical probability distributions `p` and `q`.
- http://www.aip.de/groups/soe/local/numres/bookcpdf/c14-3.pdf"
- [p q]
- (/ (reduce + (map (fn [pi qi]
- (cond
- (zero? pi) qi
- (zero? qi) pi
- :else (/ (math/sq (- pi qi))
- (+ pi qi))))
- p q))
- 2))
-
-(def ^:private ^{:arglists '([pdf])} pdf->cdf
- (partial reductions +))
-
-(defn ks-test
- "Perform the Kolmogorov-Smirnov test.
- Takes two samples parametrized by size (`m`, `n`) and distribution (`p`, `q`)
- and returns true if the samples are statistically significantly different.
- Optionally takes an additional `significance-level` parameter.
- https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test"
- ([m p n q] (ks-test 0.95 m p n q))
- ([significance-level m p n q]
- (when-not (zero? (* m n))
- (let [D (apply max (map (comp math/abs -) (pdf->cdf p) (pdf->cdf q)))
- c (math/sqrt (* -0.5 (Math/log (/ significance-level 2))))]
- (> D (* c (math/sqrt (/ (+ m n) (* m n)))))))))
-
(defn- unify-categories
"Given two PMFs add missing categories and align them so they both cover the
same set of categories."
@@ -170,15 +109,15 @@
q (map second pdf-b)
m (h.impl/total-count a)
n (h.impl/total-count b)
- distance (chi-squared-distance p q)]
+ distance (math/chi-squared-distance p q)]
{:difference distance
- :significant? (and (ks-test m p n q)
+ :significant? (and (math/ks-test m p n q)
(> distance (chi-squared-critical-value (min m n))))
:top-contributors (when (h/categorical? a)
(->> (map (fn [[bin pi] [_ qi]]
- [bin (math/abs (- pi qi))])
+ [bin (num/abs (- pi qi))])
pdf-a pdf-b)
- (head-tails-breaks second)
+ (math/head-tails-breaks second)
(map first)))}))
(defn- flatten-map
@@ -212,12 +151,12 @@
(let [differences (pairwise-differences a b)]
{:distance (transduce (keep (comp :difference val))
(redux/post-complete
- magnitude
- #(/ % (math/sqrt (count differences))))
+ math/magnitude
+ #(/ % (num/sqrt (count differences))))
differences)
:components differences
:top-contributors (->> differences
(filter (comp :difference second))
- (head-tails-breaks (comp :difference second)))
+ (math/head-tails-breaks (comp :difference second)))
:thereshold interestingness-thershold
:significant? (some :significant? (vals differences))}))
diff --git a/src/metabase/feature_extraction/core.clj b/src/metabase/feature_extraction/core.clj
index 16f11cd98f7e22bb7cb7498544ed2f4e7ac05c6f..f40d8ad643505d1f62aadc7081bb4e80ace6629b 100644
--- a/src/metabase/feature_extraction/core.clj
+++ b/src/metabase/feature_extraction/core.clj
@@ -3,7 +3,6 @@
(:require [clojure
[walk :refer [postwalk]]
[string :as s]]
- [kixi.stats.math :as math]
[medley.core :as m]
[metabase.db.metadata-queries :as metadata]
[metabase.feature-extraction
@@ -11,6 +10,7 @@
[costs :as costs]
[descriptions :refer [add-descriptions]]
[feature-extractors :as fe]
+ [math :as math]
[values :as values]]
[metabase.models
[card :refer [Card]]
@@ -263,13 +263,13 @@
{:feature feature
:difference difference})))
(->> comparisons
- (comparison/head-tails-breaks (comp :distance val))
+ (math/head-tails-breaks (comp :distance val))
(mapcat (fn [[field {:keys [top-contributors distance]}]]
(for [[feature {:keys [difference]}] top-contributors]
{:feature feature
:field field
- :contribution (* (math/sqrt distance) difference)})))
- (comparison/head-tails-breaks :contribution))))
+ :contribution (* (Math/sqrt distance) difference)})))
+ (math/head-tails-breaks :contribution))))
(defn compare-features
"Compare feature vectors of two models."
diff --git a/src/metabase/feature_extraction/feature_extractors.clj b/src/metabase/feature_extraction/feature_extractors.clj
index 6dfab7850860d8fc02cd79c182fa7fa8e501bfd0..54a3965089fbcebeff5bcfee9f2f6486b4e8a96e 100644
--- a/src/metabase/feature_extraction/feature_extractors.clj
+++ b/src/metabase/feature_extraction/feature_extractors.clj
@@ -1,18 +1,14 @@
(ns metabase.feature-extraction.feature-extractors
"Feature extractors for various models."
(:require [bigml.histogram.core :as h.impl]
- [clj-time
- [coerce :as t.coerce]
- [core :as t]
- [periodic :as t.periodic]]
- [kixi.stats
- [core :as stats :refer [somef]]
- [math :as math]]
+ [kixi.stats.core :as stats :refer [somef]]
[medley.core :as m]
[metabase.feature-extraction
[costs :as costs]
[histogram :as h]
- [stl :as stl]
+ [insights :as insights]
+ [math :as math :refer [safe-divide]]
+ [timeseries :as ts]
[values :as values]]
[metabase.models.table :refer [Table]]
[metabase.query-processor.middleware.binning :as binning]
@@ -40,28 +36,10 @@
(let [k (groupfn x)]
(assoc! acc k (f (get acc k init) x)))))))
-(defn safe-divide
- "Like `clojure.core//`, but returns nil if denominator is 0."
- [x & denominators]
- (when (or (and (not-empty denominators) (not-any? zero? denominators))
- (and (not (zero? x)) (empty? denominators)))
- (apply / x denominators)))
-
-(defn growth
- "Relative difference between `x1` an `x2`."
- [x2 x1]
- (when (and x1 x2 (not (zero? x1)))
- (let [x2 (double x2)
- x1 (double x1)]
- (cond
- (every? neg? [x1 x2]) (growth (- x1) (- x2))
- (and (neg? x1) (pos? x2)) (- (growth x1 x2))
- :else (/ (* (if (neg? x1) -1 1) (- x2 x1)) x1)))))
-
(defn- merge-juxt
[& fns]
(fn [x]
- (apply merge ((apply juxt fns) x))))
+ (apply merge ((apply juxt (remove nil? fns)) x))))
(def ^:private ^:const ^Double cardinality-error 0.01)
@@ -74,11 +52,6 @@
(.offer acc x)
acc))
-(def linear-regression
- "Transducer that calculats (simple) linear regression."
- (redux/post-complete (stats/simple-linear-regression first second)
- (partial zipmap [:offset :slope])))
-
(defn- nice-bins
[histogram]
(cond
@@ -98,29 +71,11 @@
:strategy :num-bins})]
(h/equidistant-bins min-value max-value bin-width histogram))))))
-(defn- triangle-area
- "Return the area of triangle specified by vertices `[x1, y1]`, `[x2, y2]`, and
- `[x3, y3].`
- http://mathworld.wolfram.com/TriangleArea.html"
- [[x1 y1] [x2 y2] [x3 y3]]
- (* 0.5 (+ (* (- x2) y1)
- (* x3 y1)
- (* x1 y2)
- (* (- x3) y2)
- (* (- x1) y3)
- (* x2 y3))))
-
-(def ^:private centroid
- "Calculate the centroid of given points.
- https://en.wikipedia.org/wiki/Centroid"
- (partial transduce identity (redux/juxt ((map first) stats/mean)
- ((map second) stats/mean))))
-
(defn- largest-triangle
"Find the point in `points` that frorms the largest triangle with verteices
`a` and `b`."
[a b points]
- (apply max-key (partial triangle-area a b) points))
+ (apply max-key (partial math/triangle-area a b) points))
(defn largest-triangle-three-buckets
"Downsample series `series` to (approximately) `target-size` data points using
@@ -145,7 +100,7 @@
([points] (conj points tail))
([points [middle right]]
(conj points (largest-triangle (last points)
- (centroid right)
+ (math/centroid right)
middle))))
(conj (partition bucket-size body) [tail]))))))
@@ -193,7 +148,7 @@
:columns [(:name field) "SHARE"]
:cols [(dissoc field :remapped_from)
{:name "SHARE"
- :display_name "Share"
+ :display_name "Share [%]"
:description "Share of corresponding bin in the overall population."
:base_type :type/Float}]}))
@@ -269,26 +224,22 @@
(prefer-method comparison-vector Num Category)
(prefer-method comparison-vector [DateTime Num] [Any Num])
-(def ^:private percentiles (range 0 1 0.1))
-
(defn- histogram-extractor
[{:keys [histogram]}]
(let [nil-count (h/nil-count histogram)
total-count (h/total-count histogram)]
- (merge {:histogram histogram
- :nil% (/ nil-count (max total-count 1))
- :has-nils? (pos? nil-count)
- :count total-count
- :entropy (h/entropy histogram)}
- (when-not (h/categorical? histogram)
- {:percentiles (apply h.impl/percentiles histogram percentiles)}))))
+ {:histogram histogram
+ :nil% (safe-divide nil-count total-count)
+ :has-nils? (pos? nil-count)
+ :count total-count
+ :entropy (h/entropy histogram)}))
(defn- cardinality-extractor
[{:keys [cardinality histogram]}]
- (let [uniqueness (/ cardinality (max (h/total-count histogram) 1))]
+ (let [uniqueness (safe-divide cardinality (h/total-count histogram))]
{:uniqueness uniqueness
:cardinality cardinality
- :all-distinct? (>= uniqueness (- 1 cardinality-error))}))
+ :all-distinct? (some-> uniqueness (>= (- 1 cardinality-error)))}))
(defn- field-metadata-extractor
[field]
@@ -304,116 +255,77 @@
(redux/fuse
(merge
{:histogram h/histogram
- :cardinality cardinality}
+ :cardinality cardinality
+ :kurtosis stats/kurtosis
+ :skewness stats/skewness
+ :zeros ((filter (somef zero?)) stats/count)}
(when (costs/full-scan? max-cost)
- {:sum (redux/with-xform + (keep (somef double)))
- :sum-of-squares (redux/with-xform +
- (keep (somef (comp math/sq double))))})
- (when (costs/unbounded-computation? max-cost)
- {:kurtosis (redux/pre-step stats/kurtosis (somef double))
- :skewness (redux/pre-step stats/skewness (somef double))})
+ {:sum ((keep (somef double)) +)
+ :sum-of-squares ((keep (somef #(Math/pow % 2))) +)})
(when (isa? (:special_type field) :type/Category)
{:histogram-categorical h/histogram-categorical})))
(merge-juxt
histogram-extractor
cardinality-extractor
(field-metadata-extractor field)
- (fn [{:keys [histogram histogram-categorical kurtosis skewness sum
- sum-of-squares]}]
- (let [var (h.impl/variance histogram)
- sd (some-> var math/sqrt)
- min (h.impl/minimum histogram)
- max (h.impl/maximum histogram)
- mean (h.impl/mean histogram)
- median (h.impl/median histogram)
- range (some-> max (- min))]
- {:positive-definite? (some-> min (>= 0))
- :%>mean (some->> mean ((h.impl/cdf histogram)) (- 1))
- :var>sd? (some->> sd (> var))
- :0<=x<=1? (when min (<= 0 min max 1))
- :-1<=x<=1? (when min (<= -1 min max 1))
- :cv (some-> sd (safe-divide mean))
- :range-vs-sd (some->> sd (safe-divide range))
- :mean-median-spread (some->> range (safe-divide (- mean median)))
- :min-vs-max (some->> max (safe-divide min))
- :range range
- :min min
- :max max
- :mean mean
- :median median
- :var var
- :sd sd
- :kurtosis kurtosis
- :skewness skewness
- :sum sum
- :sum-of-squares sum-of-squares
- :histogram (or histogram-categorical histogram)})))))
+ (fn [{:keys [histogram histogram-categorical kurtosis skewness sum zeros
+ sum-of-squares] :as features}]
+ (let [var (h.impl/variance histogram)
+ sd (some-> var Math/sqrt)
+ min (h.impl/minimum histogram)
+ max (h.impl/maximum histogram)
+ mean (h.impl/mean histogram)
+ median (h.impl/median histogram)
+ spread (some-> max (- min))
+ {:keys [q1 q3]} (h/iqr histogram)]
+ {:positive-definite? (some-> min (>= 0))
+ :%>mean (some->> mean ((h.impl/cdf histogram)) (- 1))
+ :var>sd? (some->> sd (> var))
+ :0<=x<=1? (when min (<= 0 min max 1))
+ :-1<=x<=1? (when min (<= -1 min max 1))
+ :cv (some-> sd (safe-divide mean))
+ :range-vs-sd (some->> sd (safe-divide spread))
+ :mean-median-spread (some->> spread (safe-divide (- mean median)))
+ :min-vs-max (some->> max (safe-divide min))
+ :range spread
+ :min min
+ :max max
+ :mean mean
+ :median median
+ :q1 q1
+ :q3 q3
+ :var var
+ :sd sd
+ :kurtosis kurtosis
+ :skewness skewness
+ :sum sum
+ :sum-of-squares sum-of-squares
+ :zero% (safe-divide zeros (h/total-count histogram))
+ :percentiles (->> (range 0 1.1 0.1)
+ (apply h.impl/percentiles histogram))
+ :histogram-categorical histogram-categorical})))))
(defmethod comparison-vector Num
[features]
(select-keys features
- [:histogram :mean :median :min :max :sd :count :kurtosis
- :skewness :entropy :nil% :uniqueness :range :min-vs-max]))
+ [:histogram :mean :median :min :max :sd :count :kurtosis :zero%
+ :skewness :entropy :nil% :uniqueness :range :min-vs-max :q1
+ :q3]))
(defmethod x-ray Num
- [{:keys [field count] :as features}]
+ [{:keys [field histogram histogram-categorical] :as features}]
(-> features
- (update :histogram (partial histogram->dataset field))
+ (assoc :histogram (histogram->dataset field (or histogram-categorical
+ histogram)))
+ (assoc :insights ((merge-juxt insights/normal-range
+ insights/zeros
+ insights/nils
+ insights/multimodal
+ insights/outliers)
+ features))
(dissoc :has-nils? :var>sd? :0<=x<=1? :-1<=x<=1? :all-distinct?
- :positive-definite? :var>sd? :uniqueness :min-vs-max)))
-
-(def ^:private ^{:arglists '([t])} to-double
- "Coerce `DateTime` to `Double`."
- (comp double t.coerce/to-long))
-
-(def ^:private ^{:arglists '([t])} from-double
- "Coerce `Double` into a `DateTime`."
- (somef (comp t.coerce/from-long long)))
-
-(defn- fill-timeseries
- "Given a coll of `[DateTime, Num]` pairs evenly spaced `step` apart, fill
- missing points with 0."
- [resolution ts]
- (let [[step rounder] (case resolution
- :month [(t/months 1) t/month]
- :quarter [(t/months 3) t/month]
- :year [(t/years 1) t/year]
- :week [(t/weeks 1) t/day]
- :day [(t/days 1) t/day]
- :hour [(t/hours 1) t/day]
- :minute [(t/minutes 1) t/minute])
- ts (for [[x y] ts]
- [(-> x from-double (t/floor rounder)) y])
- ts-index (into {} ts)]
- (into []
- (comp (take-while (partial (complement t/before?) (-> ts last first)))
- (map (fn [t]
- [(to-double t) (ts-index t 0)])))
- (some-> ts
- ffirst
- (t.periodic/periodic-seq step)))))
-
-(defn- decompose-timeseries
- "Decompose given timeseries with expected periodicty `period` into trend,
- seasonal component, and reminder.
- `period` can be one of `:hour`, `:day`, `:day-of-week`, `:week`, `:quarter`,
- `:day-of-month`, `:minute` or `:month`."
- [period ts]
- (when-let [period (case period
- :hour 24
- :minute 60
- :day-of-week 7
- :day-of-month 30
- :month 12
- :week 52
- :quarter 4
- :day 365
- nil)]
- (when (>= (count ts) (* 2 period))
- (let [{:keys [trend seasonal residual xs]} (stl/decompose period ts)]
- {:trend (map vector xs trend)
- :seasonal (map vector xs seasonal)
- :residual (map vector xs residual)}))))
+ :positive-definite? :var>sd? :uniqueness :min-vs-max
+ :histogram-categorical)))
(defn- last-n-days
[n offset {:keys [breakout filter] :as query}]
@@ -435,13 +347,7 @@
(defn- rolling-window-growth
[window query]
- (growth (last-n-days window 0 query) (last-n-days window window query)))
-
-(defn roughly=
- "Is `x` èqual to `y` within precision `precision` (default 0.05)."
- ([x y] (roughly= x y 0.05))
- ([x y precision]
- (<= (* (- 1 precision) x) y (* (+ 1 precision) x))))
+ (math/growth (last-n-days window 0 query) (last-n-days window window query)))
(defn- infer-resolution
[query series]
@@ -453,8 +359,8 @@
h/histogram
(partition 2 1 series))
median-delta (h.impl/median deltas)]
- (when (roughly= median-delta (h.impl/minimum deltas) 0.1)
- (condp roughly= median-delta
+ (when (math/roughly= median-delta (h.impl/minimum deltas) 0.1)
+ (condp math/roughly= median-delta
(* 60 1000) :minute
(* 60 60 1000) :hour
(* 24 60 60 1000) :day
@@ -467,30 +373,84 @@
(defmethod feature-extractor [DateTime Num]
[{:keys [max-cost query]} field]
(redux/post-complete
- (redux/pre-step
- (redux/fuse {:linear-regression linear-regression
- :series conj})
- (fn [[^java.util.Date x y]]
- [(some-> x .getTime double) y]))
+ ((map (fn [[^java.util.Date x y]]
+ [(some-> x .getTime double) y]))
+ (redux/fuse {; y = a + b*x
+ :linear-regression (stats/simple-linear-regression
+ first second)
+ ; y = e^a * x^b
+ :power-law-regression (stats/simple-linear-regression
+ #(Math/log (first %))
+ #(Math/log (second %)))
+ ; y = a + b*ln(x)
+ :log-linear-regression (stats/simple-linear-regression
+ #(Math/log (first %)) second)
+ :series conj}))
(merge-juxt
(field-metadata-extractor field)
- (fn [{:keys [series linear-regression]}]
- (let [resolution (infer-resolution query series)
+ (fn [{:keys [series linear-regression power-law-regression
+ log-linear-regression] :as features}]
+ (let [; We add a small regularization penalty to more complex curves to
+ ; prevent technically correct but nonsense solutions.
+ lambda 0.1
+ regularize (fn [penalty]
+ (fn [ssr]
+ (if (Double/isNaN ssr)
+ Double/MAX_VALUE
+ (+ ssr (* lambda penalty)))))
+ best-fit (transduce
+ identity
+ (redux/post-complete
+ (redux/fuse
+ {:linear-regression
+ (let [[a b] linear-regression]
+ (redux/post-complete
+ (math/ssr (fn [x]
+ (+ a (* b x))))
+ (regularize 0)))
+ :power-law-regression
+ (let [[a b] power-law-regression]
+ (redux/post-complete
+ (math/ssr (fn [x]
+ (* (Math/exp a) (Math/pow x b))))
+ (regularize 2)))
+ :log-linear-regression
+ (let [[a b] log-linear-regression]
+ (redux/post-complete
+ (math/ssr (fn [x]
+ (+ a (* b (Math/log x)))))
+ (regularize 1)))})
+ (fn [fits]
+ (let [model (key (apply min-key val fits))]
+ {:model model
+ :params (features model)})))
+ series)
+ resolution (infer-resolution query series)
series (if resolution
- (fill-timeseries resolution series)
+ (ts/fill-timeseries resolution series)
series)]
(merge {:resolution resolution
:series series
- :linear-regression linear-regression
+ :linear-regression (zipmap [:offset :slope]
+ linear-regression)
+ :best-fit best-fit
:growth-series (when resolution
(->> series
(partition 2 1)
(map (fn [[[_ y1] [x y2]]]
- [x (or (growth y2 y1) 0)]))))
+ [x (or (math/growth y2 y1)
+ 0)]))))
:seasonal-decomposition
(when (and resolution
(costs/unbounded-computation? max-cost))
- (decompose-timeseries resolution series))}
+ (ts/decompose resolution series))
+ :autocorrelation
+ (math/autocorrelation {:max-lag (min (or (some-> resolution
+ ts/period-length
+ dec)
+ Long/MAX_VALUE)
+ (/ (count series) 2))}
+ (map second series))}
(when (and (costs/allow-joins? max-cost)
(:aggregation query))
{:YoY (rolling-window-growth 365 query)
@@ -501,7 +461,7 @@
(defmethod comparison-vector [DateTime Num]
[features]
(-> features
- (dissoc :resolution)
+ (dissoc :resolution :best-fit)
((get-method comparison-vector :default))))
(defn- unpack-linear-regression
@@ -519,28 +479,37 @@
[{:keys [field series] :as features}]
(let [x-field (first field)]
(-> features
- (update :series (partial series->dataset from-double field))
- (update :growth-series (partial series->dataset from-double
+ (update :series (partial series->dataset ts/from-double field))
+ (dissoc :series :autocorrelation :best-fit)
+ (assoc :insights ((merge-juxt insights/noisiness
+ insights/variation-trend
+ insights/autocorrelation
+ insights/seasonality
+ insights/structural-breaks
+ insights/stationary
+ insights/trend)
+ features))
+ (update :growth-series (partial series->dataset ts/from-double
[x-field
{:name "GROWTH"
- :display_name "Growth"
+ :display_name "Growth [%]"
:base_type :type/Float}]))
(update :linear-regression
- (partial unpack-linear-regression from-double x-field series))
+ (partial unpack-linear-regression ts/from-double x-field series))
(update-in [:seasonal-decomposition :trend]
- (partial series->dataset from-double
+ (partial series->dataset ts/from-double
[x-field
{:name "TREND"
:display_name "Growth trend"
:base_type :type/Float}]))
(update-in [:seasonal-decomposition :seasonal]
- (partial series->dataset from-double
+ (partial series->dataset ts/from-double
[x-field
{:name "SEASONAL"
:display_name "Seasonal component"
:base_type :type/Float}]))
(update-in [:seasonal-decomposition :residual]
- (partial series->dataset from-double
+ (partial series->dataset ts/from-double
[x-field
{:name "RESIDUAL"
:display_name "Decomposition residual"
@@ -562,49 +531,28 @@
(defmethod feature-extractor Text
[_ field]
(redux/post-complete
- (redux/fuse {:histogram (redux/pre-step
- h/histogram
- (somef (comp count u/jdbc-clob->str)))})
+ (redux/fuse {:histogram ((map (somef (comp count u/jdbc-clob->str)))
+ h/histogram)})
(merge-juxt
(field-metadata-extractor field)
histogram-extractor)))
-(defprotocol Quarter
- "Quarter-of-year functionality"
- (quarter [dt] "Return which quarter (1-4) given date-like object falls into."))
-
-(extend-type java.util.Date
- Quarter
- (quarter [dt]
- (-> dt .getMonth inc (* 0.33) Math/ceil long)))
-
-(extend-type org.joda.time.DateTime
- Quarter
- (quarter [dt]
- (-> dt t/month (* 0.33) Math/ceil long)))
-
(defmethod feature-extractor DateTime
[_ {:keys [base_type unit] :as field}]
(redux/post-complete
- (redux/fuse (merge
- {:histogram (redux/pre-step
- h/histogram
- (somef (memfn ^java.util.Date getTime)))
- :histogram-day (redux/pre-step
- h/histogram-categorical
- (somef (memfn ^java.util.Date getDay)))
- :histogram-month (redux/pre-step
- h/histogram-categorical
- #(when %
- (inc (.getMonth ^java.util.Date %))))
- :histogram-quarter (redux/pre-step
- h/histogram-categorical
- (somef quarter))}
- (when-not (or (isa? base_type :type/Date)
- (#{:day :month :year :quarter :week} unit))
- {:histogram-hour (redux/pre-step
- h/histogram-categorical
- (somef (memfn ^java.util.Date getHours)))})))
+ (redux/fuse
+ (merge
+ {:histogram ((map (somef (memfn ^java.util.Date getTime)))
+ h/histogram)
+ :histogram-day ((map (somef (memfn ^java.util.Date getDay)))
+ h/histogram-categorical)
+ :histogram-month ((map #(some->> ^java.util.Date % .getMonth inc))
+ h/histogram-categorical)
+ :histogram-quarter ((map (somef ts/quarter)) h/histogram-categorical)}
+ (when-not (or (isa? base_type :type/Date)
+ (#{:day :month :year :quarter :week} unit))
+ {:histogram-hour ((map (somef (memfn ^java.util.Date getHours)))
+ h/histogram-categorical)})))
(merge-juxt
histogram-extractor
(field-metadata-extractor field)
@@ -615,40 +563,38 @@
(defmethod x-ray DateTime
[{:keys [field earliest latest histogram] :as features}]
- (let [earliest (from-double earliest)
- latest (from-double latest)]
- (-> features
- (assoc :earliest earliest)
- (assoc :latest latest)
- (update :histogram (partial histogram->dataset from-double field))
- (update :percentiles (partial m/map-vals from-double))
- (update :histogram-hour (somef
- (partial histogram->dataset
- {:name "HOUR"
- :display_name "Hour of day"
- :base_type :type/Integer
- :special_type :type/Category})))
- (update :histogram-day (partial histogram->dataset
- {:name "DAY"
- :display_name "Day of week"
- :base_type :type/Integer
- :special_type :type/Category}))
- (update :histogram-month (fn [histogram]
- (when-not (h/empty? histogram)
- (->> histogram
- (histogram->dataset
- {:name "MONTH"
- :display_name "Month of year"
- :base_type :type/Integer
- :special_type :type/Category})))))
- (update :histogram-quarter (fn [histogram]
- (when-not (h/empty? histogram)
- (->> histogram
- (histogram->dataset
- {:name "QUARTER"
- :display_name "Quarter of year"
- :base_type :type/Integer
- :special_type :type/Category}))))))))
+ (-> features
+ (update :earliest ts/from-double)
+ (update :latest ts/from-double)
+ (update :histogram (partial histogram->dataset ts/from-double field))
+ (update :percentiles (partial m/map-vals ts/from-double))
+ (update :histogram-hour (somef
+ (partial histogram->dataset
+ {:name "HOUR"
+ :display_name "Hour of day"
+ :base_type :type/Integer
+ :special_type :type/Category})))
+ (update :histogram-day (partial histogram->dataset
+ {:name "DAY"
+ :display_name "Day of week"
+ :base_type :type/Integer
+ :special_type :type/Category}))
+ (update :histogram-month (fn [histogram]
+ (when-not (h/empty? histogram)
+ (->> histogram
+ (histogram->dataset
+ {:name "MONTH"
+ :display_name "Month of year"
+ :base_type :type/Integer
+ :special_type :type/Category})))))
+ (update :histogram-quarter (fn [histogram]
+ (when-not (h/empty? histogram)
+ (->> histogram
+ (histogram->dataset
+ {:name "QUARTER"
+ :display_name "Quarter of year"
+ :base_type :type/Integer
+ :special_type :type/Category})))))))
(defmethod feature-extractor Category
[_ field]
@@ -674,19 +620,19 @@
(for [[i field] (m/indexed cols)
:when (not (or (:remapped_to field)
(= :type/PK (:special_type field))))]
- [(:name field) (redux/pre-step (feature-extractor opts field)
- #(nth % i))])))
+ [(:name field) ((map #(nth % i))
+ (feature-extractor opts field))])))
rows))
(defmethod feature-extractor :default
[opts field]
(redux/post-complete
(redux/fuse {:total-count stats/count
- :nil-count (redux/with-xform stats/count (filter nil?))})
+ :nil-count ((filter nil?) stats/count)})
(merge-juxt
(field-metadata-extractor field)
(fn [{:keys [total-count nil-count]}]
{:count total-count
- :nil% (/ nil-count (max total-count 1))
+ :nil% (safe-divide nil-count total-count)
:has-nils? (pos? nil-count)
:type nil}))))
diff --git a/src/metabase/feature_extraction/histogram.clj b/src/metabase/feature_extraction/histogram.clj
index 8de7e62f9f89076661cc9c29aef5789685f2f028..1e702f70c579ed3768d18c11518d77414b1bd11a 100644
--- a/src/metabase/feature_extraction/histogram.clj
+++ b/src/metabase/feature_extraction/histogram.clj
@@ -47,15 +47,28 @@
(+ (impl/total-count histogram)
(nil-count histogram)))
+(defn iqr
+ "Return interquartile range for a given histogram.
+ https://en.wikipedia.org/wiki/Interquartile_range"
+ [^Histogram histogram]
+ {:pre [(not (categorical? histogram))]}
+ (when-not (empty? histogram)
+ (let [{q1 0.25 q3 0.75} (impl/percentiles histogram 0.25 0.75)]
+ {:iqr (- q3 q1)
+ :q1 q1
+ :q3 q3})))
+
(defn optimal-bin-width
"Determine optimal bin width (and consequently number of bins) for a given
histogram using Freedman-Diaconis rule.
https://en.wikipedia.org/wiki/Freedman%E2%80%93Diaconis_rule"
[^Histogram histogram]
{:pre [(not (categorical? histogram))]}
- (when-not (empty? histogram)
- (let [{first-q 0.25 third-q 0.75} (impl/percentiles histogram 0.25 0.75)]
- (* 2 (- third-q first-q) (math/pow (impl/total-count histogram) (/ -3))))))
+ (some-> histogram
+ iqr
+ :iqr
+ (* 2 (math/pow (impl/total-count histogram)
+ (/ -3)))))
(defn equidistant-bins
"Split histogram into `bin-width` wide bins. If `bin-width` is not given use
diff --git a/src/metabase/feature_extraction/insights.clj b/src/metabase/feature_extraction/insights.clj
new file mode 100644
index 0000000000000000000000000000000000000000..c3d3b8928100aa34f6cff958e262a731735e0d31
--- /dev/null
+++ b/src/metabase/feature_extraction/insights.clj
@@ -0,0 +1,224 @@
+(ns metabase.feature-extraction.insights
+ "Data insights -- morsels of prepackaged analysis."
+ (:require [bigml.histogram.core :as h.impl]
+ [clojure.math.numeric-tower :as num]
+ [jdistlib.core :as d]
+ [kixi.stats
+ [core :as stats]
+ [math :refer [sq]]]
+ [metabase.feature-extraction
+ [histogram :as h]
+ [math :as math]
+ [timeseries :as ts]]
+ [net.cgrand.xforms :as x]
+ [redux.core :as redux]))
+
+(defmacro ^:private definsight
+ [insight docs features & body]
+ `(defn ~insight ~docs
+ [{:keys ~features}]
+ (when-let [insight# (do ~@body)]
+ {~(keyword insight) insight#})))
+
+(definsight normal-range
+ "What is the normal (expected) range for this data?
+ We define normal as being within interquartile range.
+ https://en.wikipedia.org/wiki/Interquartile_range"
+ [q1 q3]
+ (when q1
+ {:lower q1
+ :upper q3}))
+
+(definsight nils
+ "Are there any nils in the data?"
+ [nil% field count]
+ (when (some-> nil% pos?)
+ {:quality (if (< nil% (/ (Math/log (inc count))))
+ :some
+ :many)
+ :filter [:IS_NULL [:field-id (:id field)]]}))
+
+(definsight zeros
+ "Are there any 0s in the data?"
+ [zero% field count]
+ (when (some-> zero% pos?)
+ {:quality (if (< zero% (/ (Math/log (inc count))))
+ :some
+ :many)
+ :filter [:= [:field-id (:id field)] 0]}))
+
+(definsight autocorrelation
+ "Is there a significant autocorrelation at lag up to period length?
+ https://en.wikipedia.org/wiki/Autocorrelation"
+ [autocorrelation]
+ (let [{:keys [autocorrelation lag]} autocorrelation]
+ (when (some-> autocorrelation (> 0.3))
+ {:quality (if (< autocorrelation 0.6)
+ :weak
+ :strong)
+ :lag lag})))
+
+(definsight noisiness
+ "Is the data is noisy?
+ We determine noisiness by the relatve number of saddles in the series."
+ [series resolution]
+ (let [saddles% (/ (math/saddles series) (max (count series) 1))]
+ (when (> saddles% 0.1)
+ {:quality (if (> saddles% 0.3)
+ :very
+ :slightly)
+ :recommended-resolution (ts/higher-resolution resolution)})))
+
+(definsight variation-trend
+ "Is there a consistent thrend in changes of variation from one period to the
+ next?
+
+ We determine the trend by calculating relative variance for each period window
+ and fit a linear regression to the resulting sequence of variances. We then
+ test the hypothesis that the slope of this regression is not significantly
+ different from 0.
+ https://en.wikipedia.org/wiki/Simple_linear_regression
+ https://msu.edu/course/msc/317/slr-reg.htm
+ https://en.wikipedia.org/wiki/Variance"
+ [resolution series]
+ (when resolution
+ (transduce
+ (comp (x/partition (ts/period-length resolution)
+ (comp (map second)
+ (x/reduce h/histogram)))
+ (map-indexed (fn [idx histogram]
+ [(double idx) (/ (h.impl/variance histogram)
+ (h.impl/mean histogram))])))
+ (redux/post-complete
+ (redux/juxt (stats/sum-squares first second)
+ (redux/fuse {:s-x ((map first) +)
+ :s-xx ((map (comp sq first)) +)
+ :s-y ((map second) +)
+ :s-yy ((map (comp sq second)) +)}))
+ (fn [[{:keys [ss-xy ss-x n]} {:keys [s-x s-xx s-y s-yy]}]]
+ (when (and (> n 2) (not-any? zero? [ss-x s-x]))
+ (let [slope (/ ss-xy ss-x)
+ slope-error (/ (- (* n s-yy)
+ (sq s-y)
+ (* (sq slope)
+ (- (* n s-xx) (sq s-x))))
+ (- n 2)
+ (- (* n s-xx) (sq s-x)))]
+ (when (and (not= slope 0.0)
+ (math/significant? (/ slope slope-error)
+ (d/t-distribution (- n 2))
+ (/ 0.05 2)))
+ {:mode (if (pos? slope)
+ :increasing
+ :decreasing)})))))
+ series)))
+
+(definsight seasonality
+ "Is there a seasonal component to the changes in data?
+
+ Presence and strength of seasonal component is determined based on comparison
+ with residuals from the STL decomposition.
+ https://www.wessa.net/download/stl.pdf"
+ [seasonal-decomposition]
+ (when seasonal-decomposition
+ (let [diff (transduce identity stats/mean
+ (map (fn [[_ seasonal] [_ residual]]
+ (math/growth (num/abs seasonal)
+ (num/abs residual)))
+ (:seasonal seasonal-decomposition)
+ (:residual seasonal-decomposition)))]
+ (when (pos? diff)
+ {:quality (if (< diff 1)
+ :weak
+ :strong)}))))
+
+(definsight multimodal
+ "Is the data multimodal?
+ https://en.wikipedia.org/wiki/Multimodal_distribution
+ http://www.nicprice.net/diptest/Hartigan_1985_AnnalStat.pdf"
+ [histogram]
+ (-> histogram
+ (h.impl/sample 1000)
+ d/dip-test
+ second
+ (< 0.05)))
+
+(definsight structural-breaks
+ "Are there any structural breaks in the data?
+ https://en.wikipedia.org/wiki/Structural_break"
+ [resolution series]
+ (when resolution
+ (some->> series
+ (ts/breaks (ts/period-length resolution))
+ sort
+ (map ts/from-double)
+ not-empty
+ (hash-map :breaks))))
+
+(definsight outliers
+ "Find outliers using Tukey's fences (1.5*IQR heuristic).
+ https://en.wikipedia.org/wiki/Outlier
+ https://en.wikipedia.org/wiki/Interquartile_range"
+ [histogram field min max]
+ (let [{:keys [q1 q3 iqr]} (h/iqr histogram)
+ lower-bound (- q1 (* 1.5 iqr))
+ upper-bound (+ q3 (* 1.5 iqr))]
+ (when (or (< min lower-bound)
+ (> max upper-bound))
+ {:filter [:or [:> [:field-id (:id field)] upper-bound]
+ [:< [:field-id (:id field)] lower-bound]]})))
+
+(definsight stationary
+ "Is the data stationary.
+
+ We test for stationarity by sliding a winodw (window length is determined
+ based on `resolution`) across the time series and perform the Welch's t-test
+ on each adjacent pair of windows. If none of the pair-wise changes are
+ determined to be significant, the series is stationary.
+
+ Note: what we are doing is not entierly theoretical sound. We take population
+ size n to be equal to window length, neglecting the fact that our points are
+ already aggregations of populations of unknown size.
+ https://en.wikipedia.org/wiki/Welch%27s_t-test
+ https://en.wikipedia.org/wiki/Stationary_process"
+ [series resolution]
+ (when-let [n (ts/period-length resolution)]
+ (transduce (comp (x/partition n (comp (map second)
+ (x/reduce h/histogram)))
+ (map (fn [histogram]
+ {:mean (h.impl/mean histogram)
+ :var (h.impl/variance histogram)}))
+ (x/partition 2 1 (x/into [])))
+ (fn
+ ([] true)
+ ([acc] acc)
+ ([_ [{mean1 :mean var1 :var} {mean2 :mean var2 :var}]]
+ (if (= var1 var2 0.0)
+ true
+ (let [t (/ (- mean1 mean2)
+ (num/sqrt (/ (+ var1 var2) n)))
+ k (num/round (/ (sq (/ (+ var1 var2) n))
+ (/ (+ (sq var1)
+ (sq var2))
+ (* n (- n 1)))))]
+ (if (math/significant? t (d/t-distribution k) (/ 0.05 2))
+ (reduced false)
+ true)))))
+ series)))
+
+(definsight trend
+ "What is the best (simple) trend model?
+
+ We try fitting a linear, power law, and logarithmic models and pick the one
+ with the smallest sum of residual squares.
+ https://en.wikipedia.org/wiki/Residual_sum_of_squares
+ https://en.wikipedia.org/wiki/Simple_linear_regression
+ http://mathworld.wolfram.com/LeastSquaresFittingPowerLaw.html
+ http://mathworld.wolfram.com/LeastSquaresFittingLogarithmic.html"
+ [best-fit]
+ {:mode (if (-> best-fit :params second pos?)
+ :growing
+ :decreasing)
+ :shape ({:linear-regression :linearly
+ :power-law-regression :exponentally
+ :log-linear-regression :logarithmically} (:model best-fit))})
diff --git a/src/metabase/feature_extraction/math.clj b/src/metabase/feature_extraction/math.clj
new file mode 100644
index 0000000000000000000000000000000000000000..43cc7539d8e3e7cd90a30bd249465257b2990acf
--- /dev/null
+++ b/src/metabase/feature_extraction/math.clj
@@ -0,0 +1,173 @@
+(ns metabase.feature-extraction.math
+ "Math functions and utilities."
+ (:require [jdistlib.core :as d]
+ [kixi.stats
+ [core :as stats]
+ [math :as math]]
+ [metabase.feature-extraction.histogram :as h]
+ [redux.core :as redux]))
+
+(defn safe-divide
+ "Like `clojure.core//`, but returns nil if denominator is 0."
+ [x & denominators]
+ (when (or (and (not-empty denominators) (not-any? zero? denominators))
+ (and (not (zero? x)) (empty? denominators)))
+ (apply / x denominators)))
+
+(defn growth
+ "Relative difference between `x1` an `x2`."
+ [x2 x1]
+ (when (and x1 x2 (not (zero? x1)))
+ (let [x2 (double x2)
+ x1 (double x1)]
+ (cond
+ (every? neg? [x1 x2]) (growth (- x1) (- x2))
+ (and (neg? x1) (pos? x2)) (- (growth x1 x2))
+ (neg? x1) (- (growth x2 x1))
+ :else (/ (- x2 x1) x1)))))
+
+(defn saddles
+ "Returns the number of saddles in a given series."
+ [series]
+ (->> series
+ (partition 2 1)
+ (partition-by (fn [[[_ y1] [_ y2]]]
+ (>= y2 y1)))
+ rest
+ count))
+
+(defn roughly=
+ "Is `x` èqual to `y` within precision `precision` (default 0.05)."
+ ([x y] (roughly= x y 0.05))
+ ([x y precision]
+ (<= (* (- 1 precision) x) y (* (+ 1 precision) x))))
+
+(defn significant?
+ "Is `x` significant at `significance-level` if drawn from distribution
+ `distribution`."
+ ([x distribution] (significant? x distribution 0.95))
+ ([x distribution significance-level]
+ (> (math/abs x) (d/icdf distribution (- 1 significance-level)))))
+
+(defn autocorrelation
+ "Calculate autocorrelation at lag `lag` or find the lag with the highest
+ significant autocorrelation (if it exists) up to `max-lag` if `lag` is not
+ given.
+ https://en.wikipedia.org/wiki/Autocorrelation
+ http://sfb649.wiwi.hu-berlin.de/fedc_homepage/xplore/tutorials/xegbohtmlnode39.html"
+ ([xs] (autocorrelation {:max-lag (Math/floor (/ (count xs) 2))} xs))
+ ([{:keys [lag max-lag]} xs]
+ {:pre [(or lag max-lag)]}
+ (if lag
+ (transduce identity (stats/correlation first second)
+ (map vector xs (drop lag xs)))
+ (let [n (count xs)]
+ (reduce (fn [best lag]
+ (let [r (autocorrelation {:lag lag} xs)]
+ (if (and (some-> r
+ (* (math/sqrt (- n lag)))
+ (significant? (d/normal 0 1) (/ 0.05 2)))
+ (> (math/abs r) (math/abs (:autocorrelation best 0))))
+ {:autocorrelation r
+ :lag lag}
+ best)))
+ nil
+ (range 1 (inc max-lag)))))))
+
+(defn ssr
+ "Transducer that calculates residual sum of squares.
+ https://en.wikipedia.org/wiki/Residual_sum_of_squares"
+ [model]
+ ((map (fn [[x y]]
+ (math/sq (- y (model x)))))
+ +))
+
+(def magnitude
+ "Transducer that claclulates magnitude (Euclidean norm) of given vector.
+ https://en.wikipedia.org/wiki/Euclidean_distance"
+ (redux/post-complete ((map math/sq) +) math/sqrt))
+
+(defn cosine-distance
+ "Cosine distance between vectors `a` and `b`.
+ https://en.wikipedia.org/wiki/Cosine_similarity"
+ [a b]
+ (transduce identity
+ (redux/post-complete
+ (redux/fuse {:magnitude-a ((map first) magnitude)
+ :magnitude-b ((map second) magnitude)
+ :product ((map (partial apply *)) +)})
+ (fn [{:keys [magnitude-a magnitude-b product]}]
+ (some->> (safe-divide product magnitude-a magnitude-b) (- 1))))
+ (map (comp (partial map double) vector) a b)))
+
+(defn head-tails-breaks
+ "Pick out the cluster of N largest elements.
+ https://en.wikipedia.org/wiki/Head/tail_Breaks"
+ ([keyfn xs] (head-tails-breaks 0.6 keyfn xs))
+ ([threshold keyfn xs]
+ (let [mean (transduce (map keyfn) stats/mean xs)
+ head (filter (comp (partial < mean) keyfn) xs)]
+ (cond
+ (empty? head) xs
+ (>= threshold (/ (count head)
+ (count xs))) (recur threshold keyfn head)
+ :else head))))
+
+(defn chi-squared-distance
+ "Chi-squared distane between empirical probability distributions `p` and `q`.
+ http://www.aip.de/groups/soe/local/numres/bookcpdf/c14-3.pdf"
+ [p q]
+ (/ (reduce + (map (fn [pi qi]
+ (cond
+ (zero? pi) qi
+ (zero? qi) pi
+ :else (/ (math/sq (- pi qi))
+ (+ pi qi))))
+ p q))
+ 2))
+
+(def ^:private ^{:arglists '([pdf])} pdf->cdf
+ (partial reductions +))
+
+(defn ks-test
+ "Perform the Kolmogorov-Smirnov test.
+ Takes two samples parametrized by size (`m`, `n`) and distribution (`p`, `q`)
+ and returns true if the samples are statistically significantly different.
+ Optionally takes an additional `significance-level` parameter.
+ https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test"
+ ([m p n q] (ks-test 0.95 m p n q))
+ ([significance-level m p n q]
+ (when-not (zero? (* m n))
+ (let [D (apply max (map (comp math/abs -) (pdf->cdf p) (pdf->cdf q)))
+ c (math/sqrt (* -0.5 (Math/log (/ significance-level 2))))]
+ (> D (* c (math/sqrt (/ (+ m n) (* m n)))))))))
+
+(defn outliers
+ "Find outliers using Tukey's fences (1.5*IQR heuristic).
+ https://en.wikipedia.org/wiki/Outlier
+ https://en.wikipedia.org/wiki/Interquartile_range"
+ ([xs] (outliers identity xs))
+ ([keyfn xs]
+ (when (not-empty xs)
+ (let [{:keys [q1 q3 iqr]} (->> xs (transduce (map keyfn) h/histogram) h/iqr)
+ lower-bound (- q1 (* 1.5 iqr))
+ upper-bound (+ q3 (* 1.5 iqr))]
+ (remove (comp #(< lower-bound % upper-bound) keyfn) xs)))))
+
+(defn triangle-area
+ "Return the area of triangle specified by vertices `[x1, y1]`, `[x2, y2]`, and
+ `[x3, y3].`
+ http://mathworld.wolfram.com/TriangleArea.html"
+ [[x1 y1] [x2 y2] [x3 y3]]
+ (* 0.5 (+ (* (- x2) y1)
+ (* x3 y1)
+ (* x1 y2)
+ (* (- x3) y2)
+ (* (- x1) y3)
+ (* x2 y3))))
+
+(def centroid
+ "Calculate the centroid of given points.
+ https://en.wikipedia.org/wiki/Centroid"
+ (partial transduce identity (redux/juxt ((map first) stats/mean)
+ ((map second) stats/mean))))
diff --git a/src/metabase/feature_extraction/stl.clj b/src/metabase/feature_extraction/stl.clj
deleted file mode 100644
index 5c751a3b483dc20716395f2248c81fbfa6afd407..0000000000000000000000000000000000000000
--- a/src/metabase/feature_extraction/stl.clj
+++ /dev/null
@@ -1,46 +0,0 @@
-(ns metabase.feature-extraction.stl
- "Seasonal-Trend Decomposition"
- (:import (com.github.brandtg.stl StlDecomposition StlResult StlConfig)))
-
-(def ^:private setters
- {:inner-loop-passes (memfn ^StlConfig setNumberOfInnerLoopPasses n)
- :robustness-iterations (memfn ^StlConfig setNumberOfRobustnessIterations n)
- :trend-bandwidth (memfn ^StlConfig setTrendComponentBandwidth bw)
- :seasonal-bandwidth (memfn ^StlConfig setSeasonalComponentBandwidth bw)
- :loess-robustness-iterations (memfn ^StlConfig setLoessRobustnessIterations n)
- :periodic? (memfn ^StlConfig setPeriodic periodic?)})
-
-(defn decompose
- "Decompose time series into trend, seasonal component, and residual.
- https://www.wessa.net/download/stl.pdf"
- ([period ts]
- (decompose period {} ts))
- ([period opts ts]
- (let [xs (map first ts)
- ys (map second ts)
- preprocess (if-let [transform (:transform opts)]
- (partial map transform)
- identity)
- postprocess (if-let [transform (:reverse-transform opts)]
- (partial map transform)
- vec)]
- (transduce identity
- (let [^StlDecomposition decomposer (StlDecomposition. period)]
- (fn
- ([] (.getConfig decomposer))
- ([_]
- (let [^StlResult decomposition (.decompose
- decomposer
- (double-array xs)
- (double-array (preprocess ys)))]
- {:trend (postprocess (.getTrend decomposition))
- :seasonal (postprocess (.getSeasonal decomposition))
- :residual (postprocess (.getRemainder decomposition))
- :xs xs
- :ys ys}))
- ([^StlConfig config [k v]]
- (when-let [setter (setters k)]
- (setter config v))
- config)))
- (merge {:inner-loop-passes 100}
- opts)))))
diff --git a/src/metabase/feature_extraction/timeseries.clj b/src/metabase/feature_extraction/timeseries.clj
new file mode 100644
index 0000000000000000000000000000000000000000..87c0255f98458e553e26ef51012278f74b1b10f8
--- /dev/null
+++ b/src/metabase/feature_extraction/timeseries.clj
@@ -0,0 +1,179 @@
+(ns metabase.feature-extraction.timeseries
+ "Timeseries analysis and utilities."
+ (:require [bigml.histogram.core :as h.impl]
+ [clj-time
+ [coerce :as t.coerce]
+ [core :as t]
+ [periodic :as t.periodic]]
+ [kixi.stats
+ [core :refer [somef] :as stast]
+ [math :as k.math]]
+ [metabase.feature-extraction
+ [histogram :as h]
+ [math :as math]]
+ [redux.core :as redux])
+ (:import (com.github.brandtg.stl StlDecomposition StlResult StlConfig)))
+
+(def ^{:arglists '([t])} to-double
+ "Coerce `DateTime` to `Double`."
+ (comp double t.coerce/to-long))
+
+(def ^{:arglists '([t])} from-double
+ "Coerce `Double` into a `DateTime`."
+ (somef (comp t.coerce/from-long long)))
+
+(defn fill-timeseries
+ "Given a coll of `[DateTime, Num]` pairs evenly spaced `step` apart, fill
+ missing points with 0."
+ [resolution ts]
+ (let [[step rounder] (case resolution
+ :month [(t/months 1) t/month]
+ :quarter [(t/months 3) t/month]
+ :year [(t/years 1) t/year]
+ :week [(t/weeks 1) t/day]
+ :day [(t/days 1) t/day]
+ :hour [(t/hours 1) t/day]
+ :minute [(t/minutes 1) t/minute])
+ ts (for [[x y] ts]
+ [(-> x from-double (t/floor rounder)) y])
+ ts-index (into {} ts)]
+ (into []
+ (comp (take-while (partial (complement t/before?) (-> ts last first)))
+ (map (fn [t]
+ [(to-double t) (or (ts-index t) 0)])))
+ (some-> ts
+ ffirst
+ (t.periodic/periodic-seq step)))))
+
+(def period-length
+ "What is the period for a given time resolution."
+ {:hour 24
+ :minute 60
+ :month 12
+ :week 52
+ :quarter 4
+ :day 365})
+
+(def ^:private stl-setters
+ {:inner-loop-passes (memfn ^StlConfig setNumberOfInnerLoopPasses n)
+ :robustness-iterations (memfn ^StlConfig setNumberOfRobustnessIterations n)
+ :trend-bandwidth (memfn ^StlConfig setTrendComponentBandwidth bw)
+ :seasonal-bandwidth (memfn ^StlConfig setSeasonalComponentBandwidth bw)
+ :loess-robustness-iterations (memfn ^StlConfig setLoessRobustnessIterations n)
+ :periodic? (memfn ^StlConfig setPeriodic periodic?)})
+
+(defn decompose
+ "Decompose given timeseries with expected periodicty `period` into trend,
+ seasonal component, and residual.
+ `period` can be one of `:hour`, `:day`, `:week`, `:quarter` `:minute`,
+ `:month`, or `:year`.
+ https://www.wessa.net/download/stl.pdf"
+ ([period ts]
+ (decompose period {} ts))
+ ([period opts ts]
+ (when-let [period (period-length period)]
+ (when (>= (count ts) (* 2 period))
+ (let [xs (double-array (map first ts))
+ ys (double-array (map second ts))
+ ^StlDecomposition decomposer (StlDecomposition. period)
+ _ (reduce-kv
+ (fn [^StlConfig config k v]
+ (when-let [setter (stl-setters k)]
+ (setter config v))
+ config)
+ (.getConfig decomposer)
+ (merge {:inner-loop-passes 100}
+ opts))
+ ^StlResult decomposition (.decompose decomposer xs ys)]
+ {:trend (map vector xs (.getTrend decomposition))
+ :seasonal (map vector xs (.getSeasonal decomposition))
+ :residual (map vector xs (.getRemainder decomposition))})))))
+
+(def ^:private resolutions [:minute :hour :day :week :month :quarter :year])
+
+(defn lower-resolution
+ "Return on size bigger time resolution (eg. minute->hour, day->week, ...)."
+ [resolution]
+ (->> resolutions (take-while (complement #{resolution})) last))
+
+(defn higher-resolution
+ "Return on size smaller time resolution (eg. hour->minute, month->week, ...)."
+ [resolution]
+ (->> resolutions (drop-while (complement #{resolution})) second))
+
+(defprotocol Quarter
+ "Quarter-of-year functionality"
+ (quarter [dt] "Return which quarter (1-4) given date-like object falls into."))
+
+(extend-protocol Quarter
+ java.util.Date
+ (quarter [dt]
+ (-> dt .getMonth inc (* 0.33) Math/ceil long))
+
+ org.joda.time.DateTime
+ (quarter [dt]
+ (-> dt t/month (* 0.33) Math/ceil long)))
+
+(def ^:private quartiles
+ "Transducer that calculates 1st, 2nd, and 3rd quartile.
+ https://en.wikipedia.org/wiki/Quartile"
+ (redux/post-complete
+ h/histogram
+ #(-> % (h.impl/percentiles 0.25 0.5 0.75) vals)))
+
+(def ^:private ^{:arglists '([candidates])} most-likely-breaks
+ "Pick out true breaks from among break candidates by selecting the point with
+ the highest eta from each group of consecutive points."
+ (partial reduce (fn [[head & tail :as breaks] {:keys [idx x eta]}]
+ (if (some-> head :idx inc (= idx))
+ (concat [(if (> (:eta head) eta)
+ (update head :idx inc)
+ {:idx idx
+ :x x
+ :eta eta})]
+ tail)
+ (concat [{:idx idx
+ :x x
+ :eta eta}]
+ breaks)))
+ []))
+
+(defn breaks
+ "Find positions of structural breaks.
+
+ The idea is to slide a window of length 2w+1 across the time series (window
+ length is determined based on `resolution`). At each step we calculate the
+ eta statistic measuring the difference in distributions of the left [0, w+1]
+ and right [w+1, 2w+1] half-window centered at pivot. We normalize eta by the
+ range of values in the window to make it impervious to trend shifts in mean
+ and variance.
+ We then pick out all outlier etas. These are break candidates. However as we
+ are using a sliding window there will likely be several candidates for the
+ same break (even when the pivot is not perfectly positioned we still expect a
+ significant difference between left and right half-window). We select the point
+ with the highest eta among consecutive points (this also means we can only
+ detect breaks that are more than w apart).
+
+ https://en.wikipedia.org/wiki/Structural_break
+ http://journals.plos.org/plosone/article?id=10.1371/journal.pone.0059279#pone.0059279.s003"
+ [period series]
+ (let [half-period (-> period (/ 2) Math/floor int inc)]
+ (->> (map (fn [left right idx]
+ (let [pivot (ffirst right)
+ window (map second (concat left right))
+ range (- (apply max window) (apply min window))
+ ql (transduce (map second) quartiles left)
+ qr (transduce (map second) quartiles right)]
+ {:eta (if (zero? range)
+ 0
+ (/ (reduce + (map (comp k.math/sq -) ql qr))
+ 3 (k.math/sq range)))
+ :x pivot
+ :idx idx}))
+ ; We want pivot point to be in both halfs, hence the overlap.
+ (partition half-period 1 series)
+ (partition half-period 1 (drop (dec half-period) series))
+ (range))
+ (math/outliers :eta)
+ most-likely-breaks
+ (map :x))))
diff --git a/test/metabase/feature_extraction/async_test.clj b/test/metabase/feature_extraction/async_test.clj
index 0dde07f0fbe8627624fafe5b859220d47692b509..d5baca1cc8238bca36e7b018e2268c3f69324303 100644
--- a/test/metabase/feature_extraction/async_test.clj
+++ b/test/metabase/feature_extraction/async_test.clj
@@ -12,8 +12,7 @@
(expect
[true :canceled false]
(let [job-id (compute (gensym) #(loop [] (Thread/sleep 100) (recur)))
- r? (running? (ComputationJob job-id))]
- (Thread/sleep 100)
+ r? (running? (ComputationJob job-id))]
(cancel (ComputationJob job-id))
[r? (:status (ComputationJob job-id)) (running? (ComputationJob job-id))]))
diff --git a/test/metabase/feature_extraction/comparison_test.clj b/test/metabase/feature_extraction/comparison_test.clj
index 42bde9acf71d8f7f6f79a85900ae0f12c88cf0d6..48abaa2d305511e3a3e29d6020db5a529db1da3f 100644
--- a/test/metabase/feature_extraction/comparison_test.clj
+++ b/test/metabase/feature_extraction/comparison_test.clj
@@ -4,23 +4,6 @@
[comparison :refer :all :as c]
[histogram :as h]]))
-(expect
- (approximately 5.5 0.1)
- (transduce identity magnitude [1 2 3 4]))
-(expect
- 0.0
- (transduce identity magnitude []))
-
-(expect
- [1.0
- 0.5
- nil
- nil]
- [(cosine-distance [1 0 1] [0 1 0])
- (cosine-distance [1 0 1] [0 1 1])
- (cosine-distance [1 0 1] [0 0 0])
- (cosine-distance [] [])])
-
(expect
[0.25
0
@@ -50,13 +33,6 @@
(#'c/unify-categories {:a 0.5 :b 0.3 :c 0.2}
{:x 0.9 :y 0.1}))))
-(expect
- (approximately 0.39 0.1)
- (chi-squared-distance [0.1 0.2 0.7] [0.5 0.4 0.1]))
-(expect
- 0
- (chi-squared-distance [] []))
-
(expect
[{:foo 4 :bar 5}
{:foo 4 :bar_a 4 :bar_b_x 4 :bar_b_y 7}]
diff --git a/test/metabase/feature_extraction/feature_extractors_test.clj b/test/metabase/feature_extraction/feature_extractors_test.clj
index 76ad6dd9f0ae8e3b2692f0a09c6fece4a8828113..c52e7ea7d156ae1c8b0df1cf262f395833d26e71 100644
--- a/test/metabase/feature_extraction/feature_extractors_test.clj
+++ b/test/metabase/feature_extraction/feature_extractors_test.clj
@@ -3,41 +3,13 @@
[core :as t]
[coerce :as t.coerce]]
[expectations :refer :all]
+ [metabase.feature-extraction
+ [feature-extractors :refer :all :as fe]
+ [histogram :as h]
+ [timeseries :as ts]]
[medley.core :as m]
- [metabase.feature-extraction.feature-extractors :refer :all :as fe]
- [metabase.feature-extraction.histogram :as h]
[redux.core :as redux]))
-(expect
- [2
- (/ 4)
- nil
- nil]
- [(safe-divide 4 2)
- (safe-divide 4)
- (safe-divide 0)
- (safe-divide 4 0)])
-
-(expect
- [0.23
- 1.0
- -0.5
- -5.0
- 5.0
- 2.0
- nil
- nil
- nil]
- [(growth 123 100)
- (growth -0.1 -0.2)
- (growth -0.4 -0.2)
- (growth -0.4 0.1)
- (growth 0.1 -0.4)
- (growth Long/MAX_VALUE Long/MIN_VALUE)
- (growth 0.1 nil)
- (growth nil 0.5)
- (growth 0.5 0.0)])
-
(expect
[{:foo 2
:bar 10}
@@ -50,39 +22,6 @@
{:x :bar :y 2}])
(transduce identity (rollup (redux/pre-step + :y) :x) [])])
-(expect
- [1
- 1
- 2
- 4]
- [(#'fe/quarter (t/date-time 2017 1))
- (#'fe/quarter (t/date-time 2017 3))
- (#'fe/quarter (t/date-time 2017 5))
- (#'fe/quarter (t/date-time 2017 12))])
-
-(defn- make-timestamp
- [& args]
- (-> (apply t/date-time args)
- ((var fe/to-double))))
-
-(expect
- [[(make-timestamp 2016 1) 12]
- [(make-timestamp 2016 2) 0]
- [(make-timestamp 2016 3) 4]
- [(make-timestamp 2016 4) 0]
- [(make-timestamp 2016 5) 0]
- [(make-timestamp 2016 6) 0]
- [(make-timestamp 2016 7) 0]
- [(make-timestamp 2016 8) 0]
- [(make-timestamp 2016 9) 0]
- [(make-timestamp 2016 10) 0]
- [(make-timestamp 2016 11) 0]
- [(make-timestamp 2016 12) 0]
- [(make-timestamp 2017 1) 25]]
- (#'fe/fill-timeseries :month [[(make-timestamp 2016 1 12 4) 12]
- [(make-timestamp 2016 3 2 2) 4]
- [(make-timestamp 2017 1) 25]]))
-
(expect
[2
0]
@@ -95,15 +34,14 @@
(fn [m] {:bar (count m)})
(fn [_] {:baz 1})) {}))
-(def ^:private hist (transduce identity h/histogram (concat (range 50)
- (range 200 250))))
-
(expect
[["TEST" "SHARE"]
3
true
[[17.0 1.0]]]
- (let [dataset (#'fe/histogram->dataset {:name "TEST"} hist)]
+ (let [hist (transduce identity h/histogram (concat (range 50)
+ (range 200 250)))
+ dataset (#'fe/histogram->dataset {:name "TEST"} hist)]
[(:columns dataset)
(count (:rows dataset))
(->> (transduce identity h/histogram [])
@@ -115,10 +53,10 @@
:rows
vec)]))
-(expect
- [true false]
- [(roughly= 30 30.5 0.05)
- (roughly= 130 30.5 0.05)])
+(defn- make-timestamp
+ [& args]
+ (-> (apply t/date-time args)
+ ts/to-double))
(expect
[:day
@@ -144,7 +82,7 @@
(-> (apply t/date-time args)
t.coerce/to-sql-time))
-(def ^:private numbers [0.1 0.4 0.2 nil 0.5 0.3 0.51 0.55 0.22])
+(def ^:private numbers [0.1 0.4 0.2 nil 0.5 0.3 0.51 0.55 0.22 0.0])
(def ^:private ints [0 nil Long/MAX_VALUE Long/MIN_VALUE 5 -100])
(def ^:private datetimes [(make-sql-timestamp 2015 6 1)
nil
@@ -183,15 +121,13 @@
:type)])
(expect
- [0 1 3 0]
- [(#'fe/saddles [[1 1] [2 2] [3 3]])
- (#'fe/saddles [[1 1] [2 2] [3 -2]])
- (#'fe/saddles [[1 1] [2 2] [3 -2] [4 5] [5 2]])
- (#'fe/saddles nil)])
-
-(expect
- 8.0
- (#'fe/triangle-area [-2 0] [2 0] [0 4]))
+ [:some
+ :some
+ 0.5025]
+ (let [x-ray (-> (->features {:base_type :type/Number} numbers) x-ray :insights)]
+ [(-> x-ray :zeros :quality)
+ (-> x-ray :nils :quality)
+ (-> x-ray :normal-range :upper)]))
(expect
[(var-get #'fe/datapoint-target-smooth)
diff --git a/test/metabase/feature_extraction/insights_test.clj b/test/metabase/feature_extraction/insights_test.clj
new file mode 100644
index 0000000000000000000000000000000000000000..3934b4fb123ba50736df0c59593e6b579d83c459
--- /dev/null
+++ b/test/metabase/feature_extraction/insights_test.clj
@@ -0,0 +1,33 @@
+(ns metabase.feature-extraction.insights-test
+ (:require [expectations :refer :all]
+ [medley.core :as m]
+ [metabase.feature-extraction.insights :refer :all]))
+
+(expect
+ [true
+ nil]
+ (map :stationary [(stationary {:series (m/indexed (repeat 100 1))
+ :resolution :month})
+ (stationary {:series (m/indexed (range 100))
+ :resolution :month})]))
+
+(expect
+ [{:mode :increasing}
+ {:mode :decreasing}
+ nil]
+ (let [n 100]
+ (map :variation-trend
+ [(variation-trend {:series (map-indexed
+ (fn [i x]
+ [i (* x (+ 1 (* (/ i n)
+ (- (* 2 (rand)) 0.9))))])
+ (repeat n 2))
+ :resolution :month})
+ (variation-trend {:series (map-indexed
+ (fn [i x]
+ [i (* x (+ 1 (* (/ (- n i) n)
+ (- (* 2 (rand)) 0.9))))])
+ (repeat n 2))
+ :resolution :month})
+ (variation-trend {:series (m/indexed (repeat n 2))
+ :resolution :day})])))
diff --git a/test/metabase/feature_extraction/math_test.clj b/test/metabase/feature_extraction/math_test.clj
new file mode 100644
index 0000000000000000000000000000000000000000..d735064774c3b76c3ddbf05d4087716fd041fae6
--- /dev/null
+++ b/test/metabase/feature_extraction/math_test.clj
@@ -0,0 +1,102 @@
+(ns metabase.feature-extraction.math-test
+ (:require [expectations :refer :all]
+ [metabase.feature-extraction.math :refer :all]))
+
+(expect
+ (approximately 5.5 0.1)
+ (transduce identity magnitude [1 2 3 4]))
+(expect
+ 0.0
+ (transduce identity magnitude []))
+
+(expect
+ [1.0
+ 0.5
+ nil
+ nil]
+ [(cosine-distance [1 0 1] [0 1 0])
+ (cosine-distance [1 0 1] [0 1 1])
+ (cosine-distance [1 0 1] [0 0 0])
+ (cosine-distance [] [])])
+
+(expect
+ (approximately 0.39 0.1)
+ (chi-squared-distance [0.1 0.2 0.7] [0.5 0.4 0.1]))
+(expect
+ 0
+ (chi-squared-distance [] []))
+
+(expect
+ [2
+ (/ 4)
+ nil
+ nil]
+ [(safe-divide 4 2)
+ (safe-divide 4)
+ (safe-divide 0)
+ (safe-divide 4 0)])
+
+(expect
+ [0.23
+ 1.0
+ -0.5
+ -5.0
+ 5.0
+ 2.0
+ nil
+ nil
+ nil]
+ [(growth 123 100)
+ (growth -0.1 -0.2)
+ (growth -0.4 -0.2)
+ (growth -0.4 0.1)
+ (growth 0.1 -0.4)
+ (growth Long/MAX_VALUE Long/MIN_VALUE)
+ (growth 0.1 nil)
+ (growth nil 0.5)
+ (growth 0.5 0.0)])
+
+(expect
+ [true false]
+ [(roughly= 30 30.5 0.05)
+ (roughly= 130 30.5 0.05)])
+
+(expect
+ [0 1 3 0]
+ [(saddles [[1 1] [2 2] [3 3]])
+ (saddles [[1 1] [2 2] [3 -2]])
+ (saddles [[1 1] [2 2] [3 -2] [4 5] [5 2]])
+ (saddles nil)])
+
+(expect
+ [{:autocorrelation 1.0
+ :lag 1}
+ {:autocorrelation -1.0
+ :lag 1}
+ nil nil nil nil]
+ [(autocorrelation (range 10))
+ (autocorrelation [1 -1 1 -1 1 -1])
+ (autocorrelation [1 2 3]) ; not significant
+ (autocorrelation [1])
+ (autocorrelation [])
+ (autocorrelation nil)])
+
+(expect
+ [nil
+ #{50 100 35}
+ nil]
+ (let [xs (vec (repeatedly 100 rand))]
+ [(not-empty (outliers xs))
+ (set (outliers (-> xs
+ (assoc-in [10] 50)
+ (assoc-in [30] 100)
+ (assoc-in [70] 35))))
+ (outliers nil)]))
+
+(expect
+ [2.0 2.0]
+ (centroid [[0 1] [2 3] [4 2]]))
+
+(expect
+ 8.0
+ (triangle-area [-2 0] [2 0] [0 4]))
diff --git a/test/metabase/feature_extraction/stl_test.clj b/test/metabase/feature_extraction/stl_test.clj
deleted file mode 100644
index df35ce164490d0d5428d6213abec1c19758ab091..0000000000000000000000000000000000000000
--- a/test/metabase/feature_extraction/stl_test.clj
+++ /dev/null
@@ -1,13 +0,0 @@
-(ns metabase.feature-extraction.stl-test
- (:require [expectations :refer :all]
- [metabase.feature-extraction.stl :as stl]))
-
-(def ^:private ts (mapv vector (range) (take 100 (cycle (range 10)))))
-
-(expect
- true
- (every? (stl/decompose 10 ts) [:xs :ys :trend :residual :seasonal]))
-
-(expect
- IllegalArgumentException
- (stl/decompose 100 ts))
diff --git a/test/metabase/feature_extraction/timeseries_test.clj b/test/metabase/feature_extraction/timeseries_test.clj
new file mode 100644
index 0000000000000000000000000000000000000000..9b4300d554b0a2ec378995331e5ff71b02bfacc8
--- /dev/null
+++ b/test/metabase/feature_extraction/timeseries_test.clj
@@ -0,0 +1,57 @@
+(ns metabase.feature-extraction.timeseries-test
+ (:require [clj-time
+ [core :as t]
+ [coerce :as t.coerce]]
+ [expectations :refer :all]
+ [metabase.feature-extraction.timeseries :refer :all]))
+
+(expect
+ [1
+ 1
+ 2
+ 4]
+ [(quarter (t/date-time 2017 1))
+ (quarter (t/date-time 2017 3))
+ (quarter (t/date-time 2017 5))
+ (quarter (t/date-time 2017 12))])
+
+(defn- make-timestamp
+ [& args]
+ (-> (apply t/date-time args)
+ to-double))
+
+(expect
+ [[(make-timestamp 2016 1) 12]
+ [(make-timestamp 2016 2) 0]
+ [(make-timestamp 2016 3) 4]
+ [(make-timestamp 2016 4) 0]
+ [(make-timestamp 2016 5) 0]
+ [(make-timestamp 2016 6) 0]
+ [(make-timestamp 2016 7) 0]
+ [(make-timestamp 2016 8) 0]
+ [(make-timestamp 2016 9) 0]
+ [(make-timestamp 2016 10) 0]
+ [(make-timestamp 2016 11) 0]
+ [(make-timestamp 2016 12) 0]
+ [(make-timestamp 2017 1) 25]]
+ (fill-timeseries :month [[(make-timestamp 2016 1) 12]
+ [(make-timestamp 2016 3) 4]
+ [(make-timestamp 2017 1) 25]]))
+
+(def ^:private ts (mapv vector (range) (take 100 (cycle (range 10)))))
+
+(expect
+ true
+ (every? (decompose :month ts) [:trend :residual :seasonal]))
+(expect
+ nil
+ (decompose 100 ts))
+
+(expect
+ [[99]
+ []
+ []]
+ [(breaks 12 (map vector (range) (concat (repeat 100 10)
+ (repeat 100 20))))
+ (breaks 12 (map vector (range) (take 100 (cycle (range 10)))))
+ (breaks 4 nil)])
diff --git a/test/metabase/middleware_test.clj b/test/metabase/middleware_test.clj
index 94cf4d396ace2e23988ffac8943f6adae2d92ae7..a23a1a9ffb77d4d8cacd96228748c5d1ee911969 100644
--- a/test/metabase/middleware_test.clj
+++ b/test/metabase/middleware_test.clj
@@ -220,7 +220,7 @@
eventually (apply str (async/<!! (async/into [] connection)))]
[first-second second-second (string/trim eventually)])))))
-
+(comment
;;slow success
(expect
[\newline \newline "{\"success\":true}"]
@@ -276,3 +276,4 @@
(expect
:ran-to-compleation
(start-and-maybe-kill-test-request false))
+)
diff --git a/yarn.lock b/yarn.lock
index a20cf291e3bb8ba934fcaef360597c0a5d11dc64..4a51a3250ba88e1cb0c2a4e4628463e14e1a4d3e 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -113,21 +113,21 @@ ajv-keywords@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-2.1.0.tgz#a296e17f7bfae7c1ce4f7e0de53d29cb32162df0"
-ajv@4.9.0:
+ajv@4.9.0, ajv@^4.7.0:
version "4.9.0"
resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.9.0.tgz#5a358085747b134eb567d6d15e015f1d7802f45c"
dependencies:
co "^4.6.0"
json-stable-stringify "^1.0.1"
-ajv@^4.7.0, ajv@^4.9.1:
+ajv@^4.9.1:
version "4.11.8"
resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.8.tgz#82ffb02b29e662ae53bdc20af15947706739c536"
dependencies:
co "^4.6.0"
json-stable-stringify "^1.0.1"
-ajv@^5.0.0, ajv@^5.1.0, ajv@^5.1.5:
+ajv@^5.0.0, ajv@^5.1.5:
version "5.2.3"
resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.2.3.tgz#c06f598778c44c6b161abafe3466b81ad1814ed2"
dependencies:
@@ -419,11 +419,7 @@ aws-sign2@~0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f"
-aws-sign2@~0.7.0:
- version "0.7.0"
- resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
-
-aws4@^1.2.1, aws4@^1.6.0:
+aws4@^1.2.1:
version "1.6.0"
resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e"
@@ -1432,19 +1428,7 @@ boom@2.x.x:
dependencies:
hoek "2.x.x"
-boom@4.x.x:
- version "4.3.1"
- resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31"
- dependencies:
- hoek "4.x.x"
-
-boom@5.x.x:
- version "5.2.0"
- resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02"
- dependencies:
- hoek "4.x.x"
-
-brace-expansion@^1.0.0, brace-expansion@^1.1.7:
+brace-expansion@^1.1.7:
version "1.1.8"
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292"
dependencies:
@@ -2325,12 +2309,6 @@ cryptiles@2.x.x:
dependencies:
boom "2.x.x"
-cryptiles@3.x.x:
- version "3.1.2"
- resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.2.tgz#a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe"
- dependencies:
- boom "5.x.x"
-
crypto-browserify@^3.11.0:
version "3.11.1"
resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-3.11.1.tgz#948945efc6757a400d6e5e5af47194d10064279f"
@@ -2530,11 +2508,11 @@ dc@^2.0.0:
crossfilter2 "~1.3"
d3 "^3"
-debug@2, debug@^2.1.1:
- version "2.6.3"
- resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.3.tgz#0f7eb8c30965ec08c72accfa0130c8b79984141d"
+debug@2, debug@2.6.9, debug@^2.1.1, debug@^2.2.0, debug@^2.3.3, debug@^2.6.3, debug@^2.6.6, debug@^2.6.8, debug@~2.6.7:
+ version "2.6.9"
+ resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
dependencies:
- ms "0.7.2"
+ ms "2.0.0"
debug@2.2.0, debug@~2.2.0:
version "2.2.0"
@@ -2554,12 +2532,6 @@ debug@2.6.1:
dependencies:
ms "0.7.2"
-debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.3, debug@^2.6.6, debug@^2.6.8, debug@~2.6.7:
- version "2.6.9"
- resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
- dependencies:
- ms "2.0.0"
-
debug@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
@@ -2867,20 +2839,13 @@ domutils@1.1:
dependencies:
domelementtype "1"
-domutils@1.5.1:
+domutils@1.5.1, domutils@^1.5.1:
version "1.5.1"
resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf"
dependencies:
dom-serializer "0"
domelementtype "1"
-domutils@^1.5.1:
- version "1.6.2"
- resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.6.2.tgz#1958cc0b4c9426e9ed367fb1c8e854891b0fa3ff"
- dependencies:
- dom-serializer "0"
- domelementtype "1"
-
duplexer2@^0.1.2, duplexer2@~0.1.0:
version "0.1.4"
resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1"
@@ -3063,7 +3028,7 @@ error@^7.0.0:
string-template "~0.2.1"
xtend "~4.0.0"
-es-abstract@^1.6.1:
+es-abstract@^1.6.1, es-abstract@^1.7.0:
version "1.7.0"
resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.7.0.tgz#dfade774e01bfcd97f96180298c449c8623fb94c"
dependencies:
@@ -3072,16 +3037,6 @@ es-abstract@^1.6.1:
is-callable "^1.1.3"
is-regex "^1.0.3"
-es-abstract@^1.7.0:
- version "1.9.0"
- resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.9.0.tgz#690829a07cae36b222e7fd9b75c0d0573eb25227"
- dependencies:
- es-to-primitive "^1.1.1"
- function-bind "^1.1.1"
- has "^1.0.1"
- is-callable "^1.1.3"
- is-regex "^1.0.4"
-
es-to-primitive@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.1.1.tgz#45355248a88979034b6792e19bb81f2b7975dd0d"
@@ -3498,14 +3453,10 @@ extend-shallow@^2.0.1:
dependencies:
is-extendable "^0.1.0"
-extend@3, extend@^3.0.0:
+extend@3, extend@^3.0.0, extend@~3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.0.tgz#5a474353b9f3353ddd8176dfd37b91c83a46f1d4"
-extend@~3.0.0, extend@~3.0.1:
- version "3.0.1"
- resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444"
-
extglob@^0.3.1:
version "0.3.2"
resolved "https://registry.yarnpkg.com/extglob/-/extglob-0.3.2.tgz#2e18ff3d2f49ab2765cec9023f011daa8d8349a1"
@@ -3765,14 +3716,6 @@ form-data@~2.1.1:
combined-stream "^1.0.5"
mime-types "^2.1.12"
-form-data@~2.3.1:
- version "2.3.1"
- resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.1.tgz#6fb94fbd71885306d73d15cc497fe4cc4ecd44bf"
- dependencies:
- asynckit "^0.4.0"
- combined-stream "^1.0.5"
- mime-types "^2.1.12"
-
forwarded@~0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84"
@@ -3857,7 +3800,7 @@ fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2:
mkdirp ">=0.5 0"
rimraf "2"
-function-bind@^1.0.2, function-bind@^1.1.0, function-bind@^1.1.1:
+function-bind@^1.0.2, function-bind@^1.1.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"
@@ -3959,18 +3902,12 @@ git-url-parse@^6.0.1:
dependencies:
git-up "^2.0.0"
-github-slugger@1.1.3:
+github-slugger@1.1.3, github-slugger@^1.0.0, github-slugger@^1.1.1:
version "1.1.3"
resolved "https://registry.yarnpkg.com/github-slugger/-/github-slugger-1.1.3.tgz#314a6e759a18c2b0cc5760d512ccbab549c549a7"
dependencies:
emoji-regex ">=6.0.0 <=6.1.1"
-github-slugger@^1.0.0, github-slugger@^1.1.1:
- version "1.2.0"
- resolved "https://registry.yarnpkg.com/github-slugger/-/github-slugger-1.2.0.tgz#8ada3286fd046d8951c3c952a8d7854cfd90fd9a"
- dependencies:
- emoji-regex ">=6.0.0 <=6.1.1"
-
glob-base@^0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/glob-base/-/glob-base-0.3.0.tgz#dbb164f6221b1c0b1ccf82aea328b497df0ea3c4"
@@ -4004,7 +3941,7 @@ glob-stream@^5.3.2:
to-absolute-glob "^0.1.1"
unique-stream "^2.0.2"
-glob@7.1.1, glob@^7.0.0, glob@^7.0.6, glob@^7.1.1:
+glob@7.1.1, glob@^7.1.1:
version "7.1.1"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8"
dependencies:
@@ -4025,7 +3962,7 @@ glob@^5.0.3:
once "^1.3.0"
path-is-absolute "^1.0.0"
-glob@^7.0.3, glob@^7.0.5, glob@^7.1.2:
+glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.0.6, glob@^7.1.2:
version "7.1.2"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
dependencies:
@@ -4113,10 +4050,6 @@ har-schema@^1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e"
-har-schema@^2.0.0:
- version "2.0.0"
- resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
-
har-validator@~2.0.6:
version "2.0.6"
resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-2.0.6.tgz#cdcbc08188265ad119b6a5a7c8ab70eecfb5d27d"
@@ -4133,13 +4066,6 @@ har-validator@~4.2.1:
ajv "^4.9.1"
har-schema "^1.0.5"
-har-validator@~5.0.3:
- version "5.0.3"
- resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd"
- dependencies:
- ajv "^5.1.0"
- har-schema "^2.0.0"
-
has-ansi@^0.1.0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-0.1.0.tgz#84f265aae8c0e6a88a12d7022894b7568894c62e"
@@ -4266,15 +4192,6 @@ hawk@3.1.3, hawk@~3.1.3:
hoek "2.x.x"
sntp "1.x.x"
-hawk@~6.0.2:
- version "6.0.2"
- resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038"
- dependencies:
- boom "4.x.x"
- cryptiles "3.x.x"
- hoek "4.x.x"
- sntp "2.x.x"
-
he@1.1.x:
version "1.1.1"
resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd"
@@ -4304,10 +4221,6 @@ hoek@2.x.x:
version "2.16.3"
resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed"
-hoek@4.x.x:
- version "4.2.0"
- resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d"
-
hoist-non-react-statics@1.2.0, hoist-non-react-statics@^1.0.0, hoist-non-react-statics@^1.0.5, hoist-non-react-statics@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-1.2.0.tgz#aa448cf0986d55cc40773b17174b7dd066cb7cfb"
@@ -4445,14 +4358,6 @@ http-signature@~1.1.0:
jsprim "^1.2.2"
sshpk "^1.7.0"
-http-signature@~1.2.0:
- version "1.2.0"
- resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
- dependencies:
- assert-plus "^1.0.0"
- jsprim "^1.2.2"
- sshpk "^1.7.0"
-
https-browserify@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-0.0.1.tgz#3f91365cabe60b77ed0ebba24b454e3e09d95a82"
@@ -4477,11 +4382,11 @@ iconv-lite@0.4.13:
version "0.4.13"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.13.tgz#1f88aba4ab0b1508e8312acc39345f36e992e2f2"
-iconv-lite@0.4.15:
+iconv-lite@0.4.15, iconv-lite@~0.4.13:
version "0.4.15"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.15.tgz#fe265a218ac6a57cfe854927e9d04c19825eddeb"
-iconv-lite@0.4.19, iconv-lite@~0.4.13:
+iconv-lite@0.4.19:
version "0.4.19"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b"
@@ -4566,7 +4471,7 @@ inflight@^1.0.4:
once "^1.3.0"
wrappy "1"
-inherits@2, inherits@2.0.3, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3:
+inherits@2, inherits@2.0.3, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.0, inherits@~2.0.1:
version "2.0.3"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
@@ -4863,7 +4768,7 @@ is-property@^1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/is-property/-/is-property-1.0.2.tgz#57fe1c4e48474edd65b09911f26b1cd4095dda84"
-is-regex@^1.0.3, is-regex@^1.0.4:
+is-regex@^1.0.3:
version "1.0.4"
resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.0.4.tgz#5517489b547091b0930e095654ced25ee97e9491"
dependencies:
@@ -6161,14 +6066,10 @@ mime-types@^2.1.12, mime-types@~2.1.11, mime-types@~2.1.15, mime-types@~2.1.16,
dependencies:
mime-db "~1.30.0"
-mime@1.4.1, mime@^1.3.4:
+mime@1.4.1, mime@^1.2.11, mime@^1.3.4:
version "1.4.1"
resolved "https://registry.yarnpkg.com/mime/-/mime-1.4.1.tgz#121f9ebc49e3766f311a76e1fa1c8003c4b03aa6"
-mime@^1.2.11:
- version "1.3.4"
- resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53"
-
mimic-fn@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.1.0.tgz#e667783d92e89dbd342818b5230b9d62a672ad18"
@@ -6181,19 +6082,13 @@ minimalistic-crypto-utils@^1.0.0, minimalistic-crypto-utils@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz#f6c00c1c0b082246e5c4d99dfb8c7c083b2b582a"
-"minimatch@2 || 3", minimatch@^3.0.3:
- version "3.0.3"
- resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774"
- dependencies:
- brace-expansion "^1.0.0"
-
-minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4:
+"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.3, minimatch@^3.0.4:
version "3.0.4"
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
dependencies:
brace-expansion "^1.1.7"
-minimist@0.0.8:
+minimist@0.0.8, minimist@~0.0.1:
version "0.0.8"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d"
@@ -6201,10 +6096,6 @@ minimist@1.2.0, minimist@>=0.2.0, minimist@^1.1.0, minimist@^1.1.1, minimist@^1.
version "1.2.0"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284"
-minimist@~0.0.1:
- version "0.0.10"
- resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf"
-
mississippi@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/mississippi/-/mississippi-1.3.0.tgz#d201583eb12327e3c5c1642a404a9cacf94e34f5"
@@ -6256,14 +6147,10 @@ module-deps-sortable@4.0.6:
through2 "^2.0.0"
xtend "^4.0.0"
-moment@2.14.1:
+moment@2.14.1, moment@2.x.x:
version "2.14.1"
resolved "https://registry.yarnpkg.com/moment/-/moment-2.14.1.tgz#b35b27c47e57ed2ddc70053d6b07becdb291741c"
-moment@2.x.x:
- version "2.19.1"
- resolved "https://registry.yarnpkg.com/moment/-/moment-2.19.1.tgz#56da1a2d1cbf01d38b7e1afc31c10bcfa1929167"
-
move-concurrently@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92"
@@ -6551,7 +6438,7 @@ number-to-locale-string@^1.0.1:
version "1.4.3"
resolved "https://registry.yarnpkg.com/nwmatcher/-/nwmatcher-1.4.3.tgz#64348e3b3d80f035b40ac11563d278f8b72db89c"
-oauth-sign@~0.8.1, oauth-sign@~0.8.2:
+oauth-sign@~0.8.1:
version "0.8.2"
resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43"
@@ -6738,7 +6625,7 @@ os-locale@^2.0.0:
lcid "^1.0.0"
mem "^1.1.0"
-os-tmpdir@^1.0.0, os-tmpdir@^1.0.1, os-tmpdir@~1.0.1, os-tmpdir@~1.0.2:
+os-tmpdir@^1.0.0, os-tmpdir@^1.0.1, os-tmpdir@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
@@ -7731,7 +7618,7 @@ qs@6.4.0, qs@~6.4.0:
version "6.4.0"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233"
-qs@6.5.1, qs@^6.4.0, qs@~6.5.1:
+qs@6.5.1, qs@^6.4.0:
version "6.5.1"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8"
@@ -7868,19 +7755,12 @@ react-dom@^15.5.3, react-dom@^15.5.4:
object-assign "^4.1.0"
prop-types "^15.5.10"
-react-draggable@^2.2.3:
+react-draggable@^2.2.3, "react-draggable@^2.2.6 || ^3.0.3":
version "2.2.6"
resolved "https://registry.yarnpkg.com/react-draggable/-/react-draggable-2.2.6.tgz#3a806e10f2da6babfea4136be6510e89b0d76901"
dependencies:
classnames "^2.2.5"
-"react-draggable@^2.2.6 || ^3.0.3":
- version "3.0.3"
- resolved "https://registry.yarnpkg.com/react-draggable/-/react-draggable-3.0.3.tgz#a6f9b3a7171981b76dadecf238316925cb9eacf4"
- dependencies:
- classnames "^2.2.5"
- prop-types "^15.5.10"
-
react-element-to-jsx-string@^6.3.0:
version "6.4.0"
resolved "https://registry.yarnpkg.com/react-element-to-jsx-string/-/react-element-to-jsx-string-6.4.0.tgz#3a7cdbce3dc8576e0096c735e60a85d704210a23"
@@ -8054,16 +7934,16 @@ read-pkg@^2.0.0:
normalize-package-data "^2.3.2"
path-type "^2.0.0"
-"readable-stream@1 || 2", readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.1.4, readable-stream@^2.2.9:
- version "2.3.3"
- resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c"
+"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.4, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.1.4, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.2.6, readable-stream@^2.2.9:
+ version "2.2.9"
+ resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.9.tgz#cf78ec6f4a6d1eb43d26488cac97f042e74b7fc8"
dependencies:
+ buffer-shims "~1.0.0"
core-util-is "~1.0.0"
- inherits "~2.0.3"
+ inherits "~2.0.1"
isarray "~1.0.0"
process-nextick-args "~1.0.6"
- safe-buffer "~5.1.1"
- string_decoder "~1.0.3"
+ string_decoder "~1.0.0"
util-deprecate "~1.0.1"
readable-stream@1.0, "readable-stream@>=1.0.33-1 <1.1.0-0", readable-stream@~1.0.2:
@@ -8075,18 +7955,6 @@ readable-stream@1.0, "readable-stream@>=1.0.33-1 <1.1.0-0", readable-stream@~1.0
isarray "0.0.1"
string_decoder "~0.10.x"
-readable-stream@^2.0.0, readable-stream@^2.0.4, readable-stream@^2.0.5, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.2.6:
- version "2.2.9"
- resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.9.tgz#cf78ec6f4a6d1eb43d26488cac97f042e74b7fc8"
- dependencies:
- buffer-shims "~1.0.0"
- core-util-is "~1.0.0"
- inherits "~2.0.1"
- isarray "~1.0.0"
- process-nextick-args "~1.0.6"
- string_decoder "~1.0.0"
- util-deprecate "~1.0.1"
-
readable-stream@~2.0.0:
version "2.0.6"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.0.6.tgz#8f90341e68a53ccc928788dacfcd11b36eb9b78e"
@@ -8405,7 +8273,7 @@ replace-ext@1.0.0, replace-ext@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/replace-ext/-/replace-ext-1.0.0.tgz#de63128373fcbf7c3ccfa4de5a480c45a67958eb"
-request@2.81.0:
+request@2.81.0, request@^2.79.0:
version "2.81.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0"
dependencies:
@@ -8457,33 +8325,6 @@ request@2.81.0:
tough-cookie "~2.3.0"
tunnel-agent "~0.4.1"
-request@^2.79.0:
- version "2.83.0"
- resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356"
- dependencies:
- aws-sign2 "~0.7.0"
- aws4 "^1.6.0"
- caseless "~0.12.0"
- combined-stream "~1.0.5"
- extend "~3.0.1"
- forever-agent "~0.6.1"
- form-data "~2.3.1"
- har-validator "~5.0.3"
- hawk "~6.0.2"
- http-signature "~1.2.0"
- is-typedarray "~1.0.0"
- isstream "~0.1.2"
- json-stringify-safe "~5.0.1"
- mime-types "~2.1.17"
- oauth-sign "~0.8.2"
- performance-now "^2.1.0"
- qs "~6.5.1"
- safe-buffer "^5.1.1"
- stringstream "~0.0.5"
- tough-cookie "~2.3.3"
- tunnel-agent "^0.6.0"
- uuid "^3.1.0"
-
require-directory@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
@@ -8558,18 +8399,12 @@ right-align@^0.1.1:
dependencies:
align-text "^0.1.1"
-rimraf@2, rimraf@^2.2.8, rimraf@^2.5.1, rimraf@^2.6.1:
+rimraf@2, rimraf@^2.2.8, rimraf@^2.5.1, rimraf@^2.5.4, rimraf@^2.6.0, rimraf@^2.6.1:
version "2.6.2"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36"
dependencies:
glob "^7.0.5"
-rimraf@^2.5.4, rimraf@^2.6.0:
- version "2.6.1"
- resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.1.tgz#c2338ec643df7a1b7fe5c54fa86f57428a55f33d"
- dependencies:
- glob "^7.0.5"
-
ripemd160@^2.0.0, ripemd160@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.1.tgz#0f4584295c53a3628af7e6d79aca21ce57d1c6e7"
@@ -8603,7 +8438,7 @@ rxjs@^5.0.0-beta.11:
dependencies:
symbol-observable "^1.0.1"
-safe-buffer@5.1.1, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
+safe-buffer@5.1.1, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@~5.1.0:
version "5.1.1"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
@@ -8850,12 +8685,6 @@ sntp@1.x.x:
dependencies:
hoek "2.x.x"
-sntp@2.x.x:
- version "2.0.2"
- resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.0.2.tgz#5064110f0af85f7cfdb7d6b67a40028ce52b4b2b"
- dependencies:
- hoek "4.x.x"
-
socket.io-adapter@0.5.0:
version "0.5.0"
resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-0.5.0.tgz#cb6d4bb8bec81e1078b99677f9ced0046066bb8b"
@@ -9164,7 +8993,7 @@ string_decoder@0.10, string_decoder@^0.10.25, string_decoder@~0.10.x:
version "0.10.31"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
-string_decoder@~1.0.0, string_decoder@~1.0.3:
+string_decoder@~1.0.0:
version "1.0.3"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab"
dependencies:
@@ -9187,7 +9016,7 @@ stringify-object@^3.2.0:
is-obj "^1.0.1"
is-regexp "^1.0.0"
-stringstream@~0.0.4, stringstream@~0.0.5:
+stringstream@~0.0.4:
version "0.0.5"
resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878"
@@ -9267,13 +9096,7 @@ supports-color@^3.1.2, supports-color@^3.2.3:
dependencies:
has-flag "^1.0.0"
-supports-color@^4.0.0, supports-color@^4.1.0, supports-color@^4.4.0:
- version "4.4.0"
- resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-4.4.0.tgz#883f7ddabc165142b2a61427f3352ded195d1a3e"
- dependencies:
- has-flag "^2.0.0"
-
-supports-color@^4.2.1:
+supports-color@^4.0.0, supports-color@^4.1.0, supports-color@^4.2.1, supports-color@^4.4.0:
version "4.5.0"
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-4.5.0.tgz#be7a0de484dec5c5cddf8b3d59125044912f635b"
dependencies:
@@ -9448,18 +9271,12 @@ tmp@0.0.24:
version "0.0.24"
resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.24.tgz#d6a5e198d14a9835cc6f2d7c3d9e302428c8cf12"
-tmp@0.0.31:
+tmp@0.0.31, tmp@0.0.x:
version "0.0.31"
resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.31.tgz#8f38ab9438e17315e5dbd8b3657e8bfb277ae4a7"
dependencies:
os-tmpdir "~1.0.1"
-tmp@0.0.x:
- version "0.0.33"
- resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9"
- dependencies:
- os-tmpdir "~1.0.2"
-
tmpl@1.0.x:
version "1.0.4"
resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1"
@@ -9517,13 +9334,7 @@ toposort@^1.0.0:
version "1.0.6"
resolved "https://registry.yarnpkg.com/toposort/-/toposort-1.0.6.tgz#c31748e55d210effc00fdcdc7d6e68d7d7bb9cec"
-tough-cookie@^2.3.2:
- version "2.3.2"
- resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.2.tgz#f081f76e4c85720e6c37a5faced737150d84072a"
- dependencies:
- punycode "^1.4.1"
-
-tough-cookie@~2.3.0, tough-cookie@~2.3.3:
+tough-cookie@^2.3.2, tough-cookie@~2.3.0:
version "2.3.3"
resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561"
dependencies:
@@ -9878,7 +9689,7 @@ uuid@^2.0.2:
version "2.0.3"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a"
-uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0:
+uuid@^3.0.0, uuid@^3.0.1:
version "3.1.0"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"
@@ -10272,14 +10083,10 @@ xml2js@0.4.4:
sax "0.6.x"
xmlbuilder ">=1.0.0"
-xmlbuilder@8.2.2:
+xmlbuilder@8.2.2, xmlbuilder@>=1.0.0:
version "8.2.2"
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-8.2.2.tgz#69248673410b4ba42e1a6136551d2922335aa773"
-xmlbuilder@>=1.0.0:
- version "9.0.4"
- resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.4.tgz#519cb4ca686d005a8420d3496f3f0caeecca580f"
-
xmldom@^0.1.22:
version "0.1.27"
resolved "https://registry.yarnpkg.com/xmldom/-/xmldom-0.1.27.tgz#d501f97b3bdb403af8ef9ecc20573187aadac0e9"