fix replace missing values with zeros on time series (#42058)

frontend/src/metabase/static-viz/components/ComboChart/ComboChart.stories.tsx

@@ -604,6 +604,7 @@ HistogramTicks45Degrees.args = {
   dashcardSettings: {},
   renderingContext,
 };
+
 export const HistogramTicks90Degrees = Template.bind({});
 HistogramTicks90Degrees.args = {
   rawSeries: data.histogramTicks90Degrees as any,
@@ -618,6 +619,13 @@ LineUnpinFromZero.args = {
   renderingContext,
 };
 
+export const LineReplaceMissingValuesZero = Template.bind({});
+LineReplaceMissingValuesZero.args = {
+  rawSeries: data.lineReplaceMissingValuesZero as any,
+  dashcardSettings: {},
+  renderingContext,
+};
+
 export const Default = Template.bind({});
 Default.args = {
   rawSeries: data.messedUpAxis as any,

frontend/src/metabase/static-viz/components/ComboChart/stories-data/index.ts

@@ -51,6 +51,7 @@ import lineLinearXScaleUnsorted from "./line-linear-x-scale-unsorted.json";
 import lineLinearXScale from "./line-linear-x-scale.json";
 import lineLogYScaleNegative from "./line-log-y-scale-negative.json";
 import lineLogYScale from "./line-log-y-scale.json";
+import lineReplaceMissingValuesZero from "./line-replace-missing-values-zero.json";
 import lineShowDotsAuto from "./line-show-dots-auto.json";
 import lineShowDotsOff from "./line-show-dots-off.json";
 import lineShowDotsOn from "./line-show-dots-on.json";
@@ -170,4 +171,5 @@ export const data = {
   histogramTicks45Degrees,
   histogramTicks90Degrees,
   lineUnpinFromZero,
+  lineReplaceMissingValuesZero,
 };

frontend/src/metabase/static-viz/components/ComboChart/stories-data/line-replace-missing-values-zero.json

+[
+  {
+    "card": {
+      "public_uuid": null,
+      "parameter_usage_count": 0,
+      "created_at": "2024-05-01T23:25:14.654039Z",
+      "parameters": [],
+      "metabase_version": "v0.48.1-SNAPSHOT (8053de5)",
+      "collection": null,
+      "visualization_settings": {
+        "series_settings": {
+          "Y": {
+            "line.missing": "zero"
+          }
+        },
+        "graph.dimensions": ["X"],
+        "graph.metrics": ["Y"]
+      },
+      "collection_preview": true,
+      "entity_id": "hr7YTZePecARmmOlJMmYM",
+      "display": "line",
+      "parameter_mappings": [],
+      "id": 197,
+      "dataset_query": {
+        "database": 1,
+        "type": "native",
+        "native": {
+          "query": "SELECT DATE '2020-01-01' AS x, 10 AS y\nUNION ALL\nSELECT DATE '2023-01-01', 10\nUNION ALL\nSELECT DATE '2024-01-01', 10",
+          "template-tags": {}
+        }
+      },
+      "cache_ttl": null,
+      "embedding_params": null,
+      "made_public_by_id": null,
+      "updated_at": "2024-05-01T23:25:14.654039Z",
+      "moderation_reviews": [],
+      "creator_id": 1,
+      "average_query_time": null,
+      "type": "question",
+      "dashboard_count": 0,
+      "last_query_start": null,
+      "name": "replace missing values with zero",
+      "query_type": "native",
+      "collection_id": null,
+      "enable_embedding": false,
+      "database_id": 1,
+      "can_write": true,
+      "initially_published_at": null,
+      "result_metadata": null,
+      "table_id": null,
+      "collection_position": null,
+      "view_count": 0,
+      "archived": false,
+      "description": null,
+      "cache_invalidated_at": null,
+      "displayIsLocked": true
+    },
+    "data": {
+      "rows": [
+        ["2020-01-01T00:00:00-03:00", 10],
+        ["2023-01-01T00:00:00-03:00", 10],
+        ["2024-01-01T00:00:00-03:00", 10]
+      ],
+      "cols": [
+        {
+          "display_name": "X",
+          "source": "native",
+          "field_ref": [
+            "field",
+            "X",
+            {
+              "base-type": "type/Date"
+            }
+          ],
+          "name": "X",
+          "base_type": "type/Date",
+          "effective_type": "type/Date"
+        },
+        {
+          "display_name": "Y",
+          "source": "native",
+          "field_ref": [
+            "field",
+            "Y",
+            {
+              "base-type": "type/Integer"
+            }
+          ],
+          "name": "Y",
+          "base_type": "type/Integer",
+          "effective_type": "type/Integer"
+        }
+      ],
+      "native_form": {
+        "params": null,
+        "query": "SELECT DATE '2020-01-01' AS x, 10 AS y\nUNION ALL\nSELECT DATE '2023-01-01', 10\nUNION ALL\nSELECT DATE '2024-01-01', 10"
+      },
+      "format-rows?": true,
+      "results_timezone": "America/Montevideo",
+      "requested_timezone": "Canada/Eastern",
+      "results_metadata": {
+        "columns": [
+          {
+            "display_name": "X",
+            "field_ref": [
+              "field",
+              "X",
+              {
+                "base-type": "type/Date"
+              }
+            ],
+            "name": "X",
+            "base_type": "type/Date",
+            "effective_type": "type/Date",
+            "semantic_type": null,
+            "fingerprint": {
+              "global": {
+                "distinct-count": 3,
+                "nil%": 0
+              },
+              "type": {
+                "type/DateTime": {
+                  "earliest": "2020-01-01T00:00:00-03:00",
+                  "latest": "2024-01-01T00:00:00-03:00"
+                }
+              }
+            }
+          },
+          {
+            "display_name": "Y",
+            "field_ref": [
+              "field",
+              "Y",
+              {
+                "base-type": "type/Integer"
+              }
+            ],
+            "name": "Y",
+            "base_type": "type/Integer",
+            "effective_type": "type/Integer",
+            "semantic_type": null,
+            "fingerprint": {
+              "global": {
+                "distinct-count": 1,
+                "nil%": 0
+              },
+              "type": {
+                "type/Number": {
+                  "min": 10,
+                  "q1": 10,
+                  "q3": 10,
+                  "max": 10,
+                  "sd": 0,
+                  "avg": 10
+                }
+              }
+            }
+          }
+        ]
+      },
+      "insights": [
+        {
+          "previous-value": 10,
+          "unit": "year",
+          "offset": 10,
+          "last-change": 0,
+          "col": "Y",
+          "slope": 0,
+          "last-value": 10,
+          "best-fit": ["+", 10, ["*", 0, ["log", "x"]]]
+        }
+      ]
+    }
+  }
+]

frontend/src/metabase/visualizations/echarts/cartesian/model/dataset.ts

 import { t } from "ttag";
 
 import { getObjectKeys, getObjectValues } from "metabase/lib/objects";
+import { parseTimestamp } from "metabase/lib/time-dayjs";
 import { checkNumber, isNotNull } from "metabase/lib/types";
 import { isEmpty } from "metabase/lib/validate";
 import {
@@ -21,6 +22,7 @@ import type {
   XAxisModel,
   NumericAxisScaleTransforms,
   ShowWarning,
+  TimeSeriesXAxisModel,
 } from "metabase/visualizations/echarts/cartesian/model/types";
 import type { CartesianChartColumns } from "metabase/visualizations/lib/graph/columns";
 import { getNumberOr } from "metabase/visualizations/lib/settings/row-values";
@@ -282,12 +284,13 @@ export const getNullReplacerTransform = (
     )
     .map(seriesModel => seriesModel.dataKey);
 
-  return getKeyBasedDatasetTransform(
-    replaceNullsWithZeroDataKeys,
-    (value: RowValue) => {
-      return value === null ? 0 : value;
-    },
-  );
+  return datum => {
+    const transformedDatum = { ...datum };
+    for (const key of replaceNullsWithZeroDataKeys) {
+      transformedDatum[key] = datum[key] != null ? datum[key] : 0;
+    }
+    return transformedDatum;
+  };
 };
 
 const hasInterpolatedSeries = (
@@ -518,6 +521,43 @@ function getHistogramDataset(
   return dataset;
 }
 
+const MAX_FILL_COUNT = 10000;
+
+const interpolateTimeSeriesData = (
+  dataset: ChartDataset,
+  axisModel: TimeSeriesXAxisModel,
+): ChartDataset => {
+  if (axisModel.intervalsCount > MAX_FILL_COUNT) {
+    return dataset;
+  }
+
+  const { count, unit } = axisModel.interval;
+  const result = [];
+
+  for (let i = 0; i < dataset.length; i++) {
+    const datum = dataset[i];
+    result.push(datum);
+
+    if (i === dataset.length - 1) {
+      break;
+    }
+
+    const end = parseTimestamp(dataset[i + 1][X_AXIS_DATA_KEY]);
+
+    let start = parseTimestamp(datum[X_AXIS_DATA_KEY]);
+    while (start.add(count, unit).isBefore(end)) {
+      const interpolatedValue = start.add(count, unit);
+      result.push({
+        [X_AXIS_DATA_KEY]: interpolatedValue.toISOString(),
+      });
+
+      start = interpolatedValue;
+    }
+  }
+
+  return result;
+};
+
 /**
  * Modifies the dataset for visualization according to the specified visualization settings.
  *
@@ -539,8 +579,8 @@ export const applyVisualizationSettingsDataTransformations = (
   const seriesDataKeys = seriesModels.map(seriesModel => seriesModel.dataKey);
 
   if (
-    xAxisModel.axisType === "value" ||
-    xAxisModel.axisType === "time" ||
+    isNumericAxis(xAxisModel) ||
+    isTimeSeriesAxis(xAxisModel) ||
     xAxisModel.isHistogram
   ) {
     dataset = filterNullDimensionValues(dataset, showWarning);
@@ -550,10 +590,14 @@ export const applyVisualizationSettingsDataTransformations = (
     dataset = replaceZeroesForLogScale(dataset, seriesDataKeys);
   }
 
-  if (xAxisModel.axisType === "category" && xAxisModel.isHistogram) {
+  if (isCategoryAxis(xAxisModel) && xAxisModel.isHistogram) {
     dataset = getHistogramDataset(dataset, xAxisModel.histogramInterval);
   }
 
+  if (isTimeSeriesAxis(xAxisModel)) {
+    dataset = interpolateTimeSeriesData(dataset, xAxisModel);
+  }
+
   return transformDataset(dataset, [
     getNullReplacerTransform(settings, seriesModels),
     {

frontend/src/metabase/visualizations/echarts/cartesian/model/dataset.unit.spec.ts

+import dayjs from "dayjs";
+
 import { createMockSeriesModel } from "__support__/echarts";
 import { checkNumber } from "metabase/lib/types";
 import {
+  ORIGINAL_INDEX_DATA_KEY,
   POSITIVE_STACK_TOTAL_DATA_KEY,
   X_AXIS_DATA_KEY,
 } from "metabase/visualizations/echarts/cartesian/constants/dataset";
@@ -34,6 +37,7 @@ import type {
   ChartDataset,
   LegacySeriesSettingsObjectKey,
   NumericAxisScaleTransforms,
+  TimeSeriesXAxisModel,
   XAxisModel,
 } from "./types";
 
@@ -400,6 +404,82 @@ describe("dataset transform functions", () => {
       ]);
     });
 
+    describe("time series", () => {
+      const dataset = [
+        {
+          [X_AXIS_DATA_KEY]: "2020-01-01T00:00:00.000Z",
+          dimensionKey: "A",
+          series1: 10,
+        },
+        // Missing February
+        {
+          [X_AXIS_DATA_KEY]: "2020-03-01T00:00:00.000Z",
+          dimensionKey: "A",
+          series1: 20,
+        },
+      ];
+
+      const xAxisModel: TimeSeriesXAxisModel = {
+        axisType: "time",
+        intervalsCount: 2,
+        interval: {
+          count: 1,
+          unit: "month",
+        },
+        timezone: "UTC",
+        range: [dayjs(), dayjs()],
+        formatter: value => String(value),
+        fromEChartsAxisValue: () => dayjs(),
+        toEChartsAxisValue: val => String(val),
+      };
+
+      it("should replace missing values with zeros based on the x-axis interval", () => {
+        const result = applyVisualizationSettingsDataTransformations(
+          dataset,
+          xAxisModel,
+          [createMockSeriesModel({ dataKey: "series1" })],
+          yAxisScaleTransforms,
+          createMockComputedVisualizationSettings({
+            series: () => ({
+              "line.missing": "zero",
+            }),
+          }),
+        );
+
+        expect(result).toEqual([
+          {
+            [ORIGINAL_INDEX_DATA_KEY]: 0,
+            [X_AXIS_DATA_KEY]: "2020-01-01T00:00:00.000Z",
+            dimensionKey: "A",
+            series1: 10,
+          },
+          { [X_AXIS_DATA_KEY]: "2020-02-01T00:00:00.000Z", series1: 0 },
+          {
+            [ORIGINAL_INDEX_DATA_KEY]: 1,
+            [X_AXIS_DATA_KEY]: "2020-03-01T00:00:00.000Z",
+            dimensionKey: "A",
+            series1: 20,
+          },
+        ]);
+      });
+
+      it("should not replace missing values with zeros when x-axis interval is too big", () => {
+        const result = applyVisualizationSettingsDataTransformations(
+          dataset,
+          { ...xAxisModel, intervalsCount: 10001 },
+          [createMockSeriesModel({ dataKey: "series1" })],
+          yAxisScaleTransforms,
+          createMockComputedVisualizationSettings({
+            series: () => ({
+              "line.missing": "zero",
+            }),
+          }),
+        );
+
+        expect(result).toHaveLength(dataset.length);
+      });
+    });
+
     it("should work on empty datasets", () => {
       const result = applyVisualizationSettingsDataTransformations(
         [],