Add observations to SimulationTimeSeriesMatrix (#462)

jorgenherje · web-flow · commit 25a23a62bd18 · 2023-11-29T13:47:27.000+01:00
diff --git a/backend/src/services/sumo_access/observation_access.py b/backend/src/services/sumo_access/observation_access.py
@@ -1,5 +1,5 @@
 import logging
-from typing import List
+from typing import List, Dict
 
 import json
 from fmu.sumo.explorer.objects.dictionary import Dictionary
@@ -43,18 +43,34 @@ def _create_summary_observations(observations_dict: dict) -> List[SummaryVectorO
     summary_observations: List[SummaryVectorObservations] = []
     if ObservationType.SUMMARY not in observations_dict:
         return summary_observations
-    for smry_obs in observations_dict[ObservationType.SUMMARY]:
+
+    summary_observations_dict: Dict[str, Dict[str, list]] = observations_dict[ObservationType.SUMMARY]
+
+    for vector_name, observations_data in summary_observations_dict.items():
+        observation_names = observations_data["observation_name"]
+        observation_values = observations_data["value"]
+        observation_errors = observations_data["error"]
+        observation_dates = observations_data["date"]
+
+        num_observations = len(observation_names)
+        if (
+            len(observation_values) != num_observations
+            or len(observation_errors) != num_observations
+            or len(observation_dates) != num_observations
+        ):
+            raise ValueError(f"Inconsistent observations data for vector {vector_name}")
+
         summary_observations.append(
             SummaryVectorObservations(
-                vector_name=smry_obs["key"],
+                vector_name=vector_name,
                 observations=[
                     SummaryVectorDateObservation(
-                        date=obs["date"],
-                        value=obs["value"],
-                        error=obs["error"],
-                        label=obs["label"],
+                        date=observation_dates[i],
+                        value=observation_values[i],
+                        error=observation_errors[i],
+                        label=observation_names[i],
                     )
-                    for obs in smry_obs["observations"]
+                    for i in range(num_observations)
                 ],
             )
         )
diff --git a/backend/src/services/sumo_access/observation_types.py b/backend/src/services/sumo_access/observation_types.py
@@ -7,7 +7,7 @@
 class ObservationType(str, Enum):
     """The observation file in Sumo is a dictionary with these datatypes as keys."""
 
-    SUMMARY = "smry"
+    SUMMARY = "summary"
     RFT = "rft"
 
 
diff --git a/frontend/src/modules/SimulationTimeSeriesMatrix/queryHooks.tsx b/frontend/src/modules/SimulationTimeSeriesMatrix/queryHooks.tsx
@@ -1,4 +1,4 @@
-import { Frequency_api, VectorDescription_api } from "@api";
+import { Frequency_api, SummaryVectorObservations_api, VectorDescription_api } from "@api";
 import { VectorHistoricalData_api, VectorRealizationData_api, VectorStatisticData_api } from "@api";
 import { apiService } from "@framework/ApiService";
 import { EnsembleIdent } from "@framework/EnsembleIdent";
@@ -12,7 +12,6 @@ const CACHE_TIME = 60 * 1000;
 export function useVectorListQueries(
     caseUuidsAndEnsembleNames: EnsembleIdent[] | null
 ): UseQueryResult<VectorDescription_api[]>[] {
-    // Note: how to cancel queryFn if key is updated?
     return useQueries({
         queries: (caseUuidsAndEnsembleNames ?? []).map((item) => {
             return {
@@ -33,7 +32,6 @@ export function useVectorDataQueries(
     realizationsToInclude: number[] | null,
     allowEnable: boolean
 ): UseQueryResult<VectorRealizationData_api[]>[] {
-    // Note: how to cancel queryFn if key is updated?
     return useQueries({
         queries: (vectorSpecifications ?? []).map((item) => {
             return {
@@ -141,3 +139,90 @@ export function useHistoricalVectorDataQueries(
         }),
     });
 }
+
+/**
+ * Definition of ensemble vector observation data
+ *
+ * hasSummaryObservations: true if the ensemble has observations, i.e the summary observations array is not empty
+ * vectorsObservationData: array of vector observation data for requested vector specifications
+ */
+export type EnsembleVectorObservationData = {
+    hasSummaryObservations: boolean;
+    vectorsObservationData: { vectorSpecification: VectorSpec; data: SummaryVectorObservations_api }[];
+};
+
+/**
+ * Definition of map of ensemble ident and ensemble vector observation data
+ */
+export type EnsembleVectorObservationDataMap = Map<EnsembleIdent, EnsembleVectorObservationData>;
+
+/**
+ * Definition of vector observations queries result for combined queries
+ */
+export type VectorObservationsQueriesResult = {
+    isFetching: boolean;
+    isError: boolean;
+    ensembleVectorObservationDataMap: EnsembleVectorObservationDataMap;
+};
+
+/**
+ * This function takes vectorSpecifications and returns a map of ensembleIdent and the respective observations data for
+ * the selected vectors.
+ *
+ * If the returned summary array from back-end is empty array, the ensemble does not have observations.
+ * If the selected vectors are not among the returned summary array, the vector does not have observations.
+ */
+export function useVectorObservationsQueries(
+    vectorSpecifications: VectorSpec[] | null,
+    allowEnable: boolean
+): VectorObservationsQueriesResult {
+    const uniqueEnsembleIdents = [...new Set(vectorSpecifications?.map((item) => item.ensembleIdent) ?? [])];
+    return useQueries({
+        queries: (uniqueEnsembleIdents ?? []).map((item) => {
+            return {
+                queryKey: ["getObservations", item.getCaseUuid(), item.getEnsembleName()],
+                queryFn: () =>
+                    apiService.observations.getObservations(item.getCaseUuid() ?? "", item.getEnsembleName() ?? ""),
+                staleTime: STALE_TIME,
+                cacheTime: CACHE_TIME,
+                enabled: !!(allowEnable && item.getCaseUuid() && item.getEnsembleName()),
+            };
+        }),
+        combine: (results) => {
+            const combinedResult: EnsembleVectorObservationDataMap = new Map();
+            if (!vectorSpecifications)
+                return { isFetching: false, isError: false, ensembleVectorObservationDataMap: combinedResult };
+
+            results.forEach((result, index) => {
+                const ensembleIdent = uniqueEnsembleIdents.at(index);
+                if (!ensembleIdent) return;
+
+                const ensembleVectorSpecifications = vectorSpecifications.filter(
+                    (item) => item.ensembleIdent === ensembleIdent
+                );
+
+                const ensembleHasObservations = result.data?.summary.length !== 0;
+                combinedResult.set(ensembleIdent, {
+                    hasSummaryObservations: ensembleHasObservations,
+                    vectorsObservationData: [],
+                });
+                for (const vectorSpec of ensembleVectorSpecifications) {
+                    const vectorObservationsData =
+                        result.data?.summary.find((elm) => elm.vector_name === vectorSpec.vectorName) ?? null;
+                    if (!vectorObservationsData) continue;
+
+                    combinedResult.get(ensembleIdent)?.vectorsObservationData.push({
+                        vectorSpecification: vectorSpec,
+                        data: vectorObservationsData,
+                    });
+                }
+            });
+
+            return {
+                isFetching: results.some((result) => result.isFetching),
+                isError: results.some((result) => result.isError),
+                ensembleVectorObservationDataMap: combinedResult,
+            };
+        },
+    });
+}
diff --git a/frontend/src/modules/SimulationTimeSeriesMatrix/settings.tsx b/frontend/src/modules/SimulationTimeSeriesMatrix/settings.tsx
@@ -353,12 +353,7 @@ export function settings({ moduleContext, workbenchSession }: ModuleFCProps<Stat
                     disabled={!selectedVectorNamesHasHistorical}
                     onChange={handleShowHistorical}
                 />
-                <Checkbox
-                    label="Show observations - NEED DATA IN SUMO"
-                    checked={showObservations}
-                    disabled={true}
-                    onChange={handleShowObservations}
-                />
+                <Checkbox label="Show observations" checked={showObservations} onChange={handleShowObservations} />
                 <div
                     className={resolveClassNames({
                         "pointer-events-none opacity-80": vectorListQueries.some((query) => query.isLoading),
diff --git a/frontend/src/modules/SimulationTimeSeriesMatrix/utils/PlotlyTraceUtils/createVectorTracesUtils.ts b/frontend/src/modules/SimulationTimeSeriesMatrix/utils/PlotlyTraceUtils/createVectorTracesUtils.ts
@@ -1,5 +1,6 @@
 import {
     StatisticFunction_api,
+    SummaryVectorDateObservation_api,
     VectorHistoricalData_api,
     VectorRealizationData_api,
     VectorStatisticData_api,
@@ -150,6 +151,62 @@ export function createHistoricalVectorTrace({
     };
 }
 
+/**
+    Utility function for creating traces for vector observations
+ */
+export type CreateVectorObservationTraceOptions = {
+    vectorObservations: Array<SummaryVectorDateObservation_api>;
+    color?: string;
+    yaxis?: string;
+    xaxis?: string;
+    legendGroup?: string;
+    showLegend?: boolean;
+    name?: string;
+    type?: "scatter" | "scattergl";
+};
+export function createVectorObservationsTraces({
+    vectorObservations,
+    color = "black",
+    yaxis = "y",
+    xaxis = "x",
+    legendGroup = "Observation",
+    showLegend = false,
+    name = undefined,
+    type = "scatter",
+}: CreateVectorObservationTraceOptions): Partial<TimeSeriesPlotData>[] {
+    // NB: "scattergl" does not include "+/- error" in the hover template `(%{x}, %{y})`, "scatter" does.
+
+    const traceName = name ? `Observation<br>${name}` : "Observation";
+    return vectorObservations.map((observation) => {
+        let hoverText = observation.label;
+        let hoverData = `(%{x}, %{y})<br>`;
+        if (observation.comment) {
+            hoverText += `: ${observation.comment}`;
+        }
+        if (type === "scattergl") {
+            hoverData = `(%{x}, %{y} ± ${observation.error})<br>`;
+        }
+
+        return {
+            name: traceName,
+            legendgroup: legendGroup,
+            x: [observation.date],
+            y: [observation.value],
+            marker: { color: color },
+            yaxis: yaxis,
+            xaxis: xaxis,
+            hovertemplate: hoverText ? `${hoverData}${hoverText}` : hoverData,
+            showlegend: showLegend,
+            type: type,
+            error_y: {
+                type: "constant",
+                value: observation.error,
+                visible: true,
+            },
+        };
+    });
+}
+
 /**
     Utility function for creating traces representing statistical fanchart for given statistics data.
 
diff --git a/frontend/src/modules/SimulationTimeSeriesMatrix/utils/subplotBuilder.ts b/frontend/src/modules/SimulationTimeSeriesMatrix/utils/subplotBuilder.ts
@@ -1,4 +1,9 @@
-import { VectorHistoricalData_api, VectorRealizationData_api, VectorStatisticData_api } from "@api";
+import {
+    SummaryVectorObservations_api,
+    VectorHistoricalData_api,
+    VectorRealizationData_api,
+    VectorStatisticData_api,
+} from "@api";
 import { EnsembleIdent } from "@framework/EnsembleIdent";
 import { ColorSet } from "@lib/utils/ColorSet";
 import { simulationUnitReformat, simulationVectorDescription } from "@modules/_shared/reservoirSimulationStringUtils";
@@ -9,6 +14,7 @@ import { Annotations, Layout } from "plotly.js";
 import {
     createHistoricalVectorTrace,
     createVectorFanchartTraces,
+    createVectorObservationsTraces,
     createVectorRealizationTrace,
     createVectorRealizationTraces,
     createVectorStatisticsTraces,
@@ -466,8 +472,36 @@ export class SubplotBuilder {
         }
     }
 
-    addVectorObservations(): void {
-        throw new Error("Method not implemented.");
+    addObservationsTraces(
+        vectorsObservationData: {
+            vectorSpecification: VectorSpec;
+            data: SummaryVectorObservations_api;
+        }[]
+    ): void {
+        // Only allow selected vectors
+        const selectedVectorsObservationData = vectorsObservationData.filter((vec) =>
+            this._selectedVectorSpecifications.some(
+                (selectedVec) => selectedVec.vectorName === vec.vectorSpecification.vectorName
+            )
+        );
+
+        // Create traces for each vector
+        for (const elm of selectedVectorsObservationData) {
+            const subplotIndex = this.getSubplotIndex(elm.vectorSpecification);
+            if (subplotIndex === -1) continue;
+
+            const name = this.makeTraceNameFromVectorSpecification(elm.vectorSpecification);
+            const vectorObservationsTraces = createVectorObservationsTraces({
+                vectorObservations: elm.data.observations,
+                name: name,
+                color: this._observationColor,
+                type: this._scatterType,
+                yaxis: `y${subplotIndex + 1}`,
+            });
+
+            this._plotData.push(...vectorObservationsTraces);
+            this._hasObservationTraces = true;
+        }
     }
 
     private getSubplotIndex(vectorSpecification: VectorSpec) {
diff --git a/frontend/src/modules/SimulationTimeSeriesMatrix/utils/vectorSpecificationsAndQueriesUtils.ts b/frontend/src/modules/SimulationTimeSeriesMatrix/utils/vectorSpecificationsAndQueriesUtils.ts
@@ -11,7 +11,7 @@ import { FanchartStatisticOption, VectorSpec } from "../state";
  */
 export function createLoadedVectorSpecificationAndDataArray<T>(
     vectorSpecifications: VectorSpec[],
-    queryResults: UseQueryResult<T>[]
+    queryResults: UseQueryResult<T | null>[]
 ): { vectorSpecification: VectorSpec; data: T }[] {
     if (vectorSpecifications.length !== queryResults.length) {
         throw new Error(
diff --git a/frontend/src/modules/SimulationTimeSeriesMatrix/view.tsx b/frontend/src/modules/SimulationTimeSeriesMatrix/view.tsx
