From 30d13dc1839674ba7d30196039154453eca2c654 Mon Sep 17 00:00:00 2001 From: Kavitha Conjeevaram Mohan Date: Tue, 28 Nov 2023 10:02:29 -0800 Subject: [PATCH 01/20] List all available otel docs Signed-off-by: Kavitha Conjeevaram Mohan --- common/constants/metrics.ts | 26 ++++ common/constants/shared.ts | 2 + .../metrics/helpers/sample_data.tsx | 137 ++++++++++++++++++ public/components/metrics/index.tsx | 41 +++++- .../metrics/redux/slices/metrics_slice.ts | 81 ++++++++++- .../metrics/sidebar/data_source_picker.tsx | 43 ++++++ .../metrics/sidebar/index_picker.tsx | 45 ++++++ .../metrics/sidebar/metric_name.tsx | 3 +- .../metrics/sidebar/metrics_accordion.tsx | 4 +- public/components/metrics/sidebar/sidebar.tsx | 132 ++++++++++++++++- .../visualizations/charts/bar/bar.tsx | 2 +- .../charts/histogram/histogram.tsx | 28 +++- server/routes/metrics/metrics_rounter.ts | 63 +++++++- 13 files changed, 591 insertions(+), 16 deletions(-) create mode 100644 public/components/metrics/helpers/sample_data.tsx create mode 100644 public/components/metrics/sidebar/data_source_picker.tsx create mode 100644 public/components/metrics/sidebar/index_picker.tsx diff --git a/common/constants/metrics.ts b/common/constants/metrics.ts index 7e92e6663b..615e6b0e15 100644 --- a/common/constants/metrics.ts +++ b/common/constants/metrics.ts @@ -33,3 +33,29 @@ export const AGGREGATION_OPTIONS = [ { value: 'min', text: 'min()' }, { value: 'max', text: 'max()' }, ]; + +export const DATASOURCE_OPTIONS = [ + { + label: 'Prometheus', + 'data-test-subj': 'prometheusOption', + }, + { + label: 'OpenTelemetry', + 'data-test-subj': 'openTelemetryOption', + }, +]; +export const DATA_PREPPER_INDEX_NAME = 'ss4o-metrics-*-*'; +export const METRICS_ANALYTICS_DATA_PREPPER_INDICES_ROUTE = + '/api/observability/metrics_analytics/data_prepper_indices'; + +export const DOCUMENT_NAMES_QUERY = { + size: 0, + aggs: { + distinct_names: { + terms: { + field: 'name.keyword', + size: 500, + }, + }, + }, +}; diff --git a/common/constants/shared.ts b/common/constants/shared.ts index 9baffcf101..68d3229b8c 100644 --- a/common/constants/shared.ts +++ b/common/constants/shared.ts @@ -131,6 +131,7 @@ export enum VIS_CHART_TYPES { Pie = 'pie', HeatMap = 'heatmap', Text = 'text', + Histogram = 'histogram', } export const NUMERICAL_FIELDS = ['short', 'integer', 'long', 'float', 'double']; @@ -142,6 +143,7 @@ export const ENABLED_VIS_TYPES = [ VIS_CHART_TYPES.Pie, VIS_CHART_TYPES.HeatMap, VIS_CHART_TYPES.Text, + VIS_CHART_TYPES.Histogram, ]; // Live tail constants diff --git a/public/components/metrics/helpers/sample_data.tsx b/public/components/metrics/helpers/sample_data.tsx new file mode 100644 index 0000000000..e9aa989aef --- /dev/null +++ b/public/components/metrics/helpers/sample_data.tsx @@ -0,0 +1,137 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const sampleDataPrepperMetrics = { + data: [ + [ + { + min: -3.4028234663852886e38, + max: 0, + count: 0, + }, + { + min: 0, + max: 25000000, + count: 5, + }, + { + min: 25000000, + max: 50000000, + count: 5, + }, + { + min: 50000000, + max: 75000000, + count: 4, + }, + { + min: 75000000, + max: 100000000, + count: 4, + }, + { + min: 100000000, + max: 3.4028234663852886e38, + count: 2, + }, + ], + [ + { + min: -3.4028234663852886e38, + max: 0, + count: 0, + }, + { + min: 0, + max: 25000000, + count: 5, + }, + { + min: 25000000, + max: 50000000, + count: 5, + }, + { + min: 50000000, + max: 75000000, + count: 4, + }, + { + min: 75000000, + max: 100000000, + count: 4, + }, + { + min: 100000000, + max: 3.4028234663852886e38, + count: 2, + }, + ], + [ + { + min: -3.4028234663852886e38, + max: 0, + count: 0, + }, + { + min: 0, + max: 25000000, + count: 5, + }, + { + min: 25000000, + max: 50000000, + count: 5, + }, + { + min: 50000000, + max: 75000000, + count: 4, + }, + { + min: 75000000, + max: 100000000, + count: 4, + }, + { + min: 100000000, + max: 3.4028234663852886e38, + count: 2, + }, + ], + [ + { + min: -3.4028234663852886e38, + max: 0, + count: 0, + }, + { + min: 0, + max: 25000000, + count: 5, + }, + { + min: 25000000, + max: 50000000, + count: 5, + }, + { + min: 50000000, + max: 75000000, + count: 4, + }, + { + min: 75000000, + max: 100000000, + count: 4, + }, + { + min: 100000000, + max: 3.4028234663852886e38, + count: 2, + }, + ], + ], +}; diff --git a/public/components/metrics/index.tsx b/public/components/metrics/index.tsx index 2e7f7a1cf8..1141f8b1d7 100644 --- a/public/components/metrics/index.tsx +++ b/public/components/metrics/index.tsx @@ -12,6 +12,8 @@ import { Sidebar } from './sidebar/sidebar'; import PPLService from '../../services/requests/ppl'; import { TopMenu } from './top_menu/top_menu'; import { MetricsGrid } from './view/metrics_grid'; +import { metricsLayoutSelector, selectedMetricsSelector } from './redux/slices/metrics_slice'; +import { resolutionOptions, DATASOURCE_OPTIONS } from '../../../common/constants/metrics'; import SavedObjects from '../../services/saved_objects/event_analytics/saved_objects'; interface MetricsProps { @@ -23,6 +25,35 @@ interface MetricsProps { } export const Home = ({ chrome, parentBreadcrumb }: MetricsProps) => { + // Redux tools + const selectedMetrics = useSelector(selectedMetricsSelector); + const metricsLayout = useSelector(metricsLayoutSelector); + + // Top panel + const [IsTopPanelDisabled, setIsTopPanelDisabled] = useState(false); + const [editMode, setEditMode] = useState(false); + const [editActionType, setEditActionType] = useState(''); + const [toasts, setToasts] = useState([]); + const [toastRightSide, setToastRightSide] = useState(true); + + // Metrics constants + const [panelVisualizations, setPanelVisualizations] = useState([]); + + // Side bar constants + const [selectedDataSource, setSelectedDataSource] = useState([]); + const [selectedOTIndex, setSelectedOTIndex] = useState([]); + const [availableTestOtelDocuments, setAvailableTestOtelDocuments] = useState([]); + + const setToast = (title: string, color = 'success', text?: ReactChild, side?: string) => { + if (!text) text = ''; + setToastRightSide(!side); + setToasts([...toasts, { id: new Date().toISOString(), title, text, color } as Toast]); + }; + + const onEditClick = (savedVisualizationId: string) => { + window.location.assign(`${observabilityLogsID}#/explorer/${savedVisualizationId}`); + }; + useEffect(() => { chrome.setBreadcrumbs([ parentBreadcrumb, @@ -49,7 +80,15 @@ export const Home = ({ chrome, parentBreadcrumb }: MetricsProps) => { {(EuiResizablePanel, EuiResizableButton) => ( <> - + diff --git a/public/components/metrics/redux/slices/metrics_slice.ts b/public/components/metrics/redux/slices/metrics_slice.ts index f623585c37..b8a4e6bd44 100644 --- a/public/components/metrics/redux/slices/metrics_slice.ts +++ b/public/components/metrics/redux/slices/metrics_slice.ts @@ -6,18 +6,22 @@ import { createSlice } from '@reduxjs/toolkit'; import { keyBy, mergeWith, pick, sortBy } from 'lodash'; import { ouiPaletteColorBlindBehindText } from '@elastic/eui'; +import { Dispatch, SetStateAction, useEffect, useMemo } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; import { OBSERVABILITY_CUSTOM_METRIC, PPL_DATASOURCES_REQUEST, REDUX_SLICE_METRICS, SAVED_VISUALIZATION, + OBSERVABILITY_CUSTOM_METRIC, + DOCUMENT_NAMES_QUERY, } from '../../../../../common/constants/metrics'; import { MetricType } from '../../../../../common/types/metrics'; import { SavedObjectsActions } from '../../../../services/saved_objects/saved_object_client/saved_objects_actions'; import { ObservabilitySavedVisualization } from '../../../../services/saved_objects/saved_object_client/types'; import { pplServiceRequestor } from '../../helpers/utils'; import { coreRefs } from '../../../../framework/core_refs'; -import { PPL_METRIC_SUBTYPE, PROMQL_METRIC_SUBTYPE } from '../../../../../common/constants/shared'; +import { PPL_METRIC_SUBTYPE, PROMQL_METRIC_SUBTYPE, OBSERVABILITY_BASE } from '../../../../../common/constants/shared'; import { getPPLService } from '../../../../../common/utils'; export interface IconAttributes { @@ -63,6 +67,10 @@ const initialState = { recentlyUsedRanges: [], }, refresh: 0, // set to new Date() to trigger + selectedDataSource: '', + otelIndices: [], + // selectedOtelIndex: [], + otelDocumentNames: [], }; const mergeMetricCustomizer = function (objValue, srcValue) { @@ -87,6 +95,8 @@ export const loadMetrics = () => async (dispatch) => { const customDataRequest = fetchCustomMetrics(); const remoteDataSourcesResponse = await pplServiceRequestor(pplService!, PPL_DATASOURCES_REQUEST); const remoteDataSources = remoteDataSourcesResponse.data.DATASOURCE_NAME; + const fetchOTindices = await fetchOpenTelemetryIndices(); + // const fetchOTDocuments = await fetchOpenTelemetryDocuments(); dispatch(setDataSources(remoteDataSources)); dispatch(setDataSourceTitles(remoteDataSources)); @@ -102,7 +112,21 @@ export const loadMetrics = () => async (dispatch) => { dispatch(mergeMetrics(metricsMapById)); const sortedIds = sortBy(metricsResult, 'catalog', 'id').map((m) => m.id); - dispatch(setSortedIds(sortedIds)); + await dispatch(setSortedIds(sortedIds)); + await dispatch(setOtelIndices(fetchOTindices)); +}; +export const loadOtelDocuments = ( + dispatch: Dispatch, + setAvailableTestOtelDocuments: { (value: SetStateAction | any) }, + selectedOTIndex: string +) => async () => { + // const fetchOTindices = await fetchOpenTelemetryIndices(); + console.log('does it work in loadOtelDoc'); + const fetchOTDocuments = await fetchOpenTelemetryDocuments(selectedOTIndex)(); + // dispatch(setOtelIndices(fetchOTindices)); + console.log('here fetchOTDocuments: ', fetchOTDocuments); + setAvailableTestOtelDocuments(fetchOTDocuments.aggregations); + dispatch(setOtelDocumentNames(fetchOTDocuments.aggregations)); }; const fetchCustomMetrics = async () => { @@ -158,6 +182,35 @@ const fetchRemoteMetrics = (remoteDataSources: string[]) => })) ) ); +}; + +const fetchOpenTelemetryIndices = async () => { + const { http } = coreRefs; + console.log(`Fetching open telemetry indices`); + return http + .get(`${OBSERVABILITY_BASE}/search/indices`, { + query: { + format: 'json', + }, + }) + .catch((error) => console.error(error)); +}; + +export const fetchOpenTelemetryDocuments = (selectedOtelIndex: string) => async () => { + const { http } = coreRefs; + // const otelIndex = useSelector(selectedOtelIndexSelector); + console.log(`Fetching open telemetry indices`); + // const otelIndex = 'ss4o_metrics-sample1-us'; + return http + .get(`${OBSERVABILITY_BASE}/metrics/otel/documents`, { + query: { + format: 'json', + }, + index: selectedOtelIndex, + }) + .catch((error) => console.error(error)); + // dispatch(setOtelDocumentNames(resp.aggregations)); +}; export const metricSlice = createSlice({ name: REDUX_SLICE_METRICS, @@ -213,6 +266,18 @@ export const metricSlice = createSlice({ setRefresh: (state) => { state.refresh = Date.now(); }, + setSelectedDataSource: (state, { payload }) => { + state.selectedDataSource = payload; + }, + setOtelIndices: (state, { payload }) => { + state.otelIndices = payload; + }, + // setSelectedOtelIndex: (state, { payload }) => { + // state.selectedOtelIndex = payload; + // }, + setOtelDocumentNames: (state, { payload }) => { + state.otelDocumentNames = payload; + }, }, }); @@ -227,6 +292,10 @@ export const { setDataSourceTitles, setDataSourceIcons, updateMetric, + setSelectedDataSource, + setOtelIndices, + // setSelectedOtelIndex, + setOtelDocumentNames, } = metricSlice.actions; /** private actions */ @@ -321,4 +390,12 @@ export const metricQuerySelector = (id) => (state) => availableAttributes: [], }; +export const selectedDataSourcesSelector = (state) => state.metrics.selectedDataSource; + +export const otelIndexSelector = (state) => state.metrics.otelIndices; + +// export const selectedOtelIndexSelector = (state) => state.metrics.selectedOtelIndex; + +export const otelDocumentNamesSelector = (state) => state.metrics.otelDocumentNames; + export const metricsReducers = metricSlice.reducer; diff --git a/public/components/metrics/sidebar/data_source_picker.tsx b/public/components/metrics/sidebar/data_source_picker.tsx new file mode 100644 index 0000000000..c361bc31d9 --- /dev/null +++ b/public/components/metrics/sidebar/data_source_picker.tsx @@ -0,0 +1,43 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { EuiComboBox, EuiSearchBar, EuiTitle } from '@elastic/eui'; +import React, { useState } from 'react'; +import { EuiComboBoxOptionOption } from '@opensearch-project/oui'; +import { useDispatch } from 'react-redux'; +import { DATASOURCE_OPTIONS } from '../../../../common/constants/metrics'; +import { setSelectedDataSource as selectedDataSourceSlice } from '../redux/slices/metrics_slice'; + +export const DataSourcePicker = (props) => { + const { selectedDataSource, setSelectedDataSource } = props; + const dispatch = useDispatch(); + // const [selectedDataSource, setSelectedDataSource] = useState(); + + const onChange = ( + // eslint-disable-next-line no-shadow + selectedDataSource: React.SetStateAction> + ) => { + // console.log("maybe here"); + console.log('selectedDataSource in data pciker', selectedDataSource[0]); + setSelectedDataSource(selectedDataSource); + // dispatch(selectedDataSourceSlice(selectedDataSource[0])); + }; + + return ( +
+ +
Data source
+ + +
+ ); +}; diff --git a/public/components/metrics/sidebar/index_picker.tsx b/public/components/metrics/sidebar/index_picker.tsx new file mode 100644 index 0000000000..203ed1fa0e --- /dev/null +++ b/public/components/metrics/sidebar/index_picker.tsx @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { EuiComboBox, EuiSearchBar, EuiTitle } from '@elastic/eui'; +import React, { useEffect, useState } from 'react'; +import { EuiComboBoxOptionOption } from '@opensearch-project/oui'; +import { useDispatch, useSelector } from 'react-redux'; + +export const IndexPicker = (props) => { + const { otelIndices, setSelectedOTIndex } = props; + const dispatch = useDispatch(); + const otelIndex = otelIndices.map((item: any) => { + return { label: item.index }; + }); + // const dataSource = useSelector(selectedDataSourcesSelector); + const [selectedIndex, setSelectedIndex] = useState(); + + const onChange = ( + // eslint-disable-next-line no-shadow + selectedIndex: React.SetStateAction> + ) => { + setSelectedIndex(selectedIndex); + console.log('it does come here: ', selectedIndex); + setSelectedOTIndex(selectedIndex); + // dispatch(setSelectedOtelIndex(selectedIndex)); + }; + + return ( +
+ +
Otel Index
+ + +
+ ); +}; diff --git a/public/components/metrics/sidebar/metric_name.tsx b/public/components/metrics/sidebar/metric_name.tsx index ce25c8f405..7e6a2e3721 100644 --- a/public/components/metrics/sidebar/metric_name.tsx +++ b/public/components/metrics/sidebar/metric_name.tsx @@ -26,7 +26,8 @@ export const MetricName = (props: IMetricNameProps) => { const { metric, handleClick } = props; const name = () => { - if (metric.catalog === 'CUSTOM_METRICS') return metric.name; + if (metric.catalog === 'CUSTOM_METRICS' || metric.catalog === 'OpenTelemetry') + return metric.name; else return metric.name.split('.')[1].replace(/^prometheus_/, 'p.._'); }; diff --git a/public/components/metrics/sidebar/metrics_accordion.tsx b/public/components/metrics/sidebar/metrics_accordion.tsx index d3168e7102..3957bd150a 100644 --- a/public/components/metrics/sidebar/metrics_accordion.tsx +++ b/public/components/metrics/sidebar/metrics_accordion.tsx @@ -9,7 +9,7 @@ import { min } from 'lodash'; import { MetricName } from './metric_name'; interface IMetricNameProps { - metricsList: []; + metricsList: any; headerName: string; handleClick: (props: any) => void; dataTestSubj: string; @@ -17,6 +17,8 @@ interface IMetricNameProps { export const MetricsAccordion = (props: IMetricNameProps) => { const { metricsList, headerName, handleClick, dataTestSubj } = props; + // console.log('mertcs list type: ', typeof metricsList); + // console.log('mertcs list: ', metricsList); return ( >; + setSelectedDataSource: React.Dispatch>; + selectedOTIndex: React.SetStateAction>; + setSelectedOTIndex: React.Dispatch>; + availableTestOtelDocuments: Array<{}>; + setAvailableTestOtelDocuments: React.Dispatch>; + additionalSelectedMetricId?: string; +} export const Sidebar = ({ + selectedDataSource, + setSelectedDataSource, + selectedOTIndex, + setSelectedOTIndex, + availableTestOtelDocuments, + setAvailableTestOtelDocuments, additionalSelectedMetricId, -}: { - additionalSelectedMetricId?: string; -}) => { +}: SideBarMenuProps) => { const dispatch = useDispatch(); + const [availableOTDocuments, setAvailableOTDocuments] = useState([]); + const availableOTDocumentsRef = useRef(); + availableOTDocumentsRef.current = availableOTDocuments; + // const [selectedDataSource, setSelectedDataSource] = useState(); const availableMetrics = useSelector(availableMetricsSelector); const selectedMetrics = useSelector(selectedMetricsSelector); @@ -35,6 +62,15 @@ export const Sidebar = ({ const additionalMetric = useSelector(selectMetricByIdSelector(additionalSelectedMetricId)); + const dataSource = useSelector(selectedDataSourcesSelector); + // let isOpenTelemetry: boolean; + // console.log('dataSourceeee: ', dataSource.label); + // console.log('dataSourceeeeee.label: ', dataSource.label); + const otelIndices = useSelector(otelIndexSelector); + // const selectedOtelIndex = useSelector(selectedOtelIndexSelector); + const otelDocuments = useSelector(otelDocumentNamesSelector); + // console.log('selectedOtelIndex after useSelc: ', selectedOtelIndex); + useEffect(() => { batch(() => { dispatch(loadMetrics()); @@ -54,7 +90,82 @@ export const Sidebar = ({ return selectedMetricsIds.map((id) => selectedMetrics[id]).filter((m) => m); // filter away null entries }, [selectedMetrics, selectedMetricsIds]); - const handleAddMetric = (metric: any) => dispatch(addSelectedMetric(metric)); + // useEffect(() => { + // console.log('hereeeee'); + // if (selectedOtelIndex.length > 0) { + // // getting undefined here due to selectedOtelIndex turning into undefined + // console.log('selectedOtelIndex: ', selectedOtelIndex); + // console.log('selectedOtelIndex label: ', selectedOtelIndex[0]?.label); + // loadOtelDocuments(dispatch, setAvailableTestOtelDocuments, selectedOtelIndex[0]?.label)(); + // // console.log('temp: ', temp); + // console.log('atleasttttt'); + // const availableOtelDocuments = availableTestOtelDocuments?.distinct_names?.buckets.map((item: any) => { + // return { id: item.key }; + // }); + // setAvailableOTDocuments(availableOtelDocuments); + // console.log('otel metrics: ', availableOtelDocuments); + // // console.log('availableOTDocuments: ', availableOTDocumentsRef.current); + // } + // }, [selectedOtelIndex]); + + useEffect(() => { + console.log('hereeeee'); + if (selectedOTIndex.length > 0 && selectedDataSource) { + const fetchOtelDocuments = async () => { + try { + console.log('selectedOtelIndex: ', selectedOTIndex); + console.log('selectedOtelIndex label: ', selectedOTIndex[0]?.label); + const documents = await fetchOpenTelemetryDocuments(selectedOTIndex[0]?.label)(); + console.log('1'); + // setAvailableTestOtelDocuments(documents.aggregations); + console.log('2'); + // dispatch(setOtelDocumentNames(documents.aggregations)); + console.log('3'); + console.log('docs after 3: ', documents); + const availableOtelDocuments = documents?.aggregations?.distinct_names?.buckets.map( + (item: any) => { + return { id: item.key, name: item.key, catalog: 'OpenTelemetry' }; + } + ); + console.log('4'); + console.log('otel metrics after 4: ', availableOtelDocuments); + setAvailableOTDocuments(availableOtelDocuments); + console.log('5'); + console.log('otel metrics: ', availableOtelDocuments); + } catch (error) { + console.error('Error fetching OpenTelemetry documents:', error); + // Handle errors if needed + } + }; + + fetchOtelDocuments(); + } + }, [dispatch, selectedDataSource, selectedOTIndex, setAvailableTestOtelDocuments]); + + const indexPicker = useMemo(() => { + console.log('selectedDataSource: ', selectedDataSource); + console.log('selectedDataSource with label: ', selectedDataSource[0]?.label); + const isOpenTelemetry = selectedDataSource[0]?.label === 'OpenTelemetry' ? true : false; + console.log('isOpenTelemetry: ', isOpenTelemetry); + if (isOpenTelemetry) { + console.log('otelIndices: ', otelIndices); + return ; + } + }, [selectedDataSource]); + + // useEffect(() => { + // if (availableOTDocuments === undefined) setAvailableOTDocuments([]); + // console.log('availableOTDocuments in useEff: ', availableOTDocumentsRef.current); + // }, [availableOTDocuments]); + + // console.log('availableTestOtelDocuments: ', availableTestOtelDocuments); + + console.log('available metrics: ', availableMetrics); + console.log('availableOTDocuments outside: ', availableOTDocumentsRef.current); + + // console.log('otel metrics: ', availableOtelDocuments); + + const handleAddMetric = (metric: any) => dispatch(selectMetric(metric)); const handleRemoveMetric = (metric: any) => { dispatch(removeSelectedMetric(metric)); @@ -63,9 +174,16 @@ export const Sidebar = ({ return (