[Discover-Next] Adding datasources support for dataframes

changelogs/fragments/7106.yml

@@ -0,0 +1,2 @@
+feat:
+- Onboard dataframes support to MDS and create dataframe before request ([#7106](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7106))

package.json

@@ -65,7 +65,7 @@
     "start": "scripts/use_node scripts/opensearch_dashboards --dev",
     "start:docker": "scripts/use_node scripts/opensearch_dashboards --dev --opensearch.hosts=$OPENSEARCH_HOSTS --opensearch.ignoreVersionMismatch=true --server.host=$SERVER_HOST",
     "start:security": "scripts/use_node scripts/opensearch_dashboards --dev --security",
-    "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --data.enhancements.enabled=true --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['query:dataSource:readOnly']=false",
+    "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --data.enhancements.enabled=true --data_source.enabled=true --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['query:dataSource:readOnly']=false",
     "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev",
     "debug-break": "scripts/use_node --nolazy --inspect-brk scripts/opensearch_dashboards --dev",
     "lint": "yarn run lint:es && yarn run lint:style",

src/plugins/data/common/data_frames/utils.ts

@@ -45,6 +45,29 @@
   );
 };
 
+/**
+ * Parses a raw query string and extracts the query string and data source.
+ * @param rawQueryString - The raw query string to parse.
+ * @returns An object containing the parsed query string and data source (if found).
+ */
+export const parseRawQueryString = (rawQueryString: string) => {
+  const rawDataSource = rawQueryString.match(/::(.*?)::/);
+  return {
+    qs: rawQueryString.replace(/::.*?::/, ''),
+    formattedQs(key: string = '.'): string {
+      const parts = rawQueryString.split('::');
+      if (parts.length > 1) {
+        return (parts.slice(0, 1).join('') + parts.slice(1).join(key)).replace(
+          new RegExp(key + '$'),
+          ''
+        );
+      }
+      return rawQueryString;
+    },
+    ...(rawDataSource && { dataSource: rawDataSource[1] }),
+  };
+};
+
 /**
  * Returns the raw aggregations from the search request.
  *
@@ -379,6 +402,7 @@
   getAggQsFn: GetDataFrameAggQsFn;
 }) => {
   dataFrame.meta = {
+    ...dataFrame.meta,
     aggs: aggConfig,
     aggsQs: {
       [aggConfig.id]: getAggQsFn({

src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts

@@ -141,6 +141,25 @@
     return await this.savedObjectsClient.get<DataSourceAttributes>('data-source', id);
   };
 
+  /**
+   * Finds a data source by its title.
+   *
+   * @param title - The title of the data source to find.
+   * @param size - The number of results to return. Defaults to 10.
+   * @returns The first matching data source or undefined if not found.
+   */
+  findDataSourceByTitle = async (title: string, size: number = 10) => {
+    const savedObjectsResponse = await this.savedObjectsClient.find<DataSourceAttributes>({
+      type: 'data-source',
+      fields: ['title'],
+      search: title,
+      searchFields: ['title'],
+      perPage: size,
+    });
+
+    return savedObjectsResponse[0] || undefined;
+  };
+
   /**
    * Get list of index pattern ids
    * @param refresh Force refresh of index pattern list

src/plugins/data/common/search/search_source/search_source.ts

@@ -87,7 +87,15 @@
 import { filterDocvalueFields } from './filter_docvalue_fields';
 import { fieldWildcardFilter } from '../../../../opensearch_dashboards_utils/common';
 import { IIndexPattern } from '../../index_patterns';
-import { DATA_FRAME_TYPES, IDataFrame, IDataFrameResponse, convertResult } from '../../data_frames';
+import {
+  DATA_FRAME_TYPES,
+  IDataFrame,
+  IDataFrameResponse,
+  convertResult,
+  createDataFrame,
+  getRawQueryString,
+  parseRawQueryString,
+} from '../../data_frames';
 import { IOpenSearchSearchRequest, IOpenSearchSearchResponse, ISearchOptions } from '../..';
 import { IOpenSearchDashboardsSearchRequest, IOpenSearchDashboardsSearchResponse } from '../types';
 import { ISearchSource, SearchSourceOptions, SearchSourceFields } from './types';
@@ -305,6 +313,23 @@
     return this.getDataFrame();
   }
 
+  /**
+   * Create and set the data frame of this SearchSource
+   *
+   * @async
+   * @return {undefined|IDataFrame}
+   */
+  async createDataFrame(searchRequest: SearchRequest) {
+    const rawQueryString = this.getRawQueryStringFromRequest(searchRequest);
+    const dataFrame = createDataFrame({
+      name: searchRequest.index.title || searchRequest.index,
+      fields: [],
+      ...(rawQueryString && { meta: { queryConfig: parseRawQueryString(rawQueryString) } }),
+    });
+    await this.setDataFrame(dataFrame);
+    return this.getDataFrame();
+  }
+
   /**
    * Clear the data frame of this SearchSource
    */
@@ -401,6 +426,10 @@
   private async fetchExternalSearch(searchRequest: SearchRequest, options: ISearchOptions) {
     const { search, getConfig, onResponse } = this.dependencies;
 
+    if (!this.getDataFrame()) {
+      await this.createDataFrame(searchRequest);
+    }
+
     const params = getExternalSearchParamsFromRequest(searchRequest, {
       getConfig,
       getDataFrame: this.getDataFrame.bind(this),
@@ -444,6 +473,10 @@
     return request.body!.query.hasOwnProperty('type') && request.body!.query.type === 'unsupported';
   }
 
+  private getRawQueryStringFromRequest(request: SearchRequest): string | undefined {
+    return getRawQueryString({ params: request });
+  }
+
   /**
    *  Called by requests of this search source when they are started
    *  @param options

src/plugins/data/public/search/search_service.ts

@@ -145,6 +145,16 @@
         if (this.dfCache.get() && this.dfCache.get()?.name !== dataFrame.name) {
           indexPatterns.clearCache(this.dfCache.get()!.name, false);
         }
+        if (
+          dataFrame.meta &&
+          dataFrame.meta.queryConfig &&
+          'dataSource' in dataFrame.meta.queryConfig
+        ) {
+          const dataSource = await indexPatterns.findDataSourceByTitle(
+            dataFrame.meta.queryConfig.dataSource
+          );
+          dataFrame.meta.queryConfig.dataSourceId = dataSource?.id;
+        }
         this.dfCache.set(dataFrame);
         const existingIndexPattern = indexPatterns.getByTitle(dataFrame.name!, true);
         const dataSet = await indexPatterns.create(

src/plugins/data/server/search/search_service.ts

@@ -218,6 +218,16 @@
               if (this.dfCache.get() && this.dfCache.get()?.name !== dataFrame.name) {
                 scopedIndexPatterns.clearCache(this.dfCache.get()!.name, false);
               }
+              if (
+                dataFrame.meta &&
+                dataFrame.meta.queryConfig &&
+                'dataSource' in dataFrame.meta.queryConfig
+              ) {
+                const dataSource = await scopedIndexPatterns.findDataSourceByTitle(
+                  dataFrame.meta.queryConfig.dataSource
+                );
+                dataFrame.meta.queryConfig.dataSourceId = dataSource?.id;
+              }
               this.dfCache.set(dataFrame);
               const existingIndexPattern = scopedIndexPatterns.getByTitle(dataFrame.name!, true);
               const dataSet = await scopedIndexPatterns.create(