feat: merge 5949

Signed-off-by: SuZhou-Joe <[email protected]>

CHANGELOG.md

@@ -41,6 +41,12 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Multiple Datasource] Add interfaces to register add-on authentication method from plug-in module ([#5851](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5851))
 - [Multiple Datasource] Able to Hide "Local Cluster" option from datasource DropDown ([#5827](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5827))
 - [Multiple Datasource] Add api registry and allow it to be added into client config in data source plugin ([#5895](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5895))
+- [Multiple Datasource] Concatenate data source name with index pattern name and change delimiter to double colon ([#5907](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5907))
+- [Multiple Datasource] Refactor client and legacy client to use authentication registry ([#5881](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5881))
+- [Multiple Datasource] Improved error handling for the search API when a null value is passed for the dataSourceId ([#5882](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5882))
+- [Multiple Datasource] Hide/Show authentication method in multi data source plugin based on configuration ([#5916](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5916))
+- [[Dynamic Configurations] Add support for dynamic application configurations ([#5855](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5855))
+- [Workspace] Optional workspaces params in repository ([#5949](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5949))
 
 ### 🐛 Bug Fixes
 

config/opensearch_dashboards.yml

@@ -274,4 +274,7 @@
 # opensearchDashboards.survey.url: "https://survey.opensearch.org"
 
 # Set the value of this setting to true to enable plugin augmentation on Dashboard
-# vis_augmenter.pluginAugmentationEnabled: true
+# vis_augmenter.pluginAugmentationEnabled: true
+
+# Set the value to true to enable workspace feature
+# workspace.enabled: false

src/core/server/saved_objects/import/types.ts

@@ -189,7 +189,7 @@ export interface SavedObjectsImportOptions {
   createNewCopies: boolean;
   dataSourceId?: string;
   dataSourceTitle?: string;
-  /** if specified, will import in given workspaces, else will import as global object */
+  /** if specified, will import in given workspaces */
   workspaces?: string[];
 }
 
@@ -214,7 +214,7 @@ export interface SavedObjectsResolveImportErrorsOptions {
   createNewCopies: boolean;
   dataSourceId?: string;
   dataSourceTitle?: string;
-  /** if specified, will import in given workspaces, else will import as global object */
+  /** if specified, will import in given workspaces */
   workspaces?: string[];
 }
 

src/core/server/saved_objects/migrations/core/build_active_mappings.test.ts

@@ -30,6 +30,7 @@
 
 import { IndexMapping, SavedObjectsTypeMappingDefinitions } from './../../mappings';
 import { buildActiveMappings, diffMappings } from './build_active_mappings';
+import { configMock } from '../../../config/mocks';
 
 describe('buildActiveMappings', () => {
   test('creates a strict mapping', () => {
@@ -91,6 +92,12 @@ describe('buildActiveMappings', () => {
     expect(hashes.aaa).toEqual(hashes.bbb);
     expect(hashes.aaa).not.toEqual(hashes.ccc);
   });
+
+  test('workspaces field is added when workspace feature flag is enabled', () => {
+    const rawConfig = configMock.create();
+    rawConfig.get.mockReturnValue(true);
+    expect(buildActiveMappings({}, rawConfig)).toHaveProperty('properties.workspaces');
+  });
 });
 
 describe('diffMappings', () => {

src/core/server/saved_objects/migrations/core/build_active_mappings.ts

@@ -34,6 +34,7 @@
 
 import crypto from 'crypto';
 import { cloneDeep, mapValues } from 'lodash';
+import { Config } from 'packages/osd-config/target';
 import {
   IndexMapping,
   SavedObjectsFieldMapping,
@@ -48,11 +49,20 @@ import {
  * @param typeDefinitions - the type definitions to build mapping from.
  */
 export function buildActiveMappings(
-  typeDefinitions: SavedObjectsTypeMappingDefinitions | SavedObjectsMappingProperties
+  typeDefinitions: SavedObjectsTypeMappingDefinitions | SavedObjectsMappingProperties,
+  opensearchDashboardsRawConfig?: Config
 ): IndexMapping {
   const mapping = defaultMapping();
 
-  const mergedProperties = validateAndMerge(mapping.properties, typeDefinitions);
+  let mergedProperties = validateAndMerge(mapping.properties, typeDefinitions);
+  // if permission control for saved objects is enabled, the permissions field should be added to the mapping
+  if (opensearchDashboardsRawConfig?.get('workspace.enabled')) {
+    mergedProperties = validateAndMerge(mapping.properties, {
+      workspaces: {
+        type: 'keyword',
+      },
+    });
+  }
 
   return cloneDeep({
     ...mapping,

src/core/server/saved_objects/migrations/core/index_migrator.test.ts

@@ -36,6 +36,7 @@ import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry';
 import { IndexMigrator } from './index_migrator';
 import { MigrationOpts } from './migration_context';
 import { loggingSystemMock } from '../../../logging/logging_system.mock';
+import { configMock } from '../../../config/mocks';
 
 describe('IndexMigrator', () => {
   let testOpts: jest.Mocked<MigrationOpts> & {
@@ -59,6 +60,60 @@ describe('IndexMigrator', () => {
     };
   });
 
+  test('creates the index when workspaces feature flag is enabled', async () => {
+    const { client } = testOpts;
+
+    testOpts.mappingProperties = { foo: { type: 'long' } as any };
+    const rawConfig = configMock.create();
+    rawConfig.get.mockReturnValue(true);
+    testOpts.opensearchDashboardsRawConfig = rawConfig;
+
+    withIndex(client, { index: { statusCode: 404 }, alias: { statusCode: 404 } });
+
+    await new IndexMigrator(testOpts).migrate();
+
+    expect(client.indices.create).toHaveBeenCalledWith({
+      body: {
+        mappings: {
+          dynamic: 'strict',
+          _meta: {
+            migrationMappingPropertyHashes: {
+              foo: '18c78c995965207ed3f6e7fc5c6e55fe',
+              migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+              namespace: '2f4316de49999235636386fe51dc06c1',
+              namespaces: '2f4316de49999235636386fe51dc06c1',
+              originId: '2f4316de49999235636386fe51dc06c1',
+              references: '7997cf5a56cc02bdc9c93361bde732b0',
+              type: '2f4316de49999235636386fe51dc06c1',
+              updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+              workspaces: '2f4316de49999235636386fe51dc06c1',
+            },
+          },
+          properties: {
+            foo: { type: 'long' },
+            migrationVersion: { dynamic: 'true', type: 'object' },
+            namespace: { type: 'keyword' },
+            namespaces: { type: 'keyword' },
+            originId: { type: 'keyword' },
+            type: { type: 'keyword' },
+            updated_at: { type: 'date' },
+            references: {
+              type: 'nested',
+              properties: {
+                name: { type: 'keyword' },
+                type: { type: 'keyword' },
+                id: { type: 'keyword' },
+              },
+            },
+            workspaces: { type: 'keyword' },
+          },
+        },
+        settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+      },
+      index: '.kibana_1',
+    });
+  });
+
   test('creates the index if it does not exist', async () => {
     const { client } = testOpts;
 

src/core/server/saved_objects/migrations/core/migration_context.ts

@@ -36,6 +36,7 @@
  */
 
 import { Logger } from 'src/core/server/logging';
+import { Config } from 'packages/osd-config/target';
 import { MigrationOpenSearchClient } from './migration_opensearch_client';
 import { SavedObjectsSerializer } from '../../serialization';
 import {
@@ -65,6 +66,7 @@ export interface MigrationOpts {
    * prior to running migrations. For example: 'opensearch_dashboards_index_template*'
    */
   obsoleteIndexTemplatePattern?: string;
+  opensearchDashboardsRawConfig?: Config;
 }
 
 /**
@@ -90,10 +92,15 @@ export interface Context {
  * and various info needed to migrate the source index.
  */
 export async function migrationContext(opts: MigrationOpts): Promise<Context> {
-  const { log, client } = opts;
+  const { log, client, opensearchDashboardsRawConfig } = opts;
   const alias = opts.index;
   const source = createSourceContext(await Index.fetchInfo(client, alias), alias);
-  const dest = createDestContext(source, alias, opts.mappingProperties);
+  const dest = createDestContext(
+    source,
+    alias,
+    opts.mappingProperties,
+    opensearchDashboardsRawConfig
+  );
 
   return {
     client,
@@ -125,10 +132,11 @@ function createSourceContext(source: Index.FullIndexInfo, alias: string) {
 function createDestContext(
   source: Index.FullIndexInfo,
   alias: string,
-  typeMappingDefinitions: SavedObjectsTypeMappingDefinitions
+  typeMappingDefinitions: SavedObjectsTypeMappingDefinitions,
+  opensearchDashboardsRawConfig?: Config
 ): Index.FullIndexInfo {
   const targetMappings = disableUnknownTypeMappingFields(
-    buildActiveMappings(typeMappingDefinitions),
+    buildActiveMappings(typeMappingDefinitions, opensearchDashboardsRawConfig),
     source.mappings
   );
 

src/core/server/saved_objects/migrations/opensearch_dashboards/opensearch_dashboards_migrator.test.ts

@@ -37,6 +37,7 @@ import {
 import { loggingSystemMock } from '../../../logging/logging_system.mock';
 import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry';
 import { SavedObjectsType } from '../../types';
+import { configMock } from '../../../config/mocks';
 
 const createRegistry = (types: Array<Partial<SavedObjectsType>>) => {
   const registry = new SavedObjectTypeRegistry();
@@ -76,6 +77,12 @@ describe('OpenSearchDashboardsMigrator', () => {
       const mappings = new OpenSearchDashboardsMigrator(options).getActiveMappings();
       expect(mappings).toMatchSnapshot();
     });
+
+    it('workspaces field exists in the mappings when the feature is enabled', () => {
+      const options = mockOptions(true);
+      const mappings = new OpenSearchDashboardsMigrator(options).getActiveMappings();
+      expect(mappings).toHaveProperty('properties.workspaces');
+    });
   });
 
   describe('runMigrations', () => {
@@ -146,7 +153,12 @@ type MockedOptions = OpenSearchDashboardsMigratorOptions & {
   client: ReturnType<typeof opensearchClientMock.createOpenSearchClient>;
 };
 
-const mockOptions = () => {
+const mockOptions = (isWorkspaceEnabled?: boolean) => {
+  const rawConfig = configMock.create();
+  rawConfig.get.mockReturnValue(false);
+  if (isWorkspaceEnabled) {
+    rawConfig.get.mockReturnValue(true);
+  }
   const options: MockedOptions = {
     logger: loggingSystemMock.create().get(),
     opensearchDashboardsVersion: '8.2.3',
@@ -186,6 +198,7 @@ const mockOptions = () => {
       skip: false,
     },
     client: opensearchClientMock.createOpenSearchClient(),
+    opensearchDashboardsRawConfig: rawConfig,
   };
   return options;
 };

src/core/server/saved_objects/migrations/opensearch_dashboards/opensearch_dashboards_migrator.ts

@@ -36,6 +36,7 @@
 import { OpenSearchDashboardsConfigType } from 'src/core/server/opensearch_dashboards_config';
 import { BehaviorSubject } from 'rxjs';
 
+import { Config } from 'packages/osd-config/target';
 import { Logger } from '../../../logging';
 import { IndexMapping, SavedObjectsTypeMappingDefinitions } from '../../mappings';
 import { SavedObjectUnsanitizedDoc, SavedObjectsSerializer } from '../../serialization';
@@ -54,6 +55,7 @@ export interface OpenSearchDashboardsMigratorOptions {
   opensearchDashboardsConfig: OpenSearchDashboardsConfigType;
   opensearchDashboardsVersion: string;
   logger: Logger;
+  opensearchDashboardsRawConfig: Config;
 }
 
 export type IOpenSearchDashboardsMigrator = Pick<
@@ -83,6 +85,7 @@ export class OpenSearchDashboardsMigrator {
     status: 'waiting',
   });
   private readonly activeMappings: IndexMapping;
+  private readonly opensearchDashboardsRawConfig: Config;
 
   /**
    * Creates an instance of OpenSearchDashboardsMigrator.
@@ -94,6 +97,7 @@ export class OpenSearchDashboardsMigrator {
     savedObjectsConfig,
     opensearchDashboardsVersion,
     logger,
+    opensearchDashboardsRawConfig,
   }: OpenSearchDashboardsMigratorOptions) {
     this.client = client;
     this.opensearchDashboardsConfig = opensearchDashboardsConfig;
@@ -107,9 +111,13 @@ export class OpenSearchDashboardsMigrator {
       typeRegistry,
       log: this.log,
     });
+    this.opensearchDashboardsRawConfig = opensearchDashboardsRawConfig;
     // Building the active mappings (and associated md5sums) is an expensive
     // operation so we cache the result
-    this.activeMappings = buildActiveMappings(this.mappingProperties);
+    this.activeMappings = buildActiveMappings(
+      this.mappingProperties,
+      this.opensearchDashboardsRawConfig
+    );
   }
 
   /**
@@ -181,6 +189,7 @@ export class OpenSearchDashboardsMigrator {
             ? 'opensearch_dashboards_index_template*'
             : undefined,
         convertToAliasScript: indexMap[index].script,
+        opensearchDashboardsRawConfig: this.opensearchDashboardsRawConfig,
       });
     });
 

src/core/server/saved_objects/routes/bulk_create.ts

@@ -65,10 +65,9 @@ export const registerBulkCreateRoute = (router: IRouter) => {
     },
     router.handleLegacyErrors(async (context, req, res) => {
       const { overwrite } = req.query;
-      let workspaces = req.query.workspaces;
-      if (typeof workspaces === 'string') {
-        workspaces = [workspaces];
-      }
+      const workspaces = req.query.workspaces
+        ? Array<string>().concat(req.query.workspaces)
+        : undefined;
       const result = await context.core.savedObjects.client.bulkCreate(req.body, {
         overwrite,
         workspaces,

src/core/server/saved_objects/routes/find.ts

@@ -70,10 +70,7 @@ export const registerFindRoute = (router: IRouter) => {
 
       const namespaces =
         typeof req.query.namespaces === 'string' ? [req.query.namespaces] : req.query.namespaces;
-      let workspaces = req.query.workspaces;
-      if (typeof workspaces === 'string') {
-        workspaces = [workspaces];
-      }
+      const workspaces = query.workspaces ? Array<string>().concat(query.workspaces) : undefined;
 
       const result = await context.core.savedObjects.client.find({
         perPage: query.per_page,

src/core/server/saved_objects/routes/resolve_import_errors.ts

@@ -132,6 +132,7 @@ export const registerResolveImportErrorsRoute = (router: IRouter, config: SavedO
         retries: req.body.retries,
         objectLimit: maxImportExportSize,
         createNewCopies: req.query.createNewCopies,
+        workspaces,
         dataSourceId,
         dataSourceTitle,
         workspaces,

src/core/server/saved_objects/saved_objects_service.ts

@@ -67,6 +67,7 @@ import { registerRoutes } from './routes';
 import { ServiceStatus, ServiceStatusLevels } from '../status';
 import { calculateStatus$ } from './status';
 import { createMigrationOpenSearchClient } from './migrations/core/';
+import { Config } from '../config';
 /**
  * Saved Objects is OpenSearchDashboards's data persistence mechanism allowing plugins to
  * use OpenSearch for storing and querying state. The SavedObjectsServiceSetup API exposes methods
@@ -315,6 +316,8 @@ export class SavedObjectsService
     summary: `waiting`,
   });
 
+  private opensearchDashboardsRawConfig?: Config;
+
   constructor(private readonly coreContext: CoreContext) {
     this.logger = coreContext.logger.get('savedobjects-service');
   }
@@ -332,6 +335,10 @@ export class SavedObjectsService
       .atPath<SavedObjectsMigrationConfigType>('migrations')
       .pipe(first())
       .toPromise();
+    this.opensearchDashboardsRawConfig = await this.coreContext.configService
+      .getConfig$()
+      .pipe(first())
+      .toPromise();
     this.config = new SavedObjectConfig(savedObjectsConfig, savedObjectsMigrationConfig);
 
     registerRoutes({
@@ -557,6 +564,7 @@ export class SavedObjectsService
         this.logger,
         migrationsRetryDelay
       ),
+      opensearchDashboardsRawConfig: this.opensearchDashboardsRawConfig as Config,
     });
   }
 }