diff --git a/benchmarks/src/main/java/org/opensearch/benchmark/gateway/remote/RemoteClusterStateBenchmark.java b/benchmarks/src/main/java/org/opensearch/benchmark/gateway/remote/RemoteClusterStateBenchmark.java index 8ed6d34afbdf5..e50f694c2200b 100644 --- a/benchmarks/src/main/java/org/opensearch/benchmark/gateway/remote/RemoteClusterStateBenchmark.java +++ b/benchmarks/src/main/java/org/opensearch/benchmark/gateway/remote/RemoteClusterStateBenchmark.java @@ -26,16 +26,23 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.cluster.metadata.ComposableIndexTemplate; +import org.opensearch.cluster.metadata.ComposableIndexTemplateMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexTemplateMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.cluster.metadata.Template; +import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.Strings; import org.opensearch.env.Environment; +import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.Node; import org.opensearch.repositories.RepositoriesService; @@ -50,6 +57,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -58,6 +66,7 @@ import java.util.Random; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import static org.opensearch.env.Environment.PATH_HOME_SETTING; import static org.opensearch.env.Environment.PATH_REPO_SETTING; @@ -71,12 +80,12 @@ @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Benchmark) @Fork(3) -@Warmup(iterations = 10) @Measurement(iterations = 10) -@BenchmarkMode(Mode.All) +@BenchmarkMode(Mode.AverageTime) public class RemoteClusterStateBenchmark { @Param({ // indices| aliases| templates| + " 1000| 100| 100|", " 10000| 1000| 1000|", " 20000| 2000| 2000|", "50000| 5000| 5000|" @@ -89,8 +98,15 @@ public class RemoteClusterStateBenchmark { private final String repository = "test-fs-repository"; private RemoteClusterStateService remoteClusterStateService; private ClusterState initialClusterState; + private ClusterState updatedClusterState; + private ClusterState updatedSettingsClusterState; + private ClusterState updatedTemplatesClusterState; + private ClusterState updatedCoordinationClusterState; + private ClusterState updatedIndexMetadataClusterState; + private ClusterMetadataManifest manifest; private static final Path path; - private String randomPreviousUUID; + private String randomUUID; + Random random; static { try { @@ -107,7 +123,8 @@ public void setup() throws IOException { final int numAliases = toInt(params[1]); final int numTemplates = toInt(params[2]); - Random random = new Random(); + // keeping the seed same to ensure the benchmark isn't inconsistent in different runs + random = new Random(11111111111L); Path repoPath = Files.createDirectory(path.resolve(repository)); Settings settings = Settings.builder() .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) @@ -117,59 +134,12 @@ public void setup() throws IOException { .put(REPOSITORIES_LOCATION_SETTING.getKey(), repoPath) .build(); - // Number of nodes doesn't matter in this benchmark as DiscoveryNodes object is not uploaded to remote - int numNodes = random.nextInt(200) + 1; - DiscoveryNodes.Builder nb = setUpClusterNodes(numNodes); - - String clusterManagerNodeId = "node_s_" + (random.nextInt(numNodes) + 1); - nb.clusterManagerNodeId(clusterManagerNodeId); - nb.localNodeId(clusterManagerNodeId); - - Metadata.Builder mb = Metadata.builder(); - // Populate index metadata - for (int i = 0; i < numIndices; i++) { - int aliases = random.nextInt(numAliases); - IndexMetadata.Builder indexMetadata = IndexMetadata.builder("index_" + i) - .settings(Settings.builder().put("index.version.created", Version.CURRENT)) - .numberOfShards(numShards) - .numberOfReplicas(numReplicas); - - for (int j=0; j < aliases; j++) { - AliasMetadata.Builder aliasMetadata = AliasMetadata.builder(String.format(Locale.ROOT, "alias_{}_index_{}", j, i)); - indexMetadata.putAlias(aliasMetadata); - } - - mb.put(indexMetadata); - } - - // Populate template metadata - for (int i = 0; i< numTemplates; i++) { - IndexTemplateMetadata.Builder indexTemplate = IndexTemplateMetadata.builder("template_" + i); - indexTemplate.patterns(Collections.singletonList("index_" + i)); - int aliases = random.nextInt(numAliases); - for (int j = 0; j < aliases; j++) { - AliasMetadata.Builder aliasMetadata = AliasMetadata.builder( - String.format(Locale.ROOT, "alias_{}_template_{}", j, i) - ); - indexTemplate.putAlias(aliasMetadata); - } - mb.put(indexTemplate); - } - - // Populate coordination metadata -// CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder() -// .term(7331L) -// .addVotingConfigExclusion(new CoordinationMetadata.VotingConfigExclusion()) - - Metadata metadata = mb.build(); - - initialClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(settings)) - .metadata(metadata) - .nodes(nb) - .build(); - - randomPreviousUUID = UUID.randomUUID().toString(); - + initialClusterState = createInitialClusterState(numIndices, numAliases, numTemplates, settings); + updatedClusterState = ClusterState.builder(initialClusterState).version(initialClusterState.version() + 1).build(); + updatedSettingsClusterState = updateSettings(initialClusterState); + updatedTemplatesClusterState = updateTemplates(initialClusterState); + updatedCoordinationClusterState = updateCoordinationMetadata(initialClusterState); + updatedIndexMetadataClusterState = updateIndexMetadata(initialClusterState); String clusterManagerNode = initialClusterState.nodes().getClusterManagerNodeId(); @@ -190,7 +160,6 @@ public void setup() throws IOException { Collections.emptySet(), NoopTracer.INSTANCE ); - RepositoriesService repositoriesService = new RepositoriesService( settings, clusterService, @@ -199,10 +168,8 @@ public void setup() throws IOException { Collections.emptyMap(), threadPool ); - Map repositoryMap = new HashMap<>(); - - ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); Repository repo = new FsRepository( new RepositoryMetadata(repository, FsRepository.TYPE, settings), new Environment(settings, path), @@ -226,11 +193,195 @@ public void setup() throws IOException { threadPool ); remoteClusterStateService.start(); + manifest = remoteClusterStateService.writeFullMetadata(initialClusterState, Strings.UNKNOWN_UUID_VALUE); } @Benchmark - public void testFullMetadataUpload() throws IOException { - remoteClusterStateService.writeFullMetadata(initialClusterState, randomPreviousUUID); + public void measureFullMetadataUpload() throws IOException { + remoteClusterStateService.writeFullMetadata(updatedClusterState, randomUUID); + } + + @Benchmark + public void measureIncrementalClusterStateUpdate_Settings() throws IOException { + remoteClusterStateService.writeIncrementalMetadata(initialClusterState, updatedSettingsClusterState, manifest); + } + + @Benchmark + public void measureIncrementalClusterStateUpdate_Templates() throws IOException { + remoteClusterStateService.writeIncrementalMetadata(initialClusterState, updatedTemplatesClusterState, manifest); + } + + @Benchmark + public void measureIncrementalClusterStateUpdate_Coordination() throws IOException { + remoteClusterStateService.writeIncrementalMetadata(initialClusterState, updatedCoordinationClusterState, manifest); + } + + @Benchmark + public void measureIncrementalClusterStateUpdate_IndexMetadata() throws IOException { + remoteClusterStateService.writeIncrementalMetadata(initialClusterState, updatedIndexMetadataClusterState, manifest); + } + + @TearDown + public void tearDown() throws IOException { + deleteFolder(path.toFile()); + } + + private ClusterState createInitialClusterState(int numIndices, int numAliases, int numTemplates, Settings settings) throws IOException { + // Number of nodes doesn't matter in this benchmark as DiscoveryNodes object is not uploaded to remote + int numNodes = random.nextInt(200) + 1; + DiscoveryNodes.Builder nb = setUpClusterNodes(numNodes); + + String clusterManagerNodeId = "node_s_" + (random.nextInt(numNodes) + 1); + nb.clusterManagerNodeId(clusterManagerNodeId); + nb.localNodeId(clusterManagerNodeId); + + Metadata.Builder mb = Metadata.builder(); + // Populate index metadata + for (int i = 0; i < numIndices; i++) { + int aliases = random.nextInt(numAliases); + IndexMetadata.Builder indexMetadata = IndexMetadata.builder("index_" + i) + .settings(Settings.builder().put("index.version.created", Version.CURRENT)) + .numberOfShards(numShards) + .numberOfReplicas(numReplicas); + + for (int j=0; j < aliases; j++) { + AliasMetadata.Builder aliasMetadata = AliasMetadata.builder(String.format(Locale.ROOT, "alias_{}_index_{}", j, i)); + indexMetadata.putAlias(aliasMetadata); + } + + mb.put(indexMetadata); + } + + // Populate coordination metadata + CoordinationMetadata.Builder coordinationMetadata = CoordinationMetadata.builder().term(7331L); + for (int i = 0; i < numNodes; i++) { + String nodeId = "node_s_" + (i + 1); + if (nodeId.equals(clusterManagerNodeId)) continue; + coordinationMetadata.addVotingConfigExclusion(new CoordinationMetadata.VotingConfigExclusion(nb.get(nodeId))); + } + mb.coordinationMetadata(coordinationMetadata.build()); + + // Populate template metadata + for (int i = 0; i< numTemplates; i++) { + IndexTemplateMetadata.Builder indexTemplate = IndexTemplateMetadata.builder("template_" + i); + indexTemplate.patterns(Collections.singletonList("index_" + i)); + int aliases = random.nextInt(numAliases); + for (int j = 0; j < aliases; j++) { + AliasMetadata.Builder aliasMetadata = AliasMetadata.builder( + String.format(Locale.ROOT, "alias_{}_template_{}", j, i) + ); + indexTemplate.putAlias(aliasMetadata); + } + mb.put(indexTemplate); + } + + // Add settings to metadata + mb.persistentSettings(settings); + + // Add Custom Metadata + Map templates = new HashMap<>(); + for (int i = 0; i < numTemplates; i++) { + String alias = randomAlphaOfLength(5); + Template template = new Template( + Settings.builder() + .put(IndexMetadata.SETTING_BLOCKS_READ, randomBoolean()) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10)) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 5)) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_PRIORITY, randomIntBetween(0, 100000)) + .build(), + new CompressedXContent("{\"properties\":{\"" + randomAlphaOfLength(5) + "\":{\"type\":\"keyword\"}}}"), + Collections.singletonMap(alias, AliasMetadata.builder(alias).build()) + ); + List indexPatterns = randomList(1, 4, () -> randomAlphaOfLength(4)); + List componentTemplates = randomList(0, 10, () -> randomAlphaOfLength(5)); + + templates.put( + randomAlphaOfLength(5), + new ComposableIndexTemplate( + indexPatterns, + template, + componentTemplates, + 1L, + 1L, + Collections.emptyMap(), + null + ) + ); + } + mb.putCustom(ComposableIndexTemplateMetadata.TYPE, new ComposableIndexTemplateMetadata(templates)); + Metadata metadata = mb.build(); + + randomUUID = UUID.randomUUID().toString(); + + return ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(settings)) + .metadata(metadata) + .nodes(nb) + .stateUUID(randomUUID) + .build(); + } + + private ClusterState updateSettings(ClusterState initialClusterState) { + Settings settings = initialClusterState.getMetadata().settings(); + Settings updatedSettings = Settings.builder().put(settings).put("foo", "bar").build(); + Metadata.Builder updatedMetadata = Metadata.builder(initialClusterState.metadata()) + .persistentSettings(updatedSettings); + return ClusterState.builder(initialClusterState).metadata(updatedMetadata).build(); + } + + private ClusterState updateCoordinationMetadata(ClusterState initialClusterState) { + CoordinationMetadata.Builder coordinationMetadata = CoordinationMetadata.builder(initialClusterState.coordinationMetadata()); + coordinationMetadata.clearVotingConfigExclusions(); + Metadata.Builder updatedMetadata = Metadata.builder(initialClusterState.metadata()) + .coordinationMetadata(coordinationMetadata.build()); + return ClusterState.builder(initialClusterState).metadata(updatedMetadata).build(); + } + + private ClusterState updateTemplates(ClusterState initialClusterState) { + TemplatesMetadata.Builder tmb = TemplatesMetadata.builder(); + initialClusterState.metadata().getTemplates().values().forEach(templateMetadata -> { + IndexTemplateMetadata.Builder updatedTemplate = IndexTemplateMetadata.builder(templateMetadata.name()); + updatedTemplate.patterns(templateMetadata.patterns()); + templateMetadata.aliases().values().forEach(updatedTemplate::putAlias); + // update mappings in the template + try { + updatedTemplate.putMapping("type", "{\"type\":{\"properties\":{\"field\":{\"type\":\"keyword\"}}}}"); + } catch (IOException e) { + throw new RuntimeException(e); + } + tmb.put(updatedTemplate); + }); + Metadata.Builder updatedMetadata = Metadata.builder(initialClusterState.metadata()) + .templates(tmb.build()); + return ClusterState.builder(initialClusterState).metadata(updatedMetadata).build(); + } + + private ClusterState updateIndexMetadata(ClusterState initialClusterState) { + Metadata.Builder updatedMetadata = Metadata.builder(initialClusterState.metadata()); + initialClusterState.metadata().indices().values().forEach(indexMetadata -> { + IndexMetadata.Builder updatedIndexMetadata = IndexMetadata.builder(indexMetadata); + updatedIndexMetadata.settings(Settings.builder().put(indexMetadata.getSettings()).put("foo", "bar").build()); + if (randomBoolean()) { + updatedMetadata.put(updatedIndexMetadata); + } else { + updatedMetadata.put(indexMetadata, false); + } + }); + return ClusterState.builder(initialClusterState).metadata(updatedMetadata).build(); + } + + + private static void deleteFolder(File path) { + List files = List.of(path.listFiles()); + for (File file : files) { + if (file.isDirectory()) { + deleteFolder(file); + } else { + file.delete(); + } + } + path.delete(); } private DiscoveryNodes.Builder setUpClusterNodes(int nodes) { @@ -242,8 +393,8 @@ private DiscoveryNodes.Builder setUpClusterNodes(int nodes) { attributes.put(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, repository); attributes.put(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, repository); attributes.put( - String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, repository) + "location", - path.toAbsolutePath().toString() + String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, repository) + "location", + path.toAbsolutePath().toString() ); nb.add(Allocators.newNode("node_s_" + i, attributes)); } @@ -254,20 +405,30 @@ private int toInt(String v) { return Integer.valueOf(v.trim()); } - @TearDown - public void tearDown() throws IOException { - deleteFolder(path.toFile()); + private List randomList(int min, int max, Supplier supplier) { + int size = randomIntBetween(min, max); + List list = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + list.add(supplier.get()); + } + return list; } - private static void deleteFolder(File path) { - List files = List.of(path.listFiles()); - for (File file : files) { - if (file.isDirectory()) { - deleteFolder(file); - } else { - file.delete(); - } - } - path.delete(); + private boolean randomBoolean() { + return random.nextBoolean(); + } + + private int randomIntBetween(int min, int max) { + return random.nextInt(max - min) + min; + } + + private String randomAlphaOfLength(int length) { + int leftLimit = 97; // letter 'a' + int rightLimit = 122; // letter 'z' + + return random.ints(leftLimit, rightLimit + 1) + .limit(length) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); } }