From 3186372de8860f6f5f4336e3f7ecebc87c87d190 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 17 May 2024 11:52:13 +0530 Subject: [PATCH] Move diff manifest to new class and fix fromXContent Signed-off-by: Shivansh Arora --- .../core/xcontent/XContentParserUtils.java | 12 + .../remote/ClusterMetadataManifest.java | 395 +--------------- .../remote/ClusterStateDiffManifest.java | 445 ++++++++++++++++++ .../remote/RemoteClusterStateService.java | 8 +- .../gateway/remote/RemoteManifestManager.java | 2 +- 5 files changed, 483 insertions(+), 379 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java index b10be393f9adb..e0128a036148e 100644 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java @@ -38,6 +38,8 @@ import org.opensearch.core.xcontent.XContentParser.Token; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.function.Consumer; @@ -178,4 +180,14 @@ public static void parseTypedKeysObject(XContentParser parser, String delimi throw new ParsingException(parser.getTokenLocation(), "Failed to parse object: empty key"); } } + + public static List parseStringList(XContentParser parser) throws IOException { + List valueList = new ArrayList<>(); + ensureExpectedToken(Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != Token.END_ARRAY) { + ensureExpectedToken(Token.VALUE_STRING, parser.currentToken(), parser); + valueList.add(parser.text()); + } + return valueList; + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index f44a8fb8ebb04..9d80b53e79eaa 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -11,27 +11,19 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.Version; -import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.core.ParseField; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.core.common.util.CollectionUtils; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; -import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; -import java.util.*; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -149,8 +141,8 @@ private static UploadedMetadataAttribute clusterBlocksMetadata(Object[] fields) return (UploadedMetadataAttribute) fields[16]; } - private static ClusterDiffManifest diffManifest(Object[] fields) { - return (ClusterDiffManifest) fields[17]; + private static ClusterStateDiffManifest diffManifest(Object[] fields) { + return (ClusterStateDiffManifest) fields[17]; } private static long routingTableVersion(Object[] fields) { @@ -319,7 +311,21 @@ private static void declareParser(ConstructingObjectParser= CODEC_V3) { - parser.declareNamedObject(ConstructingObjectParser.constructorArg(), ClusterDiffManifest.PARSER, DIFF_MANIFEST); + parser.declareNamedObject( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_DISCOVERY_NODES_METADATA + ); + parser.declareNamedObject( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_CLUSTER_BLOCKS_METADATA + ); + parser.declareObject( + ConstructingObjectParser.constructorArg(), + (p, c) -> ClusterStateDiffManifest.fromXContent(p), + DIFF_MANIFEST + ); } if (codec_version >= CODEC_V4) { parser.declareLong(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_VERSION_FIELD); @@ -349,7 +355,7 @@ private static void declareParser(ConstructingObjectParser indicesRouting; @@ -422,7 +428,7 @@ public UploadedMetadataAttribute getClusterBlocksMetadata() { return uploadedClusterBlocksMetadata; } - public ClusterDiffManifest getDiffManifest() { + public ClusterStateDiffManifest getDiffManifest() { return diffManifest; } @@ -463,7 +469,7 @@ public ClusterMetadataManifest( Map uploadedCustomMetadataMap, UploadedMetadataAttribute discoveryNodesMetadata, UploadedMetadataAttribute clusterBlocksMetadata, - ClusterDiffManifest diffManifest, + ClusterStateDiffManifest diffManifest, long routingTableVersion, List indicesRouting ) { @@ -748,7 +754,7 @@ public static class Builder { private String previousClusterUUID; private boolean committed; private boolean clusterUUIDCommitted; - private ClusterDiffManifest diffManifest; + private ClusterStateDiffManifest diffManifest; private long routingTableVersion; private List indicesRouting; @@ -865,7 +871,7 @@ public Builder clusterBlocksMetadata(UploadedMetadataAttribute clusterBlocksMeta return this; } - public Builder diffManifest(ClusterDiffManifest diffManifest) { + public Builder diffManifest(ClusterStateDiffManifest diffManifest) { this.diffManifest = diffManifest; return this; } @@ -1139,361 +1145,4 @@ public String toString() { + '}'; } } - - public static class ClusterDiffManifest implements ToXContentObject { - private static final ParseField FROM_STATE_UUID_FIELD = new ParseField("from_state_uuid"); - private static final ParseField TO_STATE_UUID_FIELD = new ParseField("to_state_uuid"); - private static final ParseField METADATA_DIFF_FIELD = new ParseField("metadata_diff"); - private static final ParseField COORDINATION_METADATA_UPDATED_FIELD = new ParseField( - "coordination_metadata_diff" - ); - private static final ParseField SETTINGS_METADATA_UPDATED_FIELD = new ParseField("settings_metadata_diff"); - private static final ParseField TEMPLATES_METADATA_UPDATED_FIELD = new ParseField("templates_metadata_diff"); - private static final ParseField INDICES_DIFF_FIELD = new ParseField("indices_diff"); - private static final ParseField UPSERTS_FIELD = new ParseField("upserts"); - private static final ParseField DELETES_FIELD = new ParseField("deletes"); - private static final ParseField CLUSTER_BLOCKS_UPDATED_FIELD = new ParseField("cluster_blocks_diff"); - private static final ParseField DISCOVERY_NODES_UPDATED_FIELD = new ParseField("discovery_nodes_diff"); - private static final ParseField ROUTING_TABLE_DIFF = new ParseField("routing_table_diff"); - private static final ParseField ROUTING_TABLE_UPSERT_FIELD = new ParseField("routing_table_upsert"); - private static final ParseField ROUTING_TABLE_DELETE_FIELD = new ParseField("routing_table_delete"); - private static final ObjectParser.NamedObjectParser PARSER; - static { - ConstructingObjectParser innerParser = new ConstructingObjectParser( - "cluster_diff_manifest", - fields -> ClusterDiffManifest.builder() - .fromStateUUID((String) fields[0]) - .toStateUUID((String) fields[1]) - .coordinationMetadataUpdated((Boolean) fields[2]) - .settingsMetadataUpdated((Boolean) fields[3]) - .templatesMetadataUpdated((Boolean) fields[4]) - .indicesUpdated((List) fields[5]) - .indicesDeleted((List) fields[6]) - .clusterBlocksUpdated((Boolean) fields[7]) - .discoveryNodesUpdated((Boolean) fields[8]) - .indicesRoutingUpdated((List) fields[9]) - .indicesRoutingDeleted((List) fields[10]) - .build() - ); - innerParser.declareString(ConstructingObjectParser.constructorArg(), FROM_STATE_UUID_FIELD); - innerParser.declareString(ConstructingObjectParser.constructorArg(), TO_STATE_UUID_FIELD); - innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), COORDINATION_METADATA_UPDATED_FIELD); - innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), SETTINGS_METADATA_UPDATED_FIELD); - innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), TEMPLATES_METADATA_UPDATED_FIELD); - innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), UPSERTS_FIELD); - innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), DELETES_FIELD); - innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), DISCOVERY_NODES_UPDATED_FIELD); - innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), CLUSTER_BLOCKS_UPDATED_FIELD); - innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_UPSERT_FIELD); - innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_DELETE_FIELD); - - PARSER = ((p, c, name) -> innerParser.parse(p, null)); - } - private final String fromStateUUID; - private final String toStateUUID; - private final boolean coordinationMetadataUpdated; - private final boolean settingsMetadataUpdated; - private final boolean templatesMetadataUpdated; - private final Map customMetadataUpdated; - private final List indicesUpdated; - private final List indicesDeleted; - private final boolean clusterBlocksUpdated; - private final boolean discoveryNodesUpdated; - private final List indicesRoutingUpdated; - private final List indicesRoutingDeleted; - - ClusterDiffManifest(ClusterState state, ClusterState previousState) { - fromStateUUID = previousState.stateUUID(); - toStateUUID = state.stateUUID(); - coordinationMetadataUpdated = Metadata.isCoordinationMetadataEqual(state.metadata(), previousState.metadata()); - settingsMetadataUpdated = Metadata.isSettingsMetadataEqual(state.metadata(), previousState.metadata()); - templatesMetadataUpdated = Metadata.isTemplatesMetadataEqual(state.metadata(), previousState.metadata()); - indicesDeleted = findRemovedIndices(state.metadata().indices(), previousState.metadata().indices()); - indicesUpdated = findUpdatedIndices(state.metadata().indices(), previousState.metadata().indices()); - clusterBlocksUpdated = state.blocks().equals(previousState.blocks()); - discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges(); - customMetadataUpdated = new HashMap<>(); - for (String custom : state.metadata().customs().keySet()) { - customMetadataUpdated.put( - custom, - state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom)) - ); - } - indicesRoutingUpdated = getIndicesRoutingUpdated(previousState.routingTable(), state.routingTable()); - indicesRoutingDeleted = getIndicesRoutingDeleted(previousState.routingTable(), state.routingTable()); - } - - public ClusterDiffManifest(String fromStateUUID, - String toStateUUID, - boolean coordinationMetadataUpdated, - boolean settingsMetadataUpdated, - boolean templatesMetadataUpdated, - Map customMetadataUpdated, - List indicesUpdated, - List indicesDeleted, - boolean clusterBlocksUpdated, - boolean discoveryNodesUpdated, - ListindicesRoutingUpdated, - ListindicesRoutingDeleted) { - this.fromStateUUID = fromStateUUID; - this.toStateUUID = toStateUUID; - this.coordinationMetadataUpdated = coordinationMetadataUpdated; - this.settingsMetadataUpdated = settingsMetadataUpdated; - this.templatesMetadataUpdated = templatesMetadataUpdated; - this.customMetadataUpdated = customMetadataUpdated; - this.indicesUpdated = indicesUpdated; - this.indicesDeleted = indicesDeleted; - this.clusterBlocksUpdated = clusterBlocksUpdated; - this.discoveryNodesUpdated = discoveryNodesUpdated; - this.indicesRoutingUpdated = indicesRoutingUpdated; - this.indicesRoutingDeleted = indicesRoutingDeleted; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - { - builder.field(FROM_STATE_UUID_FIELD.getPreferredName(), fromStateUUID); - builder.field(TO_STATE_UUID_FIELD.getPreferredName(), toStateUUID); - builder.startObject(METADATA_DIFF_FIELD.getPreferredName()); - { - builder.field(COORDINATION_METADATA_UPDATED_FIELD.getPreferredName(), coordinationMetadataUpdated); - builder.field(SETTINGS_METADATA_UPDATED_FIELD.getPreferredName(), settingsMetadataUpdated); - builder.field(TEMPLATES_METADATA_UPDATED_FIELD.getPreferredName(), templatesMetadataUpdated); - builder.startObject(INDICES_DIFF_FIELD.getPreferredName()); - builder.startArray(UPSERTS_FIELD.getPreferredName()); - for (String index : indicesUpdated) { - builder.value(index); - } - builder.endArray(); - builder.startArray(DELETES_FIELD.getPreferredName()); - for (String index : indicesDeleted) { - builder.value(index); - } - builder.endArray(); - builder.endObject(); - // ToDo: add the custom metadata diff when we add a parser for this -// for (Map.Entry entry : customMetadataUpdated.entrySet()) { -// if (entry.getValue()) builder.field("customs_" + entry.getKey(), true); -// } - } - builder.endObject(); - builder.field(CLUSTER_BLOCKS_UPDATED_FIELD.getPreferredName(), clusterBlocksUpdated); - builder.field(DISCOVERY_NODES_UPDATED_FIELD.getPreferredName(), discoveryNodesUpdated); - - builder.startObject(ROUTING_TABLE_DIFF.getPreferredName()); - builder.startArray(ROUTING_TABLE_UPSERT_FIELD.getPreferredName()); - for (String index : indicesRoutingUpdated) { - builder.value(index); - } - builder.endArray(); - builder.startArray(ROUTING_TABLE_DELETE_FIELD.getPreferredName()); - for (String index : indicesRoutingDeleted) { - builder.value(index); - } - builder.endArray(); - builder.endObject(); - } - return builder; - } - - public static ClusterDiffManifest fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null, null); - } - - public List findRemovedIndices(Map indices, Map previousIndices) { - List removedIndices = new ArrayList<>(); - for (String index : previousIndices.keySet()) { - // index present in previous state but not in current - if (!indices.containsKey(index)) { - removedIndices.add(index); - } - } - return removedIndices; - } - - public List findUpdatedIndices(Map indices, Map previousIndices) { - List updatedIndices = new ArrayList<>(); - for (String index : indices.keySet()) { - if (!previousIndices.containsKey(index)) { - updatedIndices.add(index); - } else if (previousIndices.get(index).getVersion() != indices.get(index).getVersion()) { - updatedIndices.add(index); - } - } - return updatedIndices; - } - - public List getIndicesRoutingDeleted(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { - List deletedIndices = new ArrayList<>(); - for(IndexRoutingTable previousIndexRouting: previousRoutingTable.getIndicesRouting().values()) { - if(!currentRoutingTable.getIndicesRouting().containsKey(previousIndexRouting.getIndex().getName())) { - // Latest Routing Table does not have entry for the index which means the index is deleted - deletedIndices.add(previousIndexRouting.getIndex().getName()); - } - } - return deletedIndices; - } - - public List getIndicesRoutingUpdated(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { - List updatedIndicesRouting = new ArrayList<>(); - for(IndexRoutingTable currentIndicesRouting: currentRoutingTable.getIndicesRouting().values()) { - if(!previousRoutingTable.getIndicesRouting().containsKey(currentIndicesRouting.getIndex().getName())) { - // Latest Routing Table does not have entry for the index which means the index is created - updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); - } else { - if(previousRoutingTable.getIndicesRouting().get(currentIndicesRouting.getIndex().getName()).equals(currentIndicesRouting)) { - // if the latest routing table has the same routing table as the previous routing table, then the index is not updated - continue; - } - updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); - } - } - return updatedIndicesRouting; - } - - public String getFromStateUUID() { - return fromStateUUID; - } - - public String getToStateUUID() { - return toStateUUID; - } - - public boolean isCoordinationMetadataUpdated() { - return coordinationMetadataUpdated; - } - - public boolean isSettingsMetadataUpdated() { - return settingsMetadataUpdated; - } - - public boolean isTemplatesMetadataUpdated() { - return templatesMetadataUpdated; - } - - public Map getCustomMetadataUpdated() { - return customMetadataUpdated; - } - - public List getIndicesUpdated() { - return indicesUpdated; - } - - public List getIndicesDeleted() { - return indicesDeleted; - } - - public boolean isClusterBlocksUpdated() { - return clusterBlocksUpdated; - } - - public boolean isDiscoveryNodesUpdated() { - return discoveryNodesUpdated; - } - - public List getIndicesRoutingUpdated() { - return indicesRoutingUpdated; - } - - public List getIndicesRoutingDeleted() { - return indicesRoutingDeleted; - } - - public static ClusterDiffManifest.Builder builder() { - return new Builder(); - } - - public static class Builder { - private String fromStateUUID; - private String toStateUUID; - private boolean coordinationMetadataUpdated; - private boolean settingsMetadataUpdated; - private boolean templatesMetadataUpdated; - private Map customMetadataUpdated; - private List indicesUpdated; - private List indicesDeleted; - private boolean clusterBlocksUpdated; - private boolean discoveryNodesUpdated; - private List indicesRoutingUpdated; - private List indicesRoutingDeleted; - public Builder() {} - - public Builder fromStateUUID(String fromStateUUID) { - this.fromStateUUID = fromStateUUID; - return this; - } - - public Builder toStateUUID(String toStateUUID) { - this.toStateUUID = toStateUUID; - return this; - } - - public Builder coordinationMetadataUpdated(boolean coordinationMetadataUpdated) { - this.coordinationMetadataUpdated = coordinationMetadataUpdated; - return this; - } - - public Builder settingsMetadataUpdated(boolean settingsMetadataUpdated) { - this.settingsMetadataUpdated = settingsMetadataUpdated; - return this; - } - - public Builder templatesMetadataUpdated(boolean templatesMetadataUpdated) { - this.templatesMetadataUpdated = templatesMetadataUpdated; - return this; - } - - public Builder customMetadataUpdated(Map customMetadataUpdated) { - this.customMetadataUpdated = customMetadataUpdated; - return this; - } - - public Builder indicesUpdated(List indicesUpdated) { - this.indicesUpdated = indicesUpdated; - return this; - } - - public Builder indicesDeleted(List indicesDeleted) { - this.indicesDeleted = indicesDeleted; - return this; - } - - public Builder clusterBlocksUpdated(boolean clusterBlocksUpdated) { - this.clusterBlocksUpdated = clusterBlocksUpdated; - return this; - } - - public Builder discoveryNodesUpdated(boolean discoveryNodesUpdated) { - this.discoveryNodesUpdated = discoveryNodesUpdated; - return this; - } - - public Builder indicesRoutingUpdated(List indicesRoutingUpdated) { - this.indicesRoutingUpdated = indicesRoutingUpdated; - return this; - } - - public Builder indicesRoutingDeleted(List indicesRoutingDeleted) { - this.indicesRoutingDeleted = indicesRoutingDeleted; - return this; - } - - public ClusterDiffManifest build() { - return new ClusterDiffManifest( - fromStateUUID, - toStateUUID, - coordinationMetadataUpdated, - settingsMetadataUpdated, - templatesMetadataUpdated, - customMetadataUpdated, - indicesUpdated, - indicesDeleted, - clusterBlocksUpdated, - discoveryNodesUpdated, - indicesRoutingUpdated, - indicesRoutingDeleted - ); - } - } - } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java new file mode 100644 index 0000000000000..cc3603d01f78f --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -0,0 +1,445 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParseException; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.parseStringList; + +public class ClusterStateDiffManifest implements ToXContentObject { + private static final String FROM_STATE_UUID_FIELD = "from_state_uuid"; + private static final String TO_STATE_UUID_FIELD = "to_state_uuid"; + private static final String METADATA_DIFF_FIELD = "metadata_diff"; + private static final String COORDINATION_METADATA_UPDATED_FIELD = "coordination_metadata_diff"; + private static final String SETTINGS_METADATA_UPDATED_FIELD = "settings_metadata_diff"; + private static final String TEMPLATES_METADATA_UPDATED_FIELD = ("templates_metadata_diff"); + private static final String INDICES_DIFF_FIELD = ("indices_diff"); + private static final String UPSERTS_FIELD = ("upserts"); + private static final String DELETES_FIELD = ("deletes"); + private static final String CLUSTER_BLOCKS_UPDATED_FIELD = ("cluster_blocks_diff"); + private static final String DISCOVERY_NODES_UPDATED_FIELD = ("discovery_nodes_diff"); + private static final String ROUTING_TABLE_DIFF = ("routing_table_diff"); + private static final String ROUTING_TABLE_UPSERT_FIELD = ("routing_table_upsert"); + private static final String ROUTING_TABLE_DELETE_FIELD = ("routing_table_delete"); + private final String fromStateUUID; + private final String toStateUUID; + private final boolean coordinationMetadataUpdated; + private final boolean settingsMetadataUpdated; + private final boolean templatesMetadataUpdated; + private final Map customMetadataUpdated; + private final List indicesUpdated; + private final List indicesDeleted; + private final boolean clusterBlocksUpdated; + private final boolean discoveryNodesUpdated; + private final List indicesRoutingUpdated; + private final List indicesRoutingDeleted; + + ClusterStateDiffManifest(ClusterState state, ClusterState previousState) { + fromStateUUID = previousState.stateUUID(); + toStateUUID = state.stateUUID(); + coordinationMetadataUpdated = Metadata.isCoordinationMetadataEqual(state.metadata(), previousState.metadata()); + settingsMetadataUpdated = Metadata.isSettingsMetadataEqual(state.metadata(), previousState.metadata()); + templatesMetadataUpdated = Metadata.isTemplatesMetadataEqual(state.metadata(), previousState.metadata()); + indicesDeleted = findRemovedIndices(state.metadata().indices(), previousState.metadata().indices()); + indicesUpdated = findUpdatedIndices(state.metadata().indices(), previousState.metadata().indices()); + clusterBlocksUpdated = state.blocks().equals(previousState.blocks()); + discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges(); + customMetadataUpdated = new HashMap<>(); + for (String custom : state.metadata().customs().keySet()) { + customMetadataUpdated.put( + custom, + state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom)) + ); + } + indicesRoutingUpdated = getIndicesRoutingUpdated(previousState.routingTable(), state.routingTable()); + indicesRoutingDeleted = getIndicesRoutingDeleted(previousState.routingTable(), state.routingTable()); + } + + public ClusterStateDiffManifest(String fromStateUUID, + String toStateUUID, + boolean coordinationMetadataUpdated, + boolean settingsMetadataUpdated, + boolean templatesMetadataUpdated, + Map customMetadataUpdated, + List indicesUpdated, + List indicesDeleted, + boolean clusterBlocksUpdated, + boolean discoveryNodesUpdated, + ListindicesRoutingUpdated, + ListindicesRoutingDeleted) { + this.fromStateUUID = fromStateUUID; + this.toStateUUID = toStateUUID; + this.coordinationMetadataUpdated = coordinationMetadataUpdated; + this.settingsMetadataUpdated = settingsMetadataUpdated; + this.templatesMetadataUpdated = templatesMetadataUpdated; + this.customMetadataUpdated = customMetadataUpdated; + this.indicesUpdated = indicesUpdated; + this.indicesDeleted = indicesDeleted; + this.clusterBlocksUpdated = clusterBlocksUpdated; + this.discoveryNodesUpdated = discoveryNodesUpdated; + this.indicesRoutingUpdated = indicesRoutingUpdated; + this.indicesRoutingDeleted = indicesRoutingDeleted; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + { + builder.field(FROM_STATE_UUID_FIELD, fromStateUUID); + builder.field(TO_STATE_UUID_FIELD, toStateUUID); + builder.startObject(METADATA_DIFF_FIELD); + { + builder.field(COORDINATION_METADATA_UPDATED_FIELD, coordinationMetadataUpdated); + builder.field(SETTINGS_METADATA_UPDATED_FIELD, settingsMetadataUpdated); + builder.field(TEMPLATES_METADATA_UPDATED_FIELD, templatesMetadataUpdated); + builder.startObject(INDICES_DIFF_FIELD); + builder.startArray(UPSERTS_FIELD); + for (String index : indicesUpdated) { + builder.value(index); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String index : indicesDeleted) { + builder.value(index); + } + builder.endArray(); + builder.endObject(); + // ToDo: add the custom metadata diff when we add a parser for this +// for (Map.Entry entry : customMetadataUpdated.entrySet()) { +// if (entry.getValue()) builder.field("customs_" + entry.getKey(), true); +// } + } + builder.endObject(); + builder.field(CLUSTER_BLOCKS_UPDATED_FIELD, clusterBlocksUpdated); + builder.field(DISCOVERY_NODES_UPDATED_FIELD, discoveryNodesUpdated); + + builder.startObject(ROUTING_TABLE_DIFF); + builder.startArray(ROUTING_TABLE_UPSERT_FIELD); + for (String index : indicesRoutingUpdated) { + builder.value(index); + } + builder.endArray(); + builder.startArray(ROUTING_TABLE_DELETE_FIELD); + for (String index : indicesRoutingDeleted) { + builder.value(index); + } + builder.endArray(); + builder.endObject(); + } + return builder; + } + + public static ClusterStateDiffManifest fromXContent(XContentParser parser) throws IOException { + Builder builder = new Builder(); + if (parser.currentToken() == null) { // fresh parser? move to next token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + String currentFieldName = parser.currentName(); + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if (currentFieldName.equals(METADATA_DIFF_FIELD)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + if (token.isValue()) { + switch (currentFieldName) { + case COORDINATION_METADATA_UPDATED_FIELD: + builder.coordinationMetadataUpdated(parser.booleanValue()); + break; + case SETTINGS_METADATA_UPDATED_FIELD: + builder.settingsMetadataUpdated(parser.booleanValue()); + break; + case TEMPLATES_METADATA_UPDATED_FIELD: + builder.templatesMetadataUpdated(parser.booleanValue()); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (currentFieldName.equals(INDICES_DIFF_FIELD)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + switch (currentFieldName) { + case UPSERTS_FIELD: + builder.indicesUpdated(parseStringList(parser)); + break; + case DELETES_FIELD: + builder.indicesDeleted(parseStringList(parser)); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } + } else { + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else { + throw new XContentParseException("Unexpected token [" + token + "]"); + } + } + } else if (currentFieldName.equals(ROUTING_TABLE_DIFF)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + switch (currentFieldName) { + case ROUTING_TABLE_UPSERT_FIELD: + builder.indicesRoutingUpdated(parseStringList(parser)); + break; + case ROUTING_TABLE_DELETE_FIELD: + builder.indicesRoutingDeleted(parseStringList(parser)); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } + } else { + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else if (token.isValue()) { + switch (currentFieldName) { + case FROM_STATE_UUID_FIELD: + builder.fromStateUUID(parser.text()); + break; + case TO_STATE_UUID_FIELD: + builder.toStateUUID(parser.text()); + break; + case CLUSTER_BLOCKS_UPDATED_FIELD: + builder.clusterBlocksUpdated(parser.booleanValue()); + break; + case DISCOVERY_NODES_UPDATED_FIELD: + builder.discoveryNodesUpdated(parser.booleanValue()); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else { + throw new XContentParseException("Unexpected token [" + token + "]"); + } + } + return builder.build(); + } + + public List findRemovedIndices(Map indices, Map previousIndices) { + List removedIndices = new ArrayList<>(); + for (String index : previousIndices.keySet()) { + // index present in previous state but not in current + if (!indices.containsKey(index)) { + removedIndices.add(index); + } + } + return removedIndices; + } + + public List findUpdatedIndices(Map indices, Map previousIndices) { + List updatedIndices = new ArrayList<>(); + for (String index : indices.keySet()) { + if (!previousIndices.containsKey(index)) { + updatedIndices.add(index); + } else if (previousIndices.get(index).getVersion() != indices.get(index).getVersion()) { + updatedIndices.add(index); + } + } + return updatedIndices; + } + + public List getIndicesRoutingDeleted(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { + List deletedIndices = new ArrayList<>(); + for(IndexRoutingTable previousIndexRouting: previousRoutingTable.getIndicesRouting().values()) { + if(!currentRoutingTable.getIndicesRouting().containsKey(previousIndexRouting.getIndex().getName())) { + // Latest Routing Table does not have entry for the index which means the index is deleted + deletedIndices.add(previousIndexRouting.getIndex().getName()); + } + } + return deletedIndices; + } + + public List getIndicesRoutingUpdated(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { + List updatedIndicesRouting = new ArrayList<>(); + for(IndexRoutingTable currentIndicesRouting: currentRoutingTable.getIndicesRouting().values()) { + if(!previousRoutingTable.getIndicesRouting().containsKey(currentIndicesRouting.getIndex().getName())) { + // Latest Routing Table does not have entry for the index which means the index is created + updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); + } else { + if(previousRoutingTable.getIndicesRouting().get(currentIndicesRouting.getIndex().getName()).equals(currentIndicesRouting)) { + // if the latest routing table has the same routing table as the previous routing table, then the index is not updated + continue; + } + updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); + } + } + return updatedIndicesRouting; + } + + public String getFromStateUUID() { + return fromStateUUID; + } + + public String getToStateUUID() { + return toStateUUID; + } + + public boolean isCoordinationMetadataUpdated() { + return coordinationMetadataUpdated; + } + + public boolean isSettingsMetadataUpdated() { + return settingsMetadataUpdated; + } + + public boolean isTemplatesMetadataUpdated() { + return templatesMetadataUpdated; + } + + public Map getCustomMetadataUpdated() { + return customMetadataUpdated; + } + + public List getIndicesUpdated() { + return indicesUpdated; + } + + public List getIndicesDeleted() { + return indicesDeleted; + } + + public boolean isClusterBlocksUpdated() { + return clusterBlocksUpdated; + } + + public boolean isDiscoveryNodesUpdated() { + return discoveryNodesUpdated; + } + + public List getIndicesRoutingUpdated() { + return indicesRoutingUpdated; + } + + public List getIndicesRoutingDeleted() { + return indicesRoutingDeleted; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String fromStateUUID; + private String toStateUUID; + private boolean coordinationMetadataUpdated; + private boolean settingsMetadataUpdated; + private boolean templatesMetadataUpdated; + private Map customMetadataUpdated; + private List indicesUpdated; + private List indicesDeleted; + private boolean clusterBlocksUpdated; + private boolean discoveryNodesUpdated; + private List indicesRoutingUpdated; + private List indicesRoutingDeleted; + public Builder() {} + + public Builder fromStateUUID(String fromStateUUID) { + this.fromStateUUID = fromStateUUID; + return this; + } + + public Builder toStateUUID(String toStateUUID) { + this.toStateUUID = toStateUUID; + return this; + } + + public Builder coordinationMetadataUpdated(boolean coordinationMetadataUpdated) { + this.coordinationMetadataUpdated = coordinationMetadataUpdated; + return this; + } + + public Builder settingsMetadataUpdated(boolean settingsMetadataUpdated) { + this.settingsMetadataUpdated = settingsMetadataUpdated; + return this; + } + + public Builder templatesMetadataUpdated(boolean templatesMetadataUpdated) { + this.templatesMetadataUpdated = templatesMetadataUpdated; + return this; + } + + public Builder customMetadataUpdated(Map customMetadataUpdated) { + this.customMetadataUpdated = customMetadataUpdated; + return this; + } + + public Builder indicesUpdated(List indicesUpdated) { + this.indicesUpdated = indicesUpdated; + return this; + } + + public Builder indicesDeleted(List indicesDeleted) { + this.indicesDeleted = indicesDeleted; + return this; + } + + public Builder clusterBlocksUpdated(boolean clusterBlocksUpdated) { + this.clusterBlocksUpdated = clusterBlocksUpdated; + return this; + } + + public Builder discoveryNodesUpdated(boolean discoveryNodesUpdated) { + this.discoveryNodesUpdated = discoveryNodesUpdated; + return this; + } + + public Builder indicesRoutingUpdated(List indicesRoutingUpdated) { + this.indicesRoutingUpdated = indicesRoutingUpdated; + return this; + } + + public Builder indicesRoutingDeleted(List indicesRoutingDeleted) { + this.indicesRoutingDeleted = indicesRoutingDeleted; + return this; + } + + public ClusterStateDiffManifest build() { + return new ClusterStateDiffManifest( + fromStateUUID, + toStateUUID, + coordinationMetadataUpdated, + settingsMetadataUpdated, + templatesMetadataUpdated, + customMetadataUpdated, + indicesUpdated, + indicesDeleted, + clusterBlocksUpdated, + discoveryNodesUpdated, + indicesRoutingUpdated, + indicesRoutingDeleted + ); + } + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index c5d17c225a9b2..f725a0cd345e5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -18,7 +18,6 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; -import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; @@ -33,7 +32,6 @@ import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; -import org.opensearch.gateway.remote.ClusterMetadataManifest.ClusterDiffManifest; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.index.translog.transfer.BlobStoreTransferService; @@ -229,7 +227,7 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri uploadedMetadataResults.uploadedCustomMetadataMap, uploadedMetadataResults.uploadedDiscoveryNodes, uploadedMetadataResults.uploadedClusterBlocks, - new ClusterMetadataManifest.ClusterDiffManifest(clusterState, ClusterState.EMPTY_STATE), + new ClusterStateDiffManifest(clusterState, ClusterState.EMPTY_STATE), routingIndexMetadata, false ); @@ -397,7 +395,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( firstUpload || !customsToUpload.isEmpty() ? allUploadedCustomMap : previousManifest.getCustomMetadataMap(), firstUpload || updateDiscoveryNodes ? uploadedMetadataResults.uploadedDiscoveryNodes : previousManifest.getDiscoveryNodesMetadata(), firstUpload || updateClusterBlocks ? uploadedMetadataResults.uploadedClusterBlocks : previousManifest.getClusterBlocksMetadata(), - new ClusterMetadataManifest.ClusterDiffManifest(clusterState, previousClusterState), + new ClusterStateDiffManifest(clusterState, previousClusterState), routingIndexMetadata, false ); this.latestClusterName = clusterState.getClusterName().value(); @@ -803,7 +801,7 @@ public ClusterState getClusterStateForManifest(String clusterName, ClusterMetada public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadataManifest manifest, ClusterState previousState) { assert manifest.getDiffManifest() != null; - ClusterDiffManifest diff = manifest.getDiffManifest(); + ClusterStateDiffManifest diff = manifest.getDiffManifest(); ClusterState.Builder clusterStateBuilder = ClusterState.builder(previousState); Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata()); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 83a4d7f98b284..c1e0ad4ece47a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -131,7 +131,7 @@ ClusterMetadataManifest uploadManifest( Map uploadedCustomMetadataMap, ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodesMetadata, ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocksMetadata, - ClusterMetadataManifest.ClusterDiffManifest clusterDiffManifest, + ClusterStateDiffManifest clusterDiffManifest, List routingIndexMetadata, boolean committed ) throws IOException { synchronized (this) {