Skip to content

Commit

Permalink
Add custom metadata diff
Browse files Browse the repository at this point in the history 
Signed-off-by: Shivansh Arora <[email protected]>
  • Loading branch information
@shiv0408
shiv0408 committed May 23, 2024
1 parent f7fef97 commit ae2ed6b
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 30 deletions.
67 changes: 46 additions & 21 deletions server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

package org.opensearch.gateway.remote;

import org.apache.lucene.util.packed.DirectMonotonicReader;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.Metadata;
Expand Down Expand Up @@ -35,19 +36,20 @@ public class ClusterStateDiffManifest implements ToXContentObject {
private static final String METADATA_DIFF_FIELD = "metadata_diff";
private static final String COORDINATION_METADATA_UPDATED_FIELD = "coordination_metadata_diff";
private static final String SETTINGS_METADATA_UPDATED_FIELD = "settings_metadata_diff";
private static final String TEMPLATES_METADATA_UPDATED_FIELD = ("templates_metadata_diff");
private static final String INDICES_DIFF_FIELD = ("indices_diff");
private static final String UPSERTS_FIELD = ("upserts");
private static final String DELETES_FIELD = ("deletes");
private static final String CLUSTER_BLOCKS_UPDATED_FIELD = ("cluster_blocks_diff");
private static final String DISCOVERY_NODES_UPDATED_FIELD = ("discovery_nodes_diff");
private static final String ROUTING_TABLE_DIFF = ("routing_table_diff");
private static final String TEMPLATES_METADATA_UPDATED_FIELD = "templates_metadata_diff";
private static final String INDICES_DIFF_FIELD = "indices_diff";
private static final String METADATA_CUSTOM_DIFF_FIELD = "metadata_custom_diff";
private static final String UPSERTS_FIELD = "upserts";
private static final String DELETES_FIELD = "deletes";
private static final String CLUSTER_BLOCKS_UPDATED_FIELD = "cluster_blocks_diff";
private static final String DISCOVERY_NODES_UPDATED_FIELD = "discovery_nodes_diff";
private static final String ROUTING_TABLE_DIFF = "routing_table_diff";
private final String fromStateUUID;
private final String toStateUUID;
private final boolean coordinationMetadataUpdated;
private final boolean settingsMetadataUpdated;
private final boolean templatesMetadataUpdated;
private final Map<String, Boolean> customMetadataUpdated;
private List<String> customMetadataUpdated;
private final List<String> customMetadataDeleted;
private final List<String> indicesUpdated;
private final List<String> indicesDeleted;
Expand All @@ -66,12 +68,11 @@ public class ClusterStateDiffManifest implements ToXContentObject {
indicesUpdated = findUpdatedIndices(state.metadata().indices(), previousState.metadata().indices());
clusterBlocksUpdated = state.blocks().equals(previousState.blocks());
discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges();
customMetadataUpdated = new HashMap<>();
customMetadataUpdated = new ArrayList<>();
for (String custom : state.metadata().customs().keySet()) {
customMetadataUpdated.put(
custom,
state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom))
);
if (!state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom))) {
customMetadataUpdated.add(custom);
}
}
customMetadataDeleted = new ArrayList<>();
for (String custom : previousState.metadata().customs().keySet()) {
Expand All @@ -88,7 +89,8 @@ public ClusterStateDiffManifest(String fromStateUUID,
boolean coordinationMetadataUpdated,
boolean settingsMetadataUpdated,
boolean templatesMetadataUpdated,
Map<String, Boolean> customMetadataUpdated, List<String> customMetadataDeleted,
List<String> customMetadataUpdated,
List<String> customMetadataDeleted,
List<String> indicesUpdated,
List<String> indicesDeleted,
boolean clusterBlocksUpdated,
Expand Down Expand Up @@ -132,10 +134,18 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
builder.endArray();
builder.endObject();
// ToDo: add the custom metadata diff when we add a parser for this
// for (Map.Entry<String, Boolean> entry : customMetadataUpdated.entrySet()) {
// if (entry.getValue()) builder.field("customs_" + entry.getKey(), true);
// }
builder.startObject(METADATA_CUSTOM_DIFF_FIELD);
builder.startArray(UPSERTS_FIELD);
for (String custom : customMetadataUpdated) {
builder.value(custom);
}
builder.endArray();
builder.startArray(DELETES_FIELD);
for (String custom : customMetadataDeleted) {
builder.value(custom);
}
builder.endArray();
builder.endObject();
}
builder.endObject();
builder.field(CLUSTER_BLOCKS_UPDATED_FIELD, clusterBlocksUpdated);
Expand Down Expand Up @@ -206,6 +216,21 @@ public static ClusterStateDiffManifest fromXContent(XContentParser parser) throw
throw new XContentParseException("Unexpected field [" + currentFieldName + "]");
}
}
} else if (currentFieldName.equals(METADATA_CUSTOM_DIFF_FIELD)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
currentFieldName = parser.currentName();
token = parser.nextToken();
switch (currentFieldName) {
case UPSERTS_FIELD:
builder.customMetadataUpdated(parseStringList(parser));
break;
case DELETES_FIELD:
builder.customMetadataDeleted(parseStringList(parser));
break;
default:
throw new XContentParseException("Unexpected field [" + currentFieldName + "]");
}
}
} else {
throw new XContentParseException("Unexpected field [" + currentFieldName + "]");
}
Expand Down Expand Up @@ -331,7 +356,7 @@ public boolean isTemplatesMetadataUpdated() {
return templatesMetadataUpdated;
}

public Map<String, Boolean> getCustomMetadataUpdated() {
public List<String> getCustomMetadataUpdated() {
return customMetadataUpdated;
}

Expand Down Expand Up @@ -373,7 +398,7 @@ public static class Builder {
private boolean coordinationMetadataUpdated;
private boolean settingsMetadataUpdated;
private boolean templatesMetadataUpdated;
private Map<String, Boolean> customMetadataUpdated;
private List<String> customMetadataUpdated;
private List<String> customMetadataDeleted;
private List<String> indicesUpdated;
private List<String> indicesDeleted;
Expand Down Expand Up @@ -408,7 +433,7 @@ public Builder templatesMetadataUpdated(boolean templatesMetadataUpdated) {
return this;
}

public Builder customMetadataUpdated(Map<String, Boolean> customMetadataUpdated) {
public Builder customMetadataUpdated(List<String> customMetadataUpdated) {
this.customMetadataUpdated = customMetadataUpdated;
return this;
}
Expand Down
12 changes: 3 additions & 9 deletions server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,6 @@
*/
public class RemoteClusterStateService implements Closeable {
public static final int RETAINED_MANIFESTS = 10;
public static final int SKIP_CLEANUP_STATE_CHANGES = 10;

private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class);

Expand Down Expand Up @@ -1076,7 +1075,7 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata

Map<String, UploadedMetadataAttribute> updatedCustomMetadata = new HashMap<>();
if (diff.getCustomMetadataUpdated() != null) {
for (String customType : diff.getCustomMetadataUpdated().keySet()) {
for (String customType : diff.getCustomMetadataUpdated()) {
updatedCustomMetadata.put(customType, manifest.getCustomMetadataMap().get(customType));
}
}
Expand All @@ -1097,16 +1096,11 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata
);
ClusterState.Builder clusterStateBuilder = ClusterState.builder(updatedClusterState);
Metadata.Builder metadataBuilder = Metadata.builder(updatedClusterState.metadata());
// remove the deleted indices from the metadata
for (String index:diff.getIndicesDeleted()) {
metadataBuilder.remove(index);
}
if (diff.getCustomMetadataUpdated() != null) {
for (String customType : diff.getCustomMetadataUpdated().keySet()) {
Metadata.Custom custom = remoteGlobalMetadataManager.getCustomsMetadata(manifest.getClusterUUID(),
manifest.getCustomMetadataMap().get(customType).getUploadedFilename(), customType);
metadataBuilder.putCustom(customType, custom);
}
}
// remove the deleted metadata customs from the metadata
if (diff.getCustomMetadataDeleted() != null) {
for (String customType : diff.getCustomMetadataDeleted()) {
metadataBuilder.removeCustom(customType);
Expand Down

0 comments on commit ae2ed6b

Please sign in to comment.