From d0875f97cd77fb5eaeec4254471a0f08743f3fc4 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 9 Feb 2024 01:43:30 +0530 Subject: [PATCH] Upload all the metadata attributes in parallel Signed-off-by: Shivansh Arora --- .../cluster/metadata/IndexMetadata.java | 4 + .../opensearch/cluster/metadata/Metadata.java | 25 +- .../remote/ClusterMetadataManifest.java | 68 +- .../remote/RemoteClusterStateService.java | 630 ++++++++---------- 4 files changed, 353 insertions(+), 374 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 03784df509ed6..5adb07bd91572 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -752,6 +752,10 @@ public String getIndexUUID() { return index.getUUID(); } + public String getIndexName() { + return index.getName(); + } + /** * Test whether the current index UUID is the same as the given one. Returns true if either are _na_ */ diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index 464a6e89effbd..638267fd1bf32 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -948,7 +948,7 @@ public Iterator iterator() { } public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2) { - if (!metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata)) { + if (!isCoordinationMetadataEqual(metadata1, metadata2)) { return false; } if (!metadata1.hashesOfConsistentSettings.equals(metadata2.hashesOfConsistentSettings)) { @@ -967,13 +967,29 @@ public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2 * Compares Metadata entities persisted in Remote Store. */ public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metadata metadata2) { - if (!metadata1.persistentSettings.equals(metadata2.persistentSettings)) { + if (!isSettingsMetadataEqual(metadata1, metadata2)) { return false; } - if (!metadata1.templates.equals(metadata2.templates)) { + if (!isTemplatesMetadataEqual(metadata1, metadata2)) { return false; } // Check if any persistent metadata needs to be saved + return isCustomMetadataEqual(metadata1, metadata2); + } + + public static boolean isCoordinationMetadataEqual(Metadata metadata1, Metadata metadata2) { + return metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata); + } + + public static boolean isSettingsMetadataEqual(Metadata metadata1, Metadata metadata2) { + return metadata1.persistentSettings.equals(metadata2.persistentSettings); + } + + public static boolean isTemplatesMetadataEqual(Metadata metadata1, Metadata metadata2) { + return metadata1.templates.equals(metadata2.templates); + } + + public static boolean isCustomMetadataEqual(Metadata metadata1, Metadata metadata2) { int customCount1 = 0; for (Map.Entry cursor : metadata1.customs.entrySet()) { if (cursor.getValue().context().contains(XContentContext.GATEWAY)) { @@ -987,8 +1003,7 @@ public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metada customCount2++; } } - if (customCount1 != customCount2) return false; - return true; + return customCount1 == customCount2; } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index a9f59fc8b1b8c..a841ca3451795 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -299,7 +300,7 @@ public Map getCustomMetadataMap() { } public boolean hasMetadataAttributesFiles() { - return uploadedCoordinationMetadata != null || uploadedSettingsMetadata != null || uploadedTemplatesMetadata != null || uploadedCustomMetadataMap != null; + return uploadedCoordinationMetadata != null || uploadedSettingsMetadata != null || uploadedTemplatesMetadata != null || !uploadedCustomMetadataMap.isEmpty(); } public ClusterMetadataManifest( @@ -369,7 +370,7 @@ public ClusterMetadataManifest( this.uploadedCoordinationMetadata = uploadedCoordinationMetadata; this.uploadedSettingsMetadata = uploadedSettingsMetadata; this.uploadedTemplatesMetadata = uploadedTemplatesMetadata; - this.uploadedCustomMetadataMap = Collections.unmodifiableMap(uploadedCustomMetadataMap); + this.uploadedCustomMetadataMap = Collections.unmodifiableMap(uploadedCustomMetadataMap != null ? uploadedCustomMetadataMap : new HashMap<>()); } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -534,6 +535,10 @@ public static ClusterMetadataManifest fromXContentV0(XContentParser parser) thro return PARSER_V0.parse(parser, null); } + public static ClusterMetadataManifest fromXContentV1(XContentParser parser) throws IOException { + return PARSER_V1.parse(parser, null); + } + public static ClusterMetadataManifest fromXContent(XContentParser parser) throws IOException { return CURRENT_PARSER.parse(parser, null); } @@ -546,6 +551,10 @@ public static ClusterMetadataManifest fromXContent(XContentParser parser) throws public static class Builder { private String globalMetadataFileName; + private UploadedMetadataAttribute coordinationMetadata; + private UploadedMetadataAttribute settingsMetadata; + private UploadedMetadataAttribute templatesMetadata; + private Map customMetadataMap; private int codecVersion; private List indices; private long clusterTerm; @@ -573,6 +582,31 @@ public Builder globalMetadataFileName(String globalMetadataFileName) { return this; } + public Builder coordinationMetadata(UploadedMetadataAttribute coordinationMetadata) { + this.coordinationMetadata = coordinationMetadata; + return this; + } + + public Builder settingMetadata(UploadedMetadataAttribute settingsMetadata) { + this.settingsMetadata = settingsMetadata; + return this; + } + + public Builder templatesMetadata(UploadedMetadataAttribute templatesMetadata) { + this.templatesMetadata = templatesMetadata; + return this; + } + + public Builder customMetadataMap(Map customMetadataMap) { + this.customMetadataMap = customMetadataMap; + return this; + } + + public Builder put(String custom, UploadedMetadataAttribute customMetadata) { + this.customMetadataMap.put(custom, customMetadata); + return this; + } + public Builder clusterTerm(long clusterTerm) { this.clusterTerm = clusterTerm; return this; @@ -624,6 +658,7 @@ public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) { public Builder() { indices = new ArrayList<>(); + customMetadataMap = new HashMap<>(); } public Builder(ClusterMetadataManifest manifest) { @@ -635,6 +670,10 @@ public Builder(ClusterMetadataManifest manifest) { this.nodeId = manifest.nodeId; this.committed = manifest.committed; this.globalMetadataFileName = manifest.globalMetadataFileName; + this.coordinationMetadata = manifest.uploadedCoordinationMetadata; + this.settingsMetadata = manifest.uploadedSettingsMetadata; + this.templatesMetadata = manifest.uploadedTemplatesMetadata; + this.customMetadataMap = manifest.uploadedCustomMetadataMap; this.codecVersion = manifest.codecVersion; this.indices = new ArrayList<>(manifest.indices); this.previousClusterUUID = manifest.previousClusterUUID; @@ -654,18 +693,27 @@ public ClusterMetadataManifest build() { globalMetadataFileName, indices, previousClusterUUID, - clusterUUIDCommitted + clusterUUIDCommitted, + coordinationMetadata, + settingsMetadata, + templatesMetadata, + customMetadataMap ); } } + public static interface UploadedMetadata { + String getComponent(); + String getUploadedFilename(); + } + /** * Metadata for uploaded index metadata * * @opensearch.internal */ - public static class UploadedIndexMetadata implements Writeable, ToXContentFragment { + public static class UploadedIndexMetadata implements UploadedMetadata, Writeable, ToXContentFragment { private static final ParseField INDEX_NAME_FIELD = new ParseField("index_name"); private static final ParseField INDEX_UUID_FIELD = new ParseField("index_uuid"); @@ -714,6 +762,11 @@ public String getUploadedFilePath() { return uploadedFilename; } + @Override + public String getComponent() { + return getIndexName(); + } + public String getUploadedFilename() { String[] splitPath = uploadedFilename.split("/"); return splitPath[splitPath.length - 1]; @@ -772,7 +825,7 @@ public static UploadedIndexMetadata fromXContent(XContentParser parser) throws I } } - public static class UploadedMetadataAttribute implements Writeable, ToXContentFragment { + public static class UploadedMetadataAttribute implements UploadedMetadata, Writeable, ToXContentFragment { private static final ParseField UPLOADED_FILENAME_FIELD = new ParseField("uploaded_filename"); private static final ObjectParser.NamedObjectParser PARSER; @@ -807,6 +860,11 @@ public String getAttributeName() { return attributeName; } + @Override + public String getComponent() { + return getAttributeName(); + } + public String getUploadedFilename() { return uploadedFilename; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 13a1a565886fa..56cc72a07cbda 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -18,6 +18,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; @@ -29,9 +30,9 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.transfer.BlobStoreTransferService; @@ -138,8 +139,8 @@ public class RemoteClusterStateService implements Closeable { Settings::fromXContent ); - public static final ChecksumBlobStoreFormat TEMPLATE_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "template", + public static final ChecksumBlobStoreFormat TEMPLATES_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "templates", METADATA_NAME_FORMAT, TemplatesMetadata::fromXContent ); @@ -150,8 +151,6 @@ public class RemoteClusterStateService implements Closeable { Metadata.Custom::fromXContent ); - private static final Map metadataComponentBlobStoreMap = new HashMap<>(); - /** * Manifest format compatible with older codec v0, where codec version was missing. */ @@ -159,7 +158,13 @@ public class RemoteClusterStateService implements Closeable { new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV0); /** - * Manifest format compatible with codec v1, where we introduced codec versions/global metadata. + * Manifest format compatible with older codec v1, where global metadata was missing. + */ + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V1 = + new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV1); + + /** + * Manifest format compatible with codec v2, where we introduced codec versions/global metadata. */ public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT = new ChecksumBlobStoreFormat<>( "cluster-metadata-manifest", @@ -185,7 +190,7 @@ public class RemoteClusterStateService implements Closeable { public static final String METADATA_FILE_PREFIX = "metadata"; public static final String COORDINATION_METADATA = "coordination"; public static final String SETTING_METADATA = "settings"; - public static final String TEMPLATE_METADATA = "template"; + public static final String TEMPLATES_METADATA = "templates"; public static final String CUSTOM_METADATA = "custom"; public static final int SPLITED_MANIFEST_FILE_LENGTH = 6; // file name manifest__term__version__C/P__timestamp__codecversion @@ -240,11 +245,6 @@ public RemoteClusterStateService( clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout); this.remoteStateStats = new RemotePersistenceStats(); - - metadataComponentBlobStoreMap.put("coordination", COORDINATION_METADATA_FORMAT); - metadataComponentBlobStoreMap.put("settings", SETTINGS_METADATA_FORMAT); - metadataComponentBlobStoreMap.put("templates", TEMPLATE_METADATA_FORMAT); - metadataComponentBlobStoreMap.put("customs", CUSTOM_METADATA_FORMAT); } private BlobStoreTransferService getBlobStoreTransferService() { @@ -268,26 +268,22 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri return null; } - // TODO: we can upload global metadata and index metadata in parallel. [issue: #10645] - // Write globalMetadata fragments - UploadedMetadataAttribute uploadedCoordinationMetadata = writeCoordinationMetadata(clusterState); - UploadedMetadataAttribute uploadedSettingsMetadata = writeSettingsMetadata(clusterState); - UploadedMetadataAttribute uploadedTemplateMetadata = writeTemplateMetadata(clusterState); - Map uploadedCustomMetadataMap = writeCustomMetadataInParallel(clusterState); - - // any validations before/after upload ? - final List allUploadedIndexMetadata = writeIndexMetadataParallel( + UploadedMetadataResults uploadedMetadataResults = writeMetadataInParallel( clusterState, - new ArrayList<>(clusterState.metadata().indices().values()) + new ArrayList<>(clusterState.metadata().indices().values()), + clusterState.metadata().customs(), + true, + true, + true ); final ClusterMetadataManifest manifest = uploadManifest( clusterState, - allUploadedIndexMetadata, + uploadedMetadataResults.uploadedIndexMetadata, previousClusterUUID, - uploadedCoordinationMetadata, - uploadedSettingsMetadata, - uploadedTemplateMetadata, - uploadedCustomMetadataMap, + uploadedMetadataResults.uploadedCoordinationMetadata, + uploadedMetadataResults.uploadedSettingsMetadata, + uploadedMetadataResults.uploadedTemplatesMetadata, + uploadedMetadataResults.uploadedCustomMetadataMap, false ); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); @@ -298,13 +294,13 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri "writing cluster state took [{}ms] which is above the warn threshold of [{}]; " + "wrote full state with [{}] indices", durationMillis, slowWriteLoggingThreshold, - allUploadedIndexMetadata.size() + uploadedMetadataResults.uploadedIndexMetadata.size() ); } else { logger.info( "writing cluster state took [{}ms]; " + "wrote full state with [{}] indices and global metadata", durationMillis, - allUploadedIndexMetadata.size() + uploadedMetadataResults.uploadedIndexMetadata.size() ); } return manifest; @@ -335,24 +331,17 @@ public ClusterMetadataManifest writeIncrementalMetadata( previousClusterState.metadata(), clusterState.metadata() ) == false; - UploadedMetadataAttribute uploadedCoordinationMetadata; - UploadedMetadataAttribute uploadedSettingsMetadata; - UploadedMetadataAttribute uploadedTemplateMetadata; - Map uploadedCustomMetadataMap; - // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, - // If file is empty and codec is 1 then write global metadata. - if (updateGlobalMetadata || !previousManifest.hasMetadataAttributesFiles()) { - uploadedCoordinationMetadata = writeCoordinationMetadata(clusterState); - uploadedSettingsMetadata = writeSettingsMetadata(clusterState); - uploadedTemplateMetadata = writeTemplateMetadata(clusterState); - uploadedCustomMetadataMap = writeCustomMetadataInParallel(clusterState); - } else { - logger.debug("Global metadata has not updated in cluster state, skipping upload of it"); - uploadedCoordinationMetadata = previousManifest.getCoordinationMetadata(); - uploadedSettingsMetadata = previousManifest.getSettingsMetadata(); - uploadedTemplateMetadata = previousManifest.getTemplatesMetadata(); - uploadedCustomMetadataMap = previousManifest.getCustomMetadataMap(); - } + + final boolean updateCoordinationMetadata = Metadata.isCoordinationMetadataEqual( + previousClusterState.metadata(), clusterState.metadata() + ) == false; + final boolean updateSettingsMetadata = Metadata.isSettingsMetadataEqual( + previousClusterState.metadata(), clusterState.metadata() + ) == false; + final boolean updateTemplatesMetadata = Metadata.isTemplatesMetadataEqual( + previousClusterState.metadata(), clusterState.metadata() + ) == false; + final Map customsToUpload = getUpdatedCustoms(clusterState, previousClusterState); // Write Index Metadata final Map previousStateIndexMetadataVersionByName = new HashMap<>(); @@ -384,9 +373,31 @@ public ClusterMetadataManifest writeIncrementalMetadata( } previousStateIndexMetadataVersionByName.remove(indexMetadata.getIndex().getName()); } + UploadedMetadataResults uploadedMetadataResults; + boolean firstUpload = !previousManifest.hasMetadataAttributesFiles(); + // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, + // If file is empty and codec is 1 then write global metadata. + if (firstUpload) { + uploadedMetadataResults = writeMetadataInParallel( + clusterState, + toUpload, + clusterState.metadata().customs(), + true, + true, + true + ); + } else { + uploadedMetadataResults = writeMetadataInParallel( + clusterState, + toUpload, + customsToUpload, + updateCoordinationMetadata, + updateSettingsMetadata, + updateTemplatesMetadata + ); + } - List uploadedIndexMetadataList = writeIndexMetadataParallel(clusterState, toUpload); - uploadedIndexMetadataList.forEach( + uploadedMetadataResults.uploadedIndexMetadata.forEach( uploadedIndexMetadata -> allUploadedIndexMetadata.put(uploadedIndexMetadata.getIndexName(), uploadedIndexMetadata) ); @@ -397,10 +408,10 @@ public ClusterMetadataManifest writeIncrementalMetadata( clusterState, new ArrayList<>(allUploadedIndexMetadata.values()), previousManifest.getPreviousClusterUUID(), - uploadedCoordinationMetadata, - uploadedSettingsMetadata, - uploadedTemplateMetadata, - uploadedCustomMetadataMap, + firstUpload || updateCoordinationMetadata? uploadedMetadataResults.uploadedCoordinationMetadata : previousManifest.getCoordinationMetadata(), + firstUpload || updateSettingsMetadata ? uploadedMetadataResults.uploadedSettingsMetadata : previousManifest.getSettingsMetadata(), + firstUpload || updateTemplatesMetadata ? uploadedMetadataResults.uploadedTemplatesMetadata : previousManifest.getTemplatesMetadata(), + firstUpload || !customsToUpload.isEmpty() ? uploadedMetadataResults.uploadedCustomMetadataMap : previousManifest.getCustomMetadataMap(), false ); deleteStaleClusterMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), RETAINED_MANIFESTS); @@ -411,344 +422,142 @@ public ClusterMetadataManifest writeIncrementalMetadata( if (durationMillis >= slowWriteLoggingThreshold.getMillis()) { logger.warn( "writing cluster state took [{}ms] which is above the warn threshold of [{}]; " - + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, global metadata updated : [{}]", - durationMillis, - slowWriteLoggingThreshold, - numIndicesUpdated, - numIndicesUnchanged, - updateGlobalMetadata + + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, coordination metadata updated : [{}], " + + "settings metadata updated : [{}], templates metadata updated : [{}], custom metadata updated : [{}]", + durationMillis, + slowWriteLoggingThreshold, + numIndicesUpdated, + numIndicesUnchanged, + updateCoordinationMetadata, + updateSettingsMetadata, + updateTemplatesMetadata, + customsToUpload.size() ); } else { logger.info( "writing cluster state for version [{}] took [{}ms]; " - + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, global metadata updated : [{}]", - manifest.getStateVersion(), - durationMillis, - numIndicesUpdated, - numIndicesUnchanged, - updateGlobalMetadata + + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, coordination metadata updated : [{}], " + + "settings metadata updated : [{}], templates metadata updated : [{}], custom metadata updated : [{}]", + manifest.getStateVersion(), + durationMillis, + numIndicesUpdated, + numIndicesUnchanged, + updateCoordinationMetadata, + updateSettingsMetadata, + updateTemplatesMetadata, + customsToUpload.size() ); } return manifest; } - /** - * Uploads provided ClusterState's global Metadata to remote store in parallel. - * The call is blocking so the method waits for upload to finish and then return. - * - * @param clusterState current ClusterState - * @return String file name where globalMetadata file is stored. - */ - private String writeGlobalMetadata(ClusterState clusterState) throws IOException { - - AtomicReference result = new AtomicReference(); - AtomicReference exceptionReference = new AtomicReference(); - - final BlobContainer globalMetadataContainer = globalMetadataContainer( - clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID() - ); - final String globalMetadataFilename = globalMetadataFileName(clusterState.metadata()); - - // latch to wait until upload is not finished - CountDownLatch latch = new CountDownLatch(1); - - LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap(resp -> { - logger.trace(String.format(Locale.ROOT, "GlobalMetadata uploaded successfully.")); - result.set(globalMetadataContainer.path().buildAsString() + globalMetadataFilename); - }, ex -> { exceptionReference.set(ex); }), latch); - - GLOBAL_METADATA_FORMAT.writeAsyncWithUrgentPriority( - clusterState.metadata(), - globalMetadataContainer, - globalMetadataFilename, - blobStoreRepository.getCompressor(), - completionListener, - FORMAT_PARAMS - ); - - try { - if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { - // TODO: We should add metrics where transfer is timing out. [Issue: #10687] - RemoteStateTransferException ex = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of global metadata to complete") - ); - throw ex; - } - } catch (InterruptedException ex) { - RemoteStateTransferException exception = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of global metadata to complete - %s"), - ex - ); - Thread.currentThread().interrupt(); - throw exception; - } - if (exceptionReference.get() != null) { - throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); - } - return result.get(); - } - - private UploadedMetadataAttribute writeCoordinationMetadata(ClusterState clusterState) throws IOException { - AtomicReference result = new AtomicReference<>(); - AtomicReference exceptionReference = new AtomicReference<>(); - CountDownLatch latch = new CountDownLatch(1); - - LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap( - (UploadedMetadataAttribute uploadedMetadataAttribute) -> { - logger.trace(String.format(Locale.ROOT, "CoordinationMetadata uploaded successfully.")); - result.set(uploadedMetadataAttribute); + private UploadedMetadataResults writeMetadataInParallel( + ClusterState clusterState, + List indexToUpload, + Map customToUpload, + boolean uploadCoordinationMetadata, + boolean uploadSettingsMetadata, + boolean uploadTemplateMetadata + ) throws IOException { + int totalUploadTasks = indexToUpload.size() + customToUpload.size() + + (uploadCoordinationMetadata ? 1 : 0) + + (uploadSettingsMetadata ? 1 : 0) + + (uploadTemplateMetadata ? 1 : 0); + CountDownLatch latch = new CountDownLatch(totalUploadTasks); + List> uploadTasks = new ArrayList<>(totalUploadTasks); + Map results = new HashMap<>(totalUploadTasks); + List exceptionList = Collections.synchronizedList(new ArrayList<>(totalUploadTasks)); + + LatchedActionListener listener = new LatchedActionListener<>(ActionListener.wrap( + (ClusterMetadataManifest.UploadedMetadata uploadedMetadata) -> { + logger.trace(String.format(Locale.ROOT, "Metadata component %s uploaded successfully.", uploadedMetadata.getComponent())); + results.put(uploadedMetadata.getComponent(), uploadedMetadata); }, ex -> { logger.error( () -> new ParameterizedMessage("Exception during transfer of Metadata Fragment to Remote {}", ex.getMessage()), ex ); - exceptionReference.set(ex); + exceptionList.add(ex); }), latch); - writeMetadataComponentAsync(clusterState, COORDINATION_METADATA, COORDINATION_METADATA_FORMAT, clusterState.metadata().coordinationMetadata(), completionListener); - - try { - if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { - RemoteStateTransferException ex = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of coordination metadata to complete") - ); - throw ex; - } - } catch (InterruptedException ex) { - RemoteStateTransferException exception = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of coordination metadata to complete - %s"), - ex - ); - Thread.currentThread().interrupt(); - throw exception; + if (uploadSettingsMetadata) { + uploadTasks.add(getAsyncMetadataWriteAction(clusterState, SETTING_METADATA, SETTINGS_METADATA_FORMAT, clusterState.metadata().persistentSettings(), listener)); } - if (exceptionReference.get() != null) { - throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); + if (uploadCoordinationMetadata) { + uploadTasks.add(getAsyncMetadataWriteAction(clusterState, COORDINATION_METADATA, COORDINATION_METADATA_FORMAT, clusterState.metadata().coordinationMetadata(), listener)); } - return result.get(); - } - - private UploadedMetadataAttribute writeSettingsMetadata(ClusterState clusterState) throws IOException { - AtomicReference result = new AtomicReference<>(); - AtomicReference exceptionReference = new AtomicReference<>(); - CountDownLatch latch = new CountDownLatch(1); - - LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap( - (UploadedMetadataAttribute uploadedMetadataAttribute) -> { - logger.trace(String.format(Locale.ROOT, "Settings Metadata uploaded successfully.")); - result.set(uploadedMetadataAttribute); - }, ex -> { - logger.error( - () -> new ParameterizedMessage("Exception during transfer of Metadata Fragment to Remote {}", ex.getMessage()), - ex - ); - exceptionReference.set(ex); - }), - latch); - - writeMetadataComponentAsync(clusterState, SETTING_METADATA, SETTINGS_METADATA_FORMAT, clusterState.metadata().persistentSettings(), completionListener); - - try { - if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { - RemoteStateTransferException ex = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of settings metadata to complete") - ); - throw ex; - } - } catch (InterruptedException ex) { - RemoteStateTransferException exception = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of settings metadata to complete - %s"), - ex - ); - Thread.currentThread().interrupt(); - throw exception; - } - if (exceptionReference.get() != null) { - throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); + if (uploadTemplateMetadata) { + uploadTasks.add(getAsyncMetadataWriteAction(clusterState, TEMPLATES_METADATA, TEMPLATES_METADATA_FORMAT, clusterState.metadata().templatesMetadata(), listener)); } - return result.get(); - } - - private UploadedMetadataAttribute writeTemplateMetadata(ClusterState clusterState) throws IOException { - AtomicReference result = new AtomicReference<>(); - AtomicReference exceptionReference = new AtomicReference<>(); - CountDownLatch latch = new CountDownLatch(1); - - LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap( - (UploadedMetadataAttribute uploadedMetadataAttribute) -> { - logger.trace(String.format(Locale.ROOT, "Templates Metadata uploaded successfully.")); - result.set(uploadedMetadataAttribute); - }, ex -> { - logger.error( - () -> new ParameterizedMessage("Exception during transfer of Metadata Fragment to Remote {}", ex.getMessage()), - ex - ); - exceptionReference.set(ex); - }), - latch); - - writeMetadataComponentAsync(clusterState, TEMPLATE_METADATA, TEMPLATE_METADATA_FORMAT, clusterState.metadata().templatesMetadata(), completionListener); - - try { - if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { - RemoteStateTransferException ex = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of templates metadata to complete") - ); - throw ex; - } - } catch (InterruptedException ex) { - RemoteStateTransferException exception = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of templates metadata to complete - %s"), - ex - ); - Thread.currentThread().interrupt(); - throw exception; - } - if (exceptionReference.get() != null) { - throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); - } - return result.get(); - } - - private Map writeCustomMetadataInParallel(ClusterState clusterState) throws IOException { - Map result = new HashMap<>(); - List exceptionReference = new ArrayList<>(); - - Map customToUpload = clusterState.metadata().customs(); - - final CountDownLatch latch = new CountDownLatch(customToUpload.size()); - - LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap( - (UploadedMetadataAttribute uploadedMetadataAttribute) -> { - logger.trace(String.format(Locale.ROOT, "Custom Metadata uploaded successfully.")); - result.put(uploadedMetadataAttribute.getAttributeName(), uploadedMetadataAttribute); - }, ex -> { - logger.error( - () -> new ParameterizedMessage("Exception during transfer of Metadata Fragment to Remote {}", ex.getMessage()), - ex - ); - exceptionReference.add(ex); - }), - latch - ); + customToUpload.forEach((key, value) -> uploadTasks.add( + getAsyncMetadataWriteAction( + clusterState, + String.join(DELIMITER, CUSTOM_METADATA, key), + CUSTOM_METADATA_FORMAT, + value, + listener + ) + )); + indexToUpload.forEach(indexMetadata -> { + uploadTasks.add(getIndexMetadataAsyncAction(clusterState, indexMetadata, listener)); + }); - for (Map.Entry custom : customToUpload.entrySet()) { - writeMetadataComponentAsync(clusterState, custom.getKey(), CUSTOM_METADATA_FORMAT, custom.getValue(), completionListener); + // start async upload of all required metadata files + for(CheckedRunnable uploadTask : uploadTasks) { + uploadTask.run(); } try { if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { + // TODO: We should add metrics where transfer is timing out. [Issue: #10687] RemoteStateTransferException ex = new RemoteStateTransferException( - String.format( - Locale.ROOT, - "Timed out waiting for transfer of custom metadata to complete - %s", - String.join(", ", customToUpload.keySet()) - ) + String.format(Locale.ROOT, "Timed out waiting for transfer of metadata to complete") ); - exceptionReference.forEach(ex::addSuppressed); throw ex; } } catch (InterruptedException ex) { - exceptionReference.forEach(ex::addSuppressed); RemoteStateTransferException exception = new RemoteStateTransferException( - String.format( - Locale.ROOT, - "Timed out waiting for transfer of custom metadata to complete - %s", - String.join(", ", customToUpload.keySet()) - ), + String.format(Locale.ROOT, "Timed out waiting for transfer of metadata to complete - %s"), ex ); Thread.currentThread().interrupt(); throw exception; } - if (!exceptionReference.isEmpty()) { + if (!exceptionList.isEmpty()) { RemoteStateTransferException exception = new RemoteStateTransferException( String.format( Locale.ROOT, - "Exception during transfer of custom metadata to complete - %s", - String.join(", ", customToUpload.keySet()) + "Exception during transfer of following metadata to Remote - %s, %s, %s", + indexToUpload.stream().map(IndexMetadata::getIndexName).collect(Collectors.joining(",")), + customToUpload.keySet().stream().collect(Collectors.joining(", ")), + String.join(", ", (uploadSettingsMetadata ? "settings" : ""), (uploadCoordinationMetadata ? + "coordination" : ""), (uploadTemplateMetadata ? "templates" : "")) ) ); - exceptionReference.forEach(exception::addSuppressed); + exceptionList.forEach(exception::addSuppressed); throw exception; } - return result; - } - - /** - * Uploads provided IndexMetadata's to remote store in parallel. The call is blocking so the method waits for upload to finish and then return. - * - * @param clusterState current ClusterState - * @param toUpload list of IndexMetadata to upload - * @return {@code List} list of IndexMetadata uploaded to remote - */ - private List writeIndexMetadataParallel(ClusterState clusterState, List toUpload) - throws IOException { - List exceptionList = Collections.synchronizedList(new ArrayList<>(toUpload.size())); - final CountDownLatch latch = new CountDownLatch(toUpload.size()); - List result = new ArrayList<>(toUpload.size()); - - LatchedActionListener latchedActionListener = new LatchedActionListener<>( - ActionListener.wrap((UploadedIndexMetadata uploadedIndexMetadata) -> { - logger.trace( - String.format(Locale.ROOT, "IndexMetadata uploaded successfully for %s", uploadedIndexMetadata.getIndexName()) - ); - result.add(uploadedIndexMetadata); - }, ex -> { - assert ex instanceof RemoteStateTransferException; - logger.error( - () -> new ParameterizedMessage("Exception during transfer of IndexMetadata to Remote {}", ex.getMessage()), - ex - ); - exceptionList.add(ex); - }), - latch - ); - - for (IndexMetadata indexMetadata : toUpload) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/index/ftqsCnn9TgOX/metadata_4_1690947200 - writeIndexMetadataAsync(clusterState, indexMetadata, latchedActionListener); - } - - try { - if (latch.await(getIndexMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { - RemoteStateTransferException ex = new RemoteStateTransferException( - String.format( - Locale.ROOT, - "Timed out waiting for transfer of index metadata to complete - %s", - toUpload.stream().map(IndexMetadata::getIndex).map(Index::toString).collect(Collectors.joining("")) - ) - ); - exceptionList.forEach(ex::addSuppressed); - throw ex; + UploadedMetadataResults response = new UploadedMetadataResults(); + for (Map.Entry entry : results.entrySet()) { + final String name = entry.getKey(); + final ClusterMetadataManifest.UploadedMetadata uploadedMetadata = entry.getValue(); + if (uploadedMetadata.getClass().equals(UploadedIndexMetadata.class)) { + response.uploadedIndexMetadata.add((UploadedIndexMetadata) uploadedMetadata); + } else if (uploadedMetadata.getComponent().contains(CUSTOM_METADATA)) { + // component name for custom metadata will look like custom__ + response.uploadedCustomMetadataMap.put(name.split(DELIMITER)[1], (UploadedMetadataAttribute) uploadedMetadata); + } else if (COORDINATION_METADATA.equals(uploadedMetadata.getComponent())) { + response.uploadedCoordinationMetadata = (UploadedMetadataAttribute) uploadedMetadata; + } else if (SETTING_METADATA.equals(uploadedMetadata.getComponent())) { + response.uploadedSettingsMetadata = (UploadedMetadataAttribute) uploadedMetadata; + } else if (TEMPLATES_METADATA.equals(uploadedMetadata.getComponent())) { + response.uploadedTemplatesMetadata = (UploadedMetadataAttribute) uploadedMetadata; + } else { + throw new IllegalStateException("Unexpected metadata component " + uploadedMetadata.getComponent()); } - } catch (InterruptedException ex) { - exceptionList.forEach(ex::addSuppressed); - RemoteStateTransferException exception = new RemoteStateTransferException( - String.format( - Locale.ROOT, - "Timed out waiting for transfer of index metadata to complete - %s", - toUpload.stream().map(IndexMetadata::getIndex).map(Index::toString).collect(Collectors.joining("")) - ), - ex - ); - Thread.currentThread().interrupt(); - throw exception; - } - if (exceptionList.size() > 0) { - RemoteStateTransferException exception = new RemoteStateTransferException( - String.format( - Locale.ROOT, - "Exception during transfer of IndexMetadata to Remote %s", - toUpload.stream().map(IndexMetadata::getIndex).map(Index::toString).collect(Collectors.joining("")) - ) - ); - exceptionList.forEach(exception::addSuppressed); - throw exception; } - return result; + return response; } /** @@ -758,11 +567,11 @@ private List writeIndexMetadataParallel(ClusterState clus * @param indexMetadata {@link IndexMetadata} to upload * @param latchedActionListener listener to respond back on after upload finishes */ - private void writeIndexMetadataAsync( + private CheckedRunnable getIndexMetadataAsyncAction( ClusterState clusterState, IndexMetadata indexMetadata, - LatchedActionListener latchedActionListener - ) throws IOException { + LatchedActionListener latchedActionListener + ) { final BlobContainer indexMetadataContainer = indexMetadataContainer( clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), @@ -780,7 +589,7 @@ private void writeIndexMetadataAsync( ex -> latchedActionListener.onFailure(new RemoteStateTransferException(indexMetadata.getIndex().toString(), ex)) ); - INDEX_METADATA_FORMAT.writeAsyncWithUrgentPriority( + return () -> INDEX_METADATA_FORMAT.writeAsyncWithUrgentPriority( indexMetadata, indexMetadataContainer, indexMetadataFilename, @@ -794,13 +603,13 @@ private void writeIndexMetadataAsync( * Allows async upload of Metadata components to remote */ - private void writeMetadataComponentAsync( + private CheckedRunnable getAsyncMetadataWriteAction( ClusterState clusterState, String component, ChecksumBlobStoreFormat componentMetadataBlobStore, ToXContent componentMetadata, - LatchedActionListener latchedActionListener - ) throws IOException { + LatchedActionListener latchedActionListener + ) { final BlobContainer globalMetadataContainer = globalMetadataContainer( clusterState.getClusterName().value(), clusterState.metadata().clusterUUID() @@ -815,8 +624,7 @@ private void writeMetadataComponentAsync( ), ex -> latchedActionListener.onFailure(new RemoteStateTransferException(component, ex)) ); - - componentMetadataBlobStore.writeAsyncWithUrgentPriority( + return () -> componentMetadataBlobStore.writeAsyncWithUrgentPriority( componentMetadata, globalMetadataContainer, componentMetadataFilename, @@ -867,18 +675,56 @@ public void start() { blobStoreRepository = (BlobStoreRepository) repository; } + private ClusterMetadataManifest uploadV1Manifest( + ClusterState clusterState, + List uploadedIndexMetadata, + String previousClusterUUID, + String globalMetadataFileName, + boolean committed + ) throws IOException { + synchronized (this) { + final String manifestFileName = getManifestFileName( + clusterState.term(), + clusterState.version(), + committed, + ClusterMetadataManifest.CODEC_V1 + ); + final ClusterMetadataManifest manifest = new ClusterMetadataManifest( + clusterState.term(), + clusterState.getVersion(), + clusterState.metadata().clusterUUID(), + clusterState.stateUUID(), + Version.CURRENT, + nodeId, + committed, + ClusterMetadataManifest.CODEC_V1, + globalMetadataFileName, + uploadedIndexMetadata, + previousClusterUUID, + clusterState.metadata().clusterUUIDCommitted() + ); + writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); + return manifest; + } + } + private ClusterMetadataManifest uploadManifest( ClusterState clusterState, List uploadedIndexMetadata, String previousClusterUUID, UploadedMetadataAttribute uploadedCoordinationMetadata, UploadedMetadataAttribute uploadedSettingsMetadata, - UploadedMetadataAttribute uploadedTemplateMetadata, + UploadedMetadataAttribute uploadedTemplatesMetadata, Map uploadedCustomMetadataMap, boolean committed ) throws IOException { synchronized (this) { - final String manifestFileName = getManifestFileName(clusterState.term(), clusterState.version(), committed); + final String manifestFileName = getManifestFileName( + clusterState.term(), + clusterState.version(), + committed, + MANIFEST_CURRENT_CODEC_VERSION + ); final ClusterMetadataManifest manifest = new ClusterMetadataManifest( clusterState.term(), clusterState.getVersion(), @@ -894,7 +740,7 @@ private ClusterMetadataManifest uploadManifest( clusterState.metadata().clusterUUIDCommitted(), uploadedCoordinationMetadata, uploadedSettingsMetadata, - uploadedTemplateMetadata, + uploadedTemplatesMetadata, uploadedCustomMetadataMap ); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); @@ -916,7 +762,7 @@ private void writeMetadataManifest(String clusterName, String clusterUUID, Clust logger.trace(String.format(Locale.ROOT, "Manifest file uploaded successfully.")); }, ex -> { exceptionReference.set(ex); }), latch); - CLUSTER_METADATA_MANIFEST_FORMAT.writeAsyncWithUrgentPriority( + getClusterMetadataManifestBlobStoreFormat(fileName).writeAsyncWithUrgentPriority( uploadManifest, metadataManifestContainer, fileName, @@ -1006,6 +852,29 @@ private void setMetadataManifestUploadTimeout(TimeValue newMetadataManifestUploa this.metadataManifestUploadTimeout = newMetadataManifestUploadTimeout; } + private Map getUpdatedCustoms(ClusterState currentState, ClusterState previousState) { + if(Metadata.isCustomMetadataEqual(previousState.metadata(), currentState.metadata())) { + return new HashMap<>(); + } + Map updatedCustom = new HashMap<>(); + Set currentCustoms = new HashSet<>(currentState.metadata().customs().keySet()); + for (Map.Entry cursor : previousState.metadata().customs().entrySet()) { + if (cursor.getValue().context().contains(Metadata.XContentContext.GATEWAY)) { + if (!cursor.getValue().equals(currentState.metadata().custom(cursor.getKey()))) { + updatedCustom.put(cursor.getKey(), currentState.metadata().custom(cursor.getKey())); + } + currentCustoms.remove(cursor.getKey()); + } + } + for (String custom : currentCustoms) { + Metadata.Custom cursor = currentState.metadata().custom(custom); + if (cursor.context().contains(Metadata.XContentContext.GATEWAY)) { + updatedCustom.put(custom, cursor); + } + } + return updatedCustom; + } + public TimeValue getIndexMetadataUploadTimeout() { return this.indexMetadataUploadTimeout; } @@ -1018,7 +887,7 @@ public TimeValue getMetadataManifestUploadTimeout() { return this.metadataManifestUploadTimeout; } - static String getManifestFileName(long term, long version, boolean committed) { + static String getManifestFileName(long term, long version, boolean committed, int codecVersion) { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____ return String.join( DELIMITER, @@ -1027,7 +896,7 @@ static String getManifestFileName(long term, long version, boolean committed) { RemoteStoreUtils.invertLong(version), (committed ? "C" : "P"), // C for committed and P for published RemoteStoreUtils.invertLong(System.currentTimeMillis()), - String.valueOf(MANIFEST_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during read we reads last place to + String.valueOf(codecVersion) // Keep the codec version at last place only, during read we reads last place to // determine codec version. ); } @@ -1173,8 +1042,10 @@ private Metadata getGlobalMetadata(String clusterName, String clusterUUID, Clust builder.persistentSettings(settingsMetadata); builder.templates(templatesMetadata); clusterMetadataManifest.getCustomMetadataMap().forEach( - (key, value) -> - builder.putCustom(key, getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), key)) + (key, value) -> { + String custom = key.split(DELIMITER)[1]; + builder.putCustom(custom, getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), custom)); + } ); return builder.build(); } else { @@ -1235,7 +1106,7 @@ private TemplatesMetadata getTemplatesMetadata(String clusterName, String cluste // Fetch Templates metadata if (templatesMetadataFileName != null) { String[] splitPath = templatesMetadataFileName.split("/"); - return TEMPLATE_METADATA_FORMAT.read( + return TEMPLATES_METADATA_FORMAT.read( globalMetadataContainer(clusterName, clusterUUID), splitPath[splitPath.length - 1], blobStoreRepository.getNamedXContentRegistry() @@ -1515,6 +1386,8 @@ private ChecksumBlobStoreFormat getClusterMetadataManif long codecVersion = getManifestCodecVersion(fileName); if (codecVersion == MANIFEST_CURRENT_CODEC_VERSION) { return CLUSTER_METADATA_MANIFEST_FORMAT; + } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { + return CLUSTER_METADATA_MANIFEST_FORMAT_V1; } else if (codecVersion == ClusterMetadataManifest.CODEC_V0) { return CLUSTER_METADATA_MANIFEST_FORMAT_V0; } @@ -1736,4 +1609,33 @@ public void deleteStaleClusterUUIDs(ClusterState clusterState, ClusterMetadataMa public RemotePersistenceStats getStats() { return remoteStateStats; } + + private class UploadedMetadataResults { + List uploadedIndexMetadata; + Map uploadedCustomMetadataMap; + UploadedMetadataAttribute uploadedCoordinationMetadata; + UploadedMetadataAttribute uploadedSettingsMetadata; + UploadedMetadataAttribute uploadedTemplatesMetadata; + + public UploadedMetadataResults( + List uploadedIndexMetadata, + Map uploadedCustomMetadataMap, + UploadedMetadataAttribute uploadedCoordinationMetadata, + UploadedMetadataAttribute uploadedSettingsMetadata, + UploadedMetadataAttribute uploadedTemplatesMetadata + ) { + this.uploadedIndexMetadata = uploadedIndexMetadata; + this.uploadedCustomMetadataMap = uploadedCustomMetadataMap; + this.uploadedCoordinationMetadata = uploadedCoordinationMetadata; + this.uploadedSettingsMetadata = uploadedSettingsMetadata; + this.uploadedTemplatesMetadata = uploadedTemplatesMetadata; + } + public UploadedMetadataResults() { + this.uploadedIndexMetadata = new ArrayList<>(); + this.uploadedCustomMetadataMap = new HashMap<>(); + this.uploadedCoordinationMetadata = null; + this.uploadedSettingsMetadata = null; + this.uploadedTemplatesMetadata = null; + } + } }