diff --git a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java index ed4b94d2cc5..4c7f98c6ccd 100644 --- a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java +++ b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java @@ -573,6 +573,9 @@ public static void main(String[] args) throws Exception { case DUMP_TOPIC_PARTITION_INGESTION_CONTEXT: dumpTopicPartitionIngestionContext(cmd); break; + case MIGRATE_VENICE_ZK_PATHS: + migrateVeniceZKPaths(cmd); + break; case EXTRACT_VENICE_ZK_PATHS: extractVeniceZKPaths(cmd); break; @@ -3094,6 +3097,22 @@ private static void dumpTopicPartitionIngestionContext(CommandLine cmd) throws E } } + private static void migrateVeniceZKPaths(CommandLine cmd) throws Exception { + Set clusterNames = Utils.parseCommaSeparatedStringToSet(getRequiredArgument(cmd, Arg.CLUSTER_LIST)); + String srcZKUrl = getRequiredArgument(cmd, Arg.SRC_ZOOKEEPER_URL); + String srcZKSSLConfigs = getRequiredArgument(cmd, Arg.SRC_ZK_SSL_CONFIG_FILE); + String destZKUrl = getRequiredArgument(cmd, Arg.DEST_ZOOKEEPER_URL); + String destZKSSLConfigs = getRequiredArgument(cmd, Arg.DEST_ZK_SSL_CONFIG_FILE); + ZkClient srcZkClient = readZKConfigAndBuildZKClient(srcZKUrl, srcZKSSLConfigs); + ZkClient destZkClient = readZKConfigAndBuildZKClient(destZKUrl, destZKSSLConfigs); + try { + ZkCopier.migrateVenicePaths(srcZkClient, destZkClient, clusterNames, getRequiredArgument(cmd, Arg.BASE_PATH)); + } finally { + srcZkClient.close(); + destZkClient.close(); + } + } + private static void extractVeniceZKPaths(CommandLine cmd) { Set clusterNames = Utils.parseCommaSeparatedStringToSet(getRequiredArgument(cmd, Arg.CLUSTER_LIST)); ZkCopier.extractVenicePaths( diff --git a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Arg.java b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Arg.java index d100dca6404..cad43d49b65 100644 --- a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Arg.java +++ b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Arg.java @@ -11,6 +11,8 @@ public enum Arg { URL("url", "u", true, "Venice url, eg. http://localhost:1689 This can be a router or a controller"), SERVER_URL("server-url", "su", true, "Venice server url, eg. http://localhost:1690 This has to be a storage node"), VENICE_ZOOKEEPER_URL("venice-zookeeper-url", "vzu", true, "Venice Zookeeper url, eg. localhost:2622"), + SRC_ZOOKEEPER_URL("src-zookeeper-url", "szu", true, "Source Zookeeper url, eg. localhost:2181"), + DEST_ZOOKEEPER_URL("dest-zookeeper-url", "dzu", true, "Destination Zookeeper url, eg. localhost:2182"), INFILE("infile", "if", true, "Path to input text file"), OUTFILE("outfile", "of", true, "Path to output text file"), BASE_PATH("base-path", "bp", true, "Base path for ZK, eg. /venice-parent"), CLUSTER("cluster", "c", true, "Name of Venice cluster"), @@ -27,7 +29,10 @@ public enum Arg { VALUE_SCHEMA_ID("value-schema-id", "vid", true, "value schema id"), VALUE_SCHEMA("value-schema-file", "vs", true, "Path to text file with value schema"), ZK_SSL_CONFIG_FILE("zk-ssl-config-file", "zscf", true, "Path to text file with ZK SSL configs"), - DERIVED_SCHEMA_ID("derived-schema-id", "did", true, "derived schema id"), + SRC_ZK_SSL_CONFIG_FILE("src-zk-ssl-config-file", "szscf", true, "Path to text file with source ZK SSL configs"), + DEST_ZK_SSL_CONFIG_FILE( + "dest-zk-ssl-config-file", "dzscf", true, "Path to text file with destination ZK SSL configs" + ), DERIVED_SCHEMA_ID("derived-schema-id", "did", true, "derived schema id"), DERIVED_SCHEMA("derived-schema-file", "ds", true, "Path to text file with derived schema"), OWNER("owner", "o", true, "Owner email for new store creation"), STORAGE_NODE("storage-node", "n", true, "Helix instance ID for a storage node, eg. lva1-app1234_1690"), diff --git a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Command.java b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Command.java index 10d2e36a22e..024c397f342 100644 --- a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Command.java +++ b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Command.java @@ -25,6 +25,8 @@ import static com.linkedin.venice.Arg.DERIVED_SCHEMA; import static com.linkedin.venice.Arg.DERIVED_SCHEMA_ID; import static com.linkedin.venice.Arg.DEST_FABRIC; +import static com.linkedin.venice.Arg.DEST_ZK_SSL_CONFIG_FILE; +import static com.linkedin.venice.Arg.DEST_ZOOKEEPER_URL; import static com.linkedin.venice.Arg.DISABLE_DAVINCI_PUSH_STATUS_STORE; import static com.linkedin.venice.Arg.DISABLE_META_STORE; import static com.linkedin.venice.Arg.ENABLE_DISABLED_REPLICA; @@ -101,6 +103,8 @@ import static com.linkedin.venice.Arg.SKIP_DIV; import static com.linkedin.venice.Arg.SKIP_LAST_STORE_CREATION; import static com.linkedin.venice.Arg.SOURCE_FABRIC; +import static com.linkedin.venice.Arg.SRC_ZK_SSL_CONFIG_FILE; +import static com.linkedin.venice.Arg.SRC_ZOOKEEPER_URL; import static com.linkedin.venice.Arg.STARTING_OFFSET; import static com.linkedin.venice.Arg.START_DATE; import static com.linkedin.venice.Arg.STORAGE_NODE; @@ -535,6 +539,11 @@ public enum Command { "backup-store-metadata-from-graveyard", "Backup store metadata from graveyard in EI", new Arg[] { VENICE_ZOOKEEPER_URL, ZK_SSL_CONFIG_FILE, BACKUP_FOLDER } ), + MIGRATE_VENICE_ZK_PATHS( + "migrate-venice-zk-paths", "Migrate Venice-specific metadata from a source ZK to a destination ZK", + new Arg[] { SRC_ZOOKEEPER_URL, SRC_ZK_SSL_CONFIG_FILE, DEST_ZOOKEEPER_URL, DEST_ZK_SSL_CONFIG_FILE, CLUSTER_LIST, + BASE_PATH } + ), EXTRACT_VENICE_ZK_PATHS( "extract-venice-zk-paths", "Extract Venice-specific paths from a ZK snapshot input text file to an output text file", diff --git a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/ZkCopier.java b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/ZkCopier.java index 1b6bddcafdc..762f370ac05 100644 --- a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/ZkCopier.java +++ b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/ZkCopier.java @@ -1,31 +1,72 @@ package com.linkedin.venice; -import static com.linkedin.venice.zk.VeniceZkPaths.ADMIN_TOPIC_METADATA; -import static com.linkedin.venice.zk.VeniceZkPaths.EXECUTION_IDS; -import static com.linkedin.venice.zk.VeniceZkPaths.PARENT_OFFLINE_PUSHES; -import static com.linkedin.venice.zk.VeniceZkPaths.ROUTERS; -import static com.linkedin.venice.zk.VeniceZkPaths.STORES; +import static com.linkedin.venice.zk.VeniceZkPaths.CLUSTER_ZK_PATHS; import static com.linkedin.venice.zk.VeniceZkPaths.STORE_CONFIGS; -import static com.linkedin.venice.zk.VeniceZkPaths.STORE_GRAVEYARD; import com.linkedin.venice.exceptions.VeniceException; import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.commons.io.FileUtils; +import org.apache.helix.zookeeper.datamodel.serializer.ByteArraySerializer; +import org.apache.helix.zookeeper.impl.client.ZkClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.zookeeper.CreateMode; +/** + *

+ * This class contains methods to 1)migrate Venice-specific metadata from a source ZooKeeper (ZK) to a destination ZK + * and 2)extract Venice-specific paths from an input text file containing ZK paths to an output text file. + * We implement a tree data structure to represent ZK paths as nested children and use it to filter out Venice-specific + * paths efficiently. + *

+ */ public class ZkCopier { - private static final Set CLUSTER_ZK_PATHS = new HashSet<>( - Arrays.asList(ADMIN_TOPIC_METADATA, EXECUTION_IDS, PARENT_OFFLINE_PUSHES, ROUTERS, STORE_GRAVEYARD, STORES)); + private static final Logger LOGGER = LogManager.getLogger(ZkCopier.class); + + /** + * Migrate Venice-specific metadata from a source ZK to a destination ZK. + * @param srcZkClient source ZK client + * @param destZkClient destination ZK client + * @param clusterNames set of cluster names + * @param basePath base path for ZK + * @Note: {@code destZkClient} should be a fresh ZK server or must not contain any Venice-specific metadata that needs to be migrated, + * otherwise a {@code ZkNodeExistsException} will be thrown + */ + public static void migrateVenicePaths( + ZkClient srcZkClient, + ZkClient destZkClient, + Set clusterNames, + String basePath) { + if (!basePath.startsWith("/")) { + throw new VeniceException("Base path must start with a forward slash (/)"); + } + if (!srcZkClient.exists(basePath)) { + throw new VeniceException("Base path does not exist in source ZK"); + } + TreeNode srcZkPathsTree = getZkClientPathsTree(srcZkClient, clusterNames, basePath); + List destZkPathsList = pathsTreeToList(srcZkPathsTree); + srcZkClient.setZkSerializer(new ByteArraySerializer()); + destZkClient.setZkSerializer(new ByteArraySerializer()); + LOGGER.info("Starting Migration"); + for (String path: destZkPathsList) { + try { + destZkClient.create(path, srcZkClient.readData(path), CreateMode.PERSISTENT); + } catch (Exception e) { + LOGGER.error("Failed to create path: " + path, e); + throw new VeniceException(e); + } + } + LOGGER.info("Ending Migration"); + } /** * Extract Venice-specific paths from an input text file to an output text file. @@ -48,26 +89,35 @@ public static void extractVenicePaths( List zkPaths = FileUtils.readLines(inputFile); // write Venice-specific paths to output file - List venicePaths = getVenicePaths(zkPaths, clusterNames, basePath); + List venicePaths = getVenicePathsFromList(zkPaths, clusterNames, basePath); File outputFile = new File(outputPath); FileUtils.writeLines(outputFile, venicePaths); } catch (IOException e) { - throw new VeniceException(e.getMessage()); + throw new VeniceException(e); } } /** - * Get Venice-specific paths from a list of ZK paths filtered by 1)base path, 2)cluster names, and 3)required cluster ZK paths. + * Get Venice-specific paths from a list of ZK paths. * @return a list of Venice-specific paths filtered from {@code zkPaths} + * @Note: Converts the list {@code zkPaths} to a tree and calls {@code getVenicePathsFromTree()} */ - static List getVenicePaths(List zkPaths, Set clusterNames, String basePath) { - TreeNode requiredPathsTreeRoot = buildRequiredPathsTree(clusterNames, basePath); + static List getVenicePathsFromList(List zkPaths, Set clusterNames, String basePath) { TreeNode zkPathsTreeRoot = pathsListToTree(zkPaths, basePath); + return getVenicePathsFromTree(zkPathsTreeRoot, clusterNames, basePath); + } + + /** + * Get Venice-specific paths from a tree of ZK paths filtered by 1)base path, 2)cluster names, and 3)required cluster ZK paths. + * @return a list of Venice-specific paths filtered from {@code zkPathsTreeRoot} + */ + static List getVenicePathsFromTree(TreeNode zkPathsTreeRoot, Set clusterNames, String basePath) { + TreeNode requiredPathsTreeRoot = buildRequiredPathsTree(clusterNames, basePath); if (!zkPathsTreeRoot.getName().equals(requiredPathsTreeRoot.getName())) { throw new VeniceException( "Base path mismatch: " + zkPathsTreeRoot.getName() + " != " + requiredPathsTreeRoot.getName()); } - getVenicePathsHelper(zkPathsTreeRoot, requiredPathsTreeRoot); + getVenicePathsFromTreeHelper(zkPathsTreeRoot, requiredPathsTreeRoot); return pathsTreeToList(zkPathsTreeRoot); } @@ -75,9 +125,9 @@ static List getVenicePaths(List zkPaths, Set clusterName * Recursively match children nodes of {@code zkPathsTreeNode} and {@code requiredPathsTreeNode} and removes unmatched children nodes from {@code zkPathsTreeNode}. * @param zkPathsTreeNode node in the ZK paths tree * @param requiredPathsTreeNode node in the required paths tree - * @Note: This method directly modifies {@code zkPathsTreeNode} in the parent method {@code getVenicePaths()} + * @Note: This method directly modifies {@code zkPathsTreeNode} in the parent method {@code getVenicePathsFromTree()} */ - private static void getVenicePathsHelper(TreeNode zkPathsTreeNode, TreeNode requiredPathsTreeNode) { + private static void getVenicePathsFromTreeHelper(TreeNode zkPathsTreeNode, TreeNode requiredPathsTreeNode) { if (requiredPathsTreeNode.getChildren().isEmpty() || zkPathsTreeNode.getChildren().isEmpty()) { return; } @@ -87,7 +137,7 @@ private static void getVenicePathsHelper(TreeNode zkPathsTreeNode, TreeNode requ String zkChildName = zkEntry.getKey(); TreeNode zkChild = zkEntry.getValue(); if (requiredPathsTreeNode.containsChild(zkChildName)) { - getVenicePathsHelper(zkChild, requiredPathsTreeNode.getChildren().get(zkChildName)); + getVenicePathsFromTreeHelper(zkChild, requiredPathsTreeNode.getChildren().get(zkChildName)); } else { iterator.remove(); } @@ -110,6 +160,47 @@ static TreeNode buildRequiredPathsTree(Set clusterNames, String basePath return root; } + /** + * Build a tree of ZK paths with {@code basePath} and {@code clusterNames} by fetching children of each cluster path using the ZK client. + * @return the root (base path) of the tree + */ + static TreeNode getZkClientPathsTree(ZkClient zkClient, Set clusterNames, String basePath) { + TreeNode root = new TreeNode(basePath); + List basePathChildren = zkClient.getChildren(basePath); + if (basePathChildren.contains(STORE_CONFIGS)) { + TreeNode storeConfigsNode = root.addChild(STORE_CONFIGS); + String storeConfigsPath = basePath + "/" + STORE_CONFIGS; + addChildrenToTreeNode(zkClient, storeConfigsNode, storeConfigsPath); + } + for (String cluster: clusterNames) { + if (basePathChildren.contains(cluster)) { + TreeNode clusterNode = root.addChild(cluster); + String clusterPath = basePath + "/" + cluster; + List clusterChildren = zkClient.getChildren(clusterPath); + for (String venicePath: CLUSTER_ZK_PATHS) { + if (clusterChildren.contains(venicePath)) { + TreeNode venicePathNode = clusterNode.addChild(venicePath); + String clusterVenicePath = clusterPath + "/" + venicePath; + addChildrenToTreeNode(zkClient, venicePathNode, clusterVenicePath); + } + } + } + } + return root; + } + + /** + * Recursively add children to a {@code TreeNode} by fetching children of a ZK path using the ZK client. + * @Note: This method directly modifies the tree {@code node} in the parent method {@code getZkClientPathsTree()} + */ + private static void addChildrenToTreeNode(ZkClient zkClient, TreeNode node, String path) { + List children = zkClient.getChildren(path); + for (String child: children) { + TreeNode childNode = node.addChild(child); + addChildrenToTreeNode(zkClient, childNode, path + "/" + child); + } + } + /** * Convert a list of paths to a tree of paths as nested children. * @return the root of the tree diff --git a/clients/venice-admin-tool/src/test/java/com/linkedin/venice/TestZkCopier.java b/clients/venice-admin-tool/src/test/java/com/linkedin/venice/TestZkCopier.java index 83736fdd554..54ab7161866 100644 --- a/clients/venice-admin-tool/src/test/java/com/linkedin/venice/TestZkCopier.java +++ b/clients/venice-admin-tool/src/test/java/com/linkedin/venice/TestZkCopier.java @@ -54,9 +54,9 @@ public void testExtractVenicePaths() throws Exception { } @Test - public void testGetVenicePaths() { + public void testGetVenicePathsFromList() { List zkPaths = getPaths(); - List venicePaths = ZkCopier.getVenicePaths(zkPaths, CLUSTERS, BASE_PATH); + List venicePaths = ZkCopier.getVenicePathsFromList(zkPaths, CLUSTERS, BASE_PATH); Assert.assertEquals(venicePaths.size(), 22); Assert.assertFalse(venicePaths.contains("/venice")); Assert.assertFalse(venicePaths.contains("/venice/storeConfigs")); @@ -65,35 +65,27 @@ public void testGetVenicePaths() { Assert.assertFalse(venicePaths.contains("/venice/cluster1/adminTopicMetadata")); Assert.assertFalse(venicePaths.contains("/venice/cluster1/adminTopicMetadata/file1")); Assert.assertFalse(venicePaths.contains("/venice/cluster1/adminTopicMetadata/file2/file3")); - Assert.assertTrue(venicePaths.contains("/venice-parent")); - Assert.assertTrue(venicePaths.contains("/venice-parent/storeConfigs")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1")); Assert.assertFalse(venicePaths.contains("/venice-parent/cluster1/storeConfigs")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/adminTopicMetadata")); Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/adminTopicMetadata/file1")); Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/adminTopicMetadata/file2")); Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/adminTopicMetadata/file2/file3")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/executionids")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/ParentOfflinePushes")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/routers")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/StoreGraveyard")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/Stores")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2")); Assert.assertFalse(venicePaths.contains("/venice-parent/cluster2/storeConfigs")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/adminTopicMetadata")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/executionids")); Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/executionids/file1")); Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/executionids/file2")); Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/executionids/file2/file3")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/ParentOfflinePushes")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/routers")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/StoreGraveyard")); - Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/Stores")); Assert.assertFalse(venicePaths.contains("/venice-parent/helix-cluster")); Assert.assertFalse(venicePaths.contains("/venice-parent/helix-cluster/storeConfigs")); Assert.assertFalse(venicePaths.contains("/venice-parent/helix-cluster/adminTopicMetadata")); Assert.assertFalse(venicePaths.contains("/venice-parent/helix-cluster/adminTopicMetadata/file1")); Assert.assertFalse(venicePaths.contains("/venice-parent/helix-cluster/adminTopicMetadata/file2/file3")); + testVenicePathsContainsAsserts(venicePaths); + } + + @Test + public void testGetVenicePathsFromTree() { + TreeNode root = ZkCopier.buildRequiredPathsTree(CLUSTERS, BASE_PATH); + List venicePaths = ZkCopier.getVenicePathsFromTree(root, CLUSTERS, BASE_PATH); + testVenicePathsContainsAsserts(venicePaths); } @Test @@ -120,6 +112,25 @@ public void testBuildRequiredPathsTree() { } } + private void testVenicePathsContainsAsserts(List venicePaths) { + Assert.assertTrue(venicePaths.contains("/venice-parent")); + Assert.assertTrue(venicePaths.contains("/venice-parent/storeConfigs")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/adminTopicMetadata")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/executionids")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/ParentOfflinePushes")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/routers")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/StoreGraveyard")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster1/Stores")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/adminTopicMetadata")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/executionids")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/ParentOfflinePushes")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/routers")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/StoreGraveyard")); + Assert.assertTrue(venicePaths.contains("/venice-parent/cluster2/Stores")); + } + private void testContainsChildAsserts(TreeNode child) { Assert.assertTrue(child.containsChild(ADMIN_TOPIC_METADATA)); Assert.assertTrue(child.containsChild(EXECUTION_IDS)); diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/zk/VeniceZkPaths.java b/internal/venice-common/src/main/java/com/linkedin/venice/zk/VeniceZkPaths.java index 8e90563a147..10f20075c83 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/zk/VeniceZkPaths.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/zk/VeniceZkPaths.java @@ -1,5 +1,11 @@ package com.linkedin.venice.zk; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + + /** * This class contains constants that represent Venice-managed ZooKeeper paths. */ @@ -13,4 +19,10 @@ public class VeniceZkPaths { public static final String STORES = "Stores"; public static final String STORE_CONFIGS = "storeConfigs"; public static final String STORE_GRAVEYARD = "StoreGraveyard"; + + /** Set of all Venice-managed ZooKeeper cluster paths */ + private static final Set CLUSTER_ZK_PATHS_MODIFIABLE = new HashSet<>( + Arrays.asList(ADMIN_TOPIC_METADATA, EXECUTION_IDS, PARENT_OFFLINE_PUSHES, ROUTERS, STORE_GRAVEYARD, STORES)); + /** @see #CLUSTER_ZK_PATHS_MODIFIABLE */ + public static final Set CLUSTER_ZK_PATHS = Collections.unmodifiableSet(CLUSTER_ZK_PATHS_MODIFIABLE); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/zk/TestMigrateVeniceZKPaths.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/zk/TestMigrateVeniceZKPaths.java new file mode 100644 index 00000000000..e4a407aacc8 --- /dev/null +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/zk/TestMigrateVeniceZKPaths.java @@ -0,0 +1,109 @@ +package com.linkedin.venice.zk; + +import static com.linkedin.venice.zk.VeniceZkPaths.CLUSTER_ZK_PATHS; +import static com.linkedin.venice.zk.VeniceZkPaths.STORES; +import static com.linkedin.venice.zk.VeniceZkPaths.STORE_CONFIGS; + +import com.linkedin.venice.ZkCopier; +import com.linkedin.venice.helix.ZkClientFactory; +import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.ZkServerWrapper; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import org.apache.helix.zookeeper.datamodel.serializer.ByteArraySerializer; +import org.apache.helix.zookeeper.impl.client.ZkClient; +import org.apache.zookeeper.CreateMode; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + + +public class TestMigrateVeniceZKPaths { + private String srcZkAddress; + private ZkClient srcZkClient; + private ZkServerWrapper srcZkServerWrapper; + private String destZkAddress; + private ZkClient destZkClient; + private ZkServerWrapper destZkServerWrapper; + private static final String CLUSTER_1 = "cluster1"; + private static final String CLUSTER_2 = "cluster2"; + private final Set CLUSTERS = new HashSet<>(Arrays.asList(CLUSTER_1, CLUSTER_2)); + private static final String BASE_PATH = "/venice-parent"; + + @BeforeClass + public void zkSetup() { + srcZkServerWrapper = ServiceFactory.getZkServer(); + srcZkAddress = srcZkServerWrapper.getAddress(); + srcZkClient = ZkClientFactory.newZkClient(srcZkAddress); + destZkServerWrapper = ServiceFactory.getZkServer(); + destZkAddress = destZkServerWrapper.getAddress(); + destZkClient = ZkClientFactory.newZkClient(destZkAddress); + Assert.assertNotEquals(srcZkAddress, destZkAddress); + ByteArraySerializer serializer = new ByteArraySerializer(); + srcZkClient.setZkSerializer(serializer); + destZkClient.setZkSerializer(serializer); + createZkClientPaths(srcZkClient); + } + + @AfterClass + public void zkCleanup() { + srcZkClient.deleteRecursively(BASE_PATH); + srcZkClient.close(); + srcZkServerWrapper.close(); + destZkClient.deleteRecursively(BASE_PATH); + destZkClient.close(); + destZkServerWrapper.close(); + } + + /** Test migrating metadata from local source ZK to local destination ZK using ZkCopier.migrateVenicePaths()*/ + @Test + public void testMigrateVenicePaths() { + ZkCopier.migrateVenicePaths(srcZkClient, destZkClient, CLUSTERS, BASE_PATH); + testZkClientPathsAsserts(destZkClient); + } + + private void createZkClientPaths(ZkClient zkClient) { + zkClient.create(BASE_PATH, BASE_PATH.getBytes(), CreateMode.PERSISTENT); + String storeConfigsPath = BASE_PATH + "/" + STORE_CONFIGS; + zkClient.create(storeConfigsPath, STORE_CONFIGS.getBytes(), CreateMode.PERSISTENT); + zkClient.create(storeConfigsPath + "/file", "file".getBytes(), CreateMode.PERSISTENT); + for (String cluster: CLUSTERS) { + String clusterPath = BASE_PATH + "/" + cluster; + zkClient.create(clusterPath, cluster.getBytes(), CreateMode.PERSISTENT); + for (String zkPath: CLUSTER_ZK_PATHS) { + String clusterZkPath = clusterPath + "/" + zkPath; + zkClient.create(clusterZkPath, zkPath.getBytes(), CreateMode.PERSISTENT); + if (zkPath.equals(STORES)) { + zkClient.create(clusterZkPath + "/testStore", "testStore".getBytes(), CreateMode.PERSISTENT); + } + } + } + } + + private void testZkClientPathsAsserts(ZkClient zkClient) { + Assert.assertTrue(zkClient.exists(BASE_PATH)); + Assert.assertEquals(zkClient.readData(BASE_PATH), BASE_PATH.getBytes()); + String storeConfigsPath = BASE_PATH + "/" + STORE_CONFIGS; + Assert.assertTrue(zkClient.exists(storeConfigsPath)); + Assert.assertEquals(zkClient.readData(storeConfigsPath), STORE_CONFIGS.getBytes()); + Assert.assertTrue(zkClient.exists(storeConfigsPath + "/file")); + Assert.assertEquals(zkClient.readData(storeConfigsPath + "/file"), "file".getBytes()); + for (String cluster: CLUSTERS) { + String clusterPath = BASE_PATH + "/" + cluster; + Assert.assertTrue(zkClient.exists(clusterPath)); + Assert.assertEquals(zkClient.readData(clusterPath), cluster.getBytes()); + for (String zkPath: CLUSTER_ZK_PATHS) { + String clusterZkPath = clusterPath + "/" + zkPath; + Assert.assertTrue(zkClient.exists(clusterZkPath)); + Assert.assertEquals(zkClient.readData(clusterZkPath), zkPath.getBytes()); + if (zkPath.equals(STORES)) { + Assert.assertFalse(zkClient.exists(clusterZkPath + "/assertFalse")); + Assert.assertTrue(zkClient.exists(clusterZkPath + "/testStore")); + Assert.assertEquals(zkClient.readData(clusterZkPath + "/testStore"), "testStore".getBytes()); + } + } + } + } +}