From 702dd681d6ca1e0d1794ed05a46b0287f4d3bffd Mon Sep 17 00:00:00 2001 From: Sourav Maji Date: Thu, 6 Feb 2025 09:14:12 -0800 Subject: [PATCH 01/12] [server] Validate block-cache config against system memory --- .../linkedin/davinci/store/rocksdb/RocksDBServerConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java index bdcb1eb315..988aa19621 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java @@ -338,9 +338,9 @@ public RocksDBServerConfig(VeniceProperties props) { } this.rocksDBBlockCacheSizeInBytes = - props.getSizeInBytes(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 16 * 1024 * 1024 * 1024L); // 16GB + props.getSizeInBytes(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024 * 1024L); // 2GB this.rocksDBRMDBlockCacheSizeInBytes = - props.getSizeInBytes(ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024 * 1024L); // 2GB + props.getSizeInBytes(ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES, 1 * 1024 * 1024 * 1024L); // 1GB this.rocksDBBlockCacheImplementation = RocksDBBlockCacheImplementations .valueOf(props.getString(ROCKSDB_BLOCK_CACHE_IMPLEMENTATION, RocksDBBlockCacheImplementations.LRU.toString())); From 829e95728b07fdb7021a56082448a6e1230d5226 Mon Sep 17 00:00:00 2001 From: Sourav Maji Date: Thu, 6 Feb 2025 09:19:14 -0800 Subject: [PATCH 02/12] [server] Validate block-cache config against system memory --- .../store/rocksdb/RocksDBStorageEngineFactory.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java index 8cf6ce186a..2e122ea6ed 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java @@ -1,5 +1,6 @@ package com.linkedin.davinci.store.rocksdb; +import static com.linkedin.venice.utils.ByteUtils.generateHumanReadableByteCountString; import static org.rocksdb.RateLimiter.DEFAULT_FAIRNESS; import static org.rocksdb.RateLimiter.DEFAULT_MODE; import static org.rocksdb.RateLimiter.DEFAULT_REFILL_PERIOD_MICROS; @@ -139,6 +140,15 @@ public RocksDBStorageEngineFactory( this.env.setBackgroundThreads(rocksDBServerConfig.getRocksDBEnvFlushPoolSize(), Priority.HIGH); this.env.setBackgroundThreads(rocksDBServerConfig.getRocksDBEnvCompactionPoolSize(), Priority.LOW); + long cacheBytesNeeded = + rocksDBServerConfig.getRocksDBBlockCacheSizeInBytes() + (rocksDBServerConfig.isUseSeparateRMDCacheEnabled() + ? rocksDBServerConfig.getRocksDBRMDBlockCacheSizeInBytes() + : 0); + if (Runtime.getRuntime().maxMemory() * 0.8 < cacheBytesNeeded) { + throw new RuntimeException( + "Cannot setup rocksdb instance with block-cache size " + + generateHumanReadableByteCountString(cacheBytesNeeded)); + } // Shared cache across all the RocksDB databases if (RocksDBBlockCacheImplementations.CLOCK.equals(rocksDBServerConfig.getRocksDBBlockCacheImplementation())) { if (rocksDBServerConfig.isUseSeparateRMDCacheEnabled()) { From 209b60e156535c252fd54d1676f218ca98bfb8bd Mon Sep 17 00:00:00 2001 From: Sourav Maji Date: Thu, 6 Feb 2025 09:54:16 -0800 Subject: [PATCH 03/12] [server] Validate block-cache config against system memory --- .../ReplicationMeadataRocksDBStoragePartitionCFTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/rocksdb/ReplicationMeadataRocksDBStoragePartitionCFTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/rocksdb/ReplicationMeadataRocksDBStoragePartitionCFTest.java index 90a38c495d..b028a8bbf1 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/rocksdb/ReplicationMeadataRocksDBStoragePartitionCFTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/rocksdb/ReplicationMeadataRocksDBStoragePartitionCFTest.java @@ -1,5 +1,7 @@ package com.linkedin.davinci.store.rocksdb; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_SEPARATE_RMD_CACHE_ENABLED; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; @@ -49,6 +51,8 @@ public void createStorageEngineForTest() { Properties properties = new Properties(); properties.put(ROCKSDB_SEPARATE_RMD_CACHE_ENABLED, "true"); + properties.put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 1024 * 1024 * 1024L); + properties.put(ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES, 1024 * 1024L); VeniceProperties serverProps = AbstractStorageEngineTest.getServerProperties(PersistenceType.ROCKS_DB, properties); storageService = new StorageService( AbstractStorageEngineTest.getVeniceConfigLoader(serverProps), From df43cf98e6da9cb08ba55cc2ae2b7e0d79ae808f Mon Sep 17 00:00:00 2001 From: Sourav Maji Date: Fri, 7 Feb 2025 10:25:35 -0800 Subject: [PATCH 04/12] addressed comments --- .../davinci/store/rocksdb/RocksDBServerConfig.java | 4 ++-- .../store/rocksdb/RocksDBStorageEngineFactory.java | 5 ++++- .../main/java/com/linkedin/venice/utils/Utils.java | 13 +++++++++++++ .../integration/utils/VeniceServerWrapper.java | 4 ++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java index 988aa19621..bdcb1eb315 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java @@ -338,9 +338,9 @@ public RocksDBServerConfig(VeniceProperties props) { } this.rocksDBBlockCacheSizeInBytes = - props.getSizeInBytes(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024 * 1024L); // 2GB + props.getSizeInBytes(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 16 * 1024 * 1024 * 1024L); // 16GB this.rocksDBRMDBlockCacheSizeInBytes = - props.getSizeInBytes(ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES, 1 * 1024 * 1024 * 1024L); // 1GB + props.getSizeInBytes(ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024 * 1024L); // 2GB this.rocksDBBlockCacheImplementation = RocksDBBlockCacheImplementations .valueOf(props.getString(ROCKSDB_BLOCK_CACHE_IMPLEMENTATION, RocksDBBlockCacheImplementations.LRU.toString())); diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java index 2e122ea6ed..c2a3cfe9b3 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java @@ -1,6 +1,7 @@ package com.linkedin.davinci.store.rocksdb; import static com.linkedin.venice.utils.ByteUtils.generateHumanReadableByteCountString; +import static com.linkedin.venice.utils.Utils.getOSMemorySize; import static org.rocksdb.RateLimiter.DEFAULT_FAIRNESS; import static org.rocksdb.RateLimiter.DEFAULT_MODE; import static org.rocksdb.RateLimiter.DEFAULT_REFILL_PERIOD_MICROS; @@ -144,7 +145,9 @@ public RocksDBStorageEngineFactory( rocksDBServerConfig.getRocksDBBlockCacheSizeInBytes() + (rocksDBServerConfig.isUseSeparateRMDCacheEnabled() ? rocksDBServerConfig.getRocksDBRMDBlockCacheSizeInBytes() : 0); - if (Runtime.getRuntime().maxMemory() * 0.8 < cacheBytesNeeded) { + + long systemMemorySize = getOSMemorySize(); + if (systemMemorySize > 0 && (systemMemorySize * 0.8 < cacheBytesNeeded)) { throw new RuntimeException( "Cannot setup rocksdb instance with block-cache size " + generateHumanReadableByteCountString(cacheBytesNeeded)); diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java b/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java index 2985276434..f4f5a3d557 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.io.InputStream; import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.file.Files; @@ -1141,4 +1142,16 @@ public static long parseDateTimeToEpoch(String dateTime, String dateTimeFormat, dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone)); return dateFormat.parse(dateTime).getTime(); } + + public static long getOSMemorySize() { + OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); + + if (osBean instanceof com.sun.management.OperatingSystemMXBean) { + com.sun.management.OperatingSystemMXBean extendedOsBean = (com.sun.management.OperatingSystemMXBean) osBean; + return extendedOsBean.getTotalPhysicalMemorySize(); + } else { + System.out.println("OS Bean not available."); + } + return -1; + } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java index dec15c6ad8..727fd79fb9 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java @@ -1,7 +1,9 @@ package com.linkedin.venice.integration.utils; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_OPTIONS_USE_DIRECT_READS; import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.venice.ConfigKeys.ADMIN_PORT; import static com.linkedin.venice.ConfigKeys.CLUSTER_DISCOVERY_D2_SERVICE; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; @@ -264,6 +266,8 @@ static StatefulServiceProvider generateService( .put(SERVER_INGESTION_HEARTBEAT_INTERVAL_MS, 5000) .put(SERVER_LEADER_COMPLETE_STATE_CHECK_IN_FOLLOWER_VALID_INTERVAL_MS, 5000) .put(SERVER_RESUBSCRIPTION_TRIGGERED_BY_VERSION_INGESTION_CONTEXT_CHANGE_ENABLED, true) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 8 * 1024 * 1024 * 1024L) + .put(ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES, 1024 * 1024 * 1024L) .put(SERVER_DELETE_UNASSIGNED_PARTITIONS_ON_STARTUP, serverDeleteUnassignedPartitionsOnStartup); if (sslToKafka) { serverPropsBuilder.put(KAFKA_SECURITY_PROTOCOL, PubSubSecurityProtocol.SSL.name()); From d66803148961e948f2a10ead3dbf55f537eb73a8 Mon Sep 17 00:00:00 2001 From: Sourav Maji Date: Fri, 7 Feb 2025 15:59:54 -0800 Subject: [PATCH 05/12] updated tests --- .../davinci/store/rocksdb/RocksDBStorageEngineFactory.java | 3 ++- .../linkedin/venice/endToEnd/DaVinciClientDiskFullTest.java | 2 ++ .../linkedin/venice/integration/utils/DaVinciTestContext.java | 2 ++ .../src/main/java/com/linkedin/venice/utils/TestUtils.java | 2 ++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java index c2a3cfe9b3..2f9066f696 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java @@ -150,7 +150,8 @@ public RocksDBStorageEngineFactory( if (systemMemorySize > 0 && (systemMemorySize * 0.8 < cacheBytesNeeded)) { throw new RuntimeException( "Cannot setup rocksdb instance with block-cache size " - + generateHumanReadableByteCountString(cacheBytesNeeded)); + + generateHumanReadableByteCountString(cacheBytesNeeded) + ". System memory : " + + generateHumanReadableByteCountString(systemMemorySize)); } // Shared cache across all the RocksDB databases if (RocksDBBlockCacheImplementations.CLOCK.equals(rocksDBServerConfig.getRocksDBBlockCacheImplementation())) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientDiskFullTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientDiskFullTest.java index c7a7a32176..d10307af36 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientDiskFullTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientDiskFullTest.java @@ -1,5 +1,6 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; import static com.linkedin.venice.ConfigKeys.CLIENT_USE_SYSTEM_STORE_REPOSITORY; import static com.linkedin.venice.ConfigKeys.CLUSTER_DISCOVERY_D2_SERVICE; @@ -142,6 +143,7 @@ private VeniceProperties getDaVinciBackendConfig(boolean useDaVinciSpecificExecu .put(PERSISTENCE_TYPE, ROCKS_DB) .put(PUSH_STATUS_STORE_ENABLED, true) .put(D2_ZK_HOSTS_ADDRESS, venice.getZk().getAddress()) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(CLUSTER_DISCOVERY_D2_SERVICE, VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME) .put(USE_DA_VINCI_SPECIFIC_EXECUTION_STATUS_FOR_ERROR, useDaVinciSpecificExecutionStatusForError) .put(SERVER_DISK_FULL_THRESHOLD, getDiskFullThreshold(largePushRecordCount, largePushRecordMinSize)); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/DaVinciTestContext.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/DaVinciTestContext.java index d142a57b43..e24fe5b372 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/DaVinciTestContext.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/DaVinciTestContext.java @@ -1,5 +1,6 @@ package com.linkedin.venice.integration.utils; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; import static com.linkedin.venice.ConfigKeys.CLIENT_USE_SYSTEM_STORE_REPOSITORY; import static com.linkedin.venice.ConfigKeys.CLUSTER_DISCOVERY_D2_SERVICE; @@ -132,6 +133,7 @@ public static PropertyBuilder getDaVinciPropertyBuilder(String zkAddress) { .put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(D2_ZK_HOSTS_ADDRESS, zkAddress) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 4 * 1024 * 1024 * 1024L) .put(CLUSTER_DISCOVERY_D2_SERVICE, VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME); } } diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java index 2ee342811d..eee3f9f056 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/TestUtils.java @@ -1,5 +1,6 @@ package com.linkedin.venice.utils; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; import static com.linkedin.venice.ConfigKeys.PARTITIONER_CLASS; import static com.linkedin.venice.ConfigKeys.SERVER_FORKED_PROCESS_JVM_ARGUMENT_LIST; @@ -875,6 +876,7 @@ public static void shutdownExecutor(ExecutorService executor, long timeout, Time public static Map getIngestionIsolationPropertyMap() { Map propertyMap = new HashMap<>(); propertyMap.put(SERVER_INGESTION_MODE, IngestionMode.ISOLATED); + propertyMap.put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 1 * 1024 * 1024 * 1024L); propertyMap.put(SERVER_FORKED_PROCESS_JVM_ARGUMENT_LIST, "-Xms256M;-Xmx1G"); return propertyMap; } From 7c673eac4ff4ef64fe11a691f7c3b0fd9aba2da8 Mon Sep 17 00:00:00 2001 From: Sourav Maji Date: Tue, 11 Feb 2025 10:48:49 -0800 Subject: [PATCH 06/12] updated tests --- .../davinci/store/rocksdb/RocksDBServerConfig.java | 8 ++++++++ .../store/rocksdb/RocksDBStorageEngineFactory.java | 3 ++- .../endToEnd/ActiveActiveReplicationForHybridTest.java | 7 +++++-- .../com/linkedin/venice/endToEnd/DaVinciComputeTest.java | 3 +++ .../venice/endToEnd/TestBatchReportIncrementalPush.java | 2 ++ .../venice/endToEnd/TestMaterializedViewEndToEnd.java | 2 ++ .../venice/endToEnd/TestPushJobWithNativeReplication.java | 2 ++ .../fastclient/FastClientDaVinciClientCompatTest.java | 2 ++ .../com/linkedin/venice/router/TestBlobDiscovery.java | 2 ++ 9 files changed, 28 insertions(+), 3 deletions(-) diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java index bdcb1eb315..cc866b6dee 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java @@ -237,6 +237,8 @@ public class RocksDBServerConfig { "rocksdb.blob.garbage.collection.force.threshold"; public static final String ROCKSDB_BLOB_FILE_STARTING_LEVEL = "rocksdb.blob.file.starting.level"; + public static final String ROCKSDB_BLOCK_CACHE_MEMORY_LIMIT = "rocksdb.block.cache.memory.limit"; + private final boolean rocksDBUseDirectReads; private final int rocksDBEnvFlushPoolSize; @@ -312,6 +314,7 @@ public class RocksDBServerConfig { private final double blobGarbageCollectionAgeCutOff; private final double blobGarbageCollectionForceThreshold; private final int blobFileStartingLevel; + private final double rocksdbBlockCacheMemoryLimit; public RocksDBServerConfig(VeniceProperties props) { // Do not use Direct IO for reads by default @@ -450,6 +453,7 @@ public RocksDBServerConfig(VeniceProperties props) { this.blobGarbageCollectionAgeCutOff = props.getDouble(ROCKSDB_BLOB_GARBAGE_COLLECTION_AGE_CUTOFF, 0.25); this.blobGarbageCollectionForceThreshold = props.getDouble(ROCKSDB_BLOB_GARBAGE_COLLECTION_FORCE_THRESHOLD, 0.8); this.blobFileStartingLevel = props.getInt(ROCKSDB_BLOB_FILE_STARTING_LEVEL, 0); + this.rocksdbBlockCacheMemoryLimit = props.getDouble(ROCKSDB_BLOCK_CACHE_MEMORY_LIMIT, 0.8); } public int getLevel0FileNumCompactionTriggerWriteOnlyVersion() { @@ -686,4 +690,8 @@ public double getBlobGarbageCollectionForceThreshold() { public int getBlobFileStartingLevel() { return blobFileStartingLevel; } + + public double getRocksdbBlockCacheMemoryLimit() { + return rocksdbBlockCacheMemoryLimit; + } } diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java index 2f9066f696..4eec89b549 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java @@ -147,7 +147,8 @@ public RocksDBStorageEngineFactory( : 0); long systemMemorySize = getOSMemorySize(); - if (systemMemorySize > 0 && (systemMemorySize * 0.8 < cacheBytesNeeded)) { + if (systemMemorySize > 0 + && (systemMemorySize * rocksDBServerConfig.getRocksdbBlockCacheMemoryLimit() < cacheBytesNeeded)) { throw new RuntimeException( "Cannot setup rocksdb instance with block-cache size " + generateHumanReadableByteCountString(cacheBytesNeeded) + ". System memory : " diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java index c41f4cde89..7b0a7cd5af 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java @@ -1,5 +1,6 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED; import static com.linkedin.venice.CommonConfigKeys.SSL_ENABLED; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; @@ -480,8 +481,10 @@ public void testAAReplicationCanConsumeFromAllRegions(boolean isChunkingEnabled, // Verify that DaVinci client can successfully bootstrap all partitions from AA enabled stores String baseDataPath = Utils.getTempDataDirectory().getAbsolutePath(); - VeniceProperties backendConfig = - new PropertyBuilder().put(DATA_BASE_PATH, baseDataPath).put(PERSISTENCE_TYPE, ROCKS_DB).build(); + VeniceProperties backendConfig = new PropertyBuilder().put(DATA_BASE_PATH, baseDataPath) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) + .put(PERSISTENCE_TYPE, ROCKS_DB) + .build(); MetricsRepository metricsRepository = new MetricsRepository(); try ( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciComputeTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciComputeTest.java index 67e6b361a9..5dee5315f2 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciComputeTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciComputeTest.java @@ -1,5 +1,6 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; import static com.linkedin.venice.ConfigKeys.CLIENT_USE_SYSTEM_STORE_REPOSITORY; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; @@ -659,6 +660,7 @@ public void testPartialKeyLookupWithRocksDBBlockBasedTable() throws ExecutionExc VeniceProperties backendConfig = new PropertyBuilder().put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(DATA_BASE_PATH, baseDataPath) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(PERSISTENCE_TYPE, ROCKS_DB) .build(); @@ -760,6 +762,7 @@ public void testPartialKeyLookupWithRocksDBPlainTable() throws ExecutionExceptio VeniceProperties backendConfig = new PropertyBuilder().put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(DATA_BASE_PATH, baseDataPath) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(PERSISTENCE_TYPE, ROCKS_DB) .build(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchReportIncrementalPush.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchReportIncrementalPush.java index 2e01a0a0f1..5e83b3bfe7 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchReportIncrementalPush.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchReportIncrementalPush.java @@ -1,5 +1,6 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; import static com.linkedin.venice.ConfigKeys.CLIENT_USE_SYSTEM_STORE_REPOSITORY; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; @@ -164,6 +165,7 @@ public void testBatchReportIncrementalPush() throws IOException { .put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(PUSH_STATUS_STORE_ENABLED, true) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(SERVER_BATCH_REPORT_END_OF_INCREMENTAL_PUSH_STATUS_ENABLED, true) .put(DAVINCI_PUSH_STATUS_SCAN_INTERVAL_IN_SECONDS, 1) .build(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java index 2f0fd279ad..a4f988fd83 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java @@ -1,5 +1,6 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED; import static com.linkedin.venice.ConfigKeys.CHILD_DATA_CENTER_KAFKA_URL_PREFIX; import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; @@ -207,6 +208,7 @@ public void testBatchOnlyMaterializedViewDVCConsumer() throws IOException, Execu new PropertyBuilder().put(DATA_BASE_PATH, Utils.getTempDataDirectory().getAbsolutePath()) .put(PERSISTENCE_TYPE, PersistenceType.ROCKS_DB) .put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .build(); DaVinciConfig daVinciConfig = new DaVinciConfig(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java index c7edff172a..220bb36bf1 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java @@ -1,5 +1,6 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED; import static com.linkedin.venice.CommonConfigKeys.SSL_ENABLED; import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; @@ -886,6 +887,7 @@ private void validateDaVinciClient(String storeName, int recordCount) VeniceProperties backendConfig = new PropertyBuilder().put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(DATA_BASE_PATH, baseDataPath) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(PERSISTENCE_TYPE, ROCKS_DB) .build(); MetricsRepository metricsRepository = new MetricsRepository(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/FastClientDaVinciClientCompatTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/FastClientDaVinciClientCompatTest.java index 14c0c70b66..37b227854b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/FastClientDaVinciClientCompatTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/FastClientDaVinciClientCompatTest.java @@ -1,5 +1,6 @@ package com.linkedin.venice.fastclient; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.venice.ConfigKeys.CLIENT_USE_SYSTEM_STORE_REPOSITORY; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; import static com.linkedin.venice.ConfigKeys.PERSISTENCE_TYPE; @@ -89,6 +90,7 @@ private DaVinciClient setupDaVinciClient(String storeNa new PropertyBuilder().put(DATA_BASE_PATH, Utils.getTempDataDirectory().getAbsolutePath()) .put(PERSISTENCE_TYPE, ROCKS_DB) .put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(DATA_BASE_PATH, dataPath) .build(); daVinciClientFactory = new CachingDaVinciClientFactory( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestBlobDiscovery.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestBlobDiscovery.java index 5ad00428c5..ec80d29506 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestBlobDiscovery.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestBlobDiscovery.java @@ -1,5 +1,6 @@ package com.linkedin.venice.router; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; import static com.linkedin.venice.ConfigKeys.CLIENT_USE_SYSTEM_STORE_REPOSITORY; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; @@ -186,6 +187,7 @@ public void setUp() { .put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(PUSH_STATUS_STORE_ENABLED, true) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(DAVINCI_PUSH_STATUS_SCAN_INTERVAL_IN_SECONDS, 1) .build(); From 14c77c15b16392fb07468bf3dc315f79d07369e8 Mon Sep 17 00:00:00 2001 From: Sourav Maji Date: Tue, 11 Feb 2025 12:11:08 -0800 Subject: [PATCH 07/12] updated tests --- .../endToEnd/DaVinciClientMemoryLimitTest.java | 2 ++ .../linkedin/venice/endToEnd/DaVinciClientTest.java | 13 ++++++++++++- .../venice/endToEnd/DaVinciClusterAgnosticTest.java | 3 +++ .../endToEnd/TestMaterializedViewEndToEnd.java | 1 + 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientMemoryLimitTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientMemoryLimitTest.java index a1476ef8e5..1cfd034364 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientMemoryLimitTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientMemoryLimitTest.java @@ -1,5 +1,6 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_MEMTABLE_SIZE_IN_BYTES; import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_TOTAL_MEMTABLE_USAGE_CAP_IN_BYTES; import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; @@ -143,6 +144,7 @@ private VeniceProperties getDaVinciBackendConfig( .put(D2_ZK_HOSTS_ADDRESS, veniceCluster.getZk().getAddress()) .put(CLUSTER_DISCOVERY_D2_SERVICE, VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME) .put(ROCKSDB_MEMTABLE_SIZE_IN_BYTES, "2MB") + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 4 * 1024 * 1024L) .put(ROCKSDB_TOTAL_MEMTABLE_USAGE_CAP_IN_BYTES, "10MB") .put(INGESTION_MEMORY_LIMIT_STORE_LIST, String.join(",", memoryLimitStores)) .put(USE_DA_VINCI_SPECIFIC_EXECUTION_STATUS_FOR_ERROR, useDaVinciSpecificExecutionStatusForError); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientTest.java index 6dec9463d8..c374049ab6 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientTest.java @@ -1,5 +1,6 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED; import static com.linkedin.venice.ConfigKeys.BLOB_TRANSFER_MANAGER_ENABLED; import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; @@ -193,6 +194,7 @@ public void testConcurrentGetAndStart() throws Exception { VeniceProperties backendConfig = new PropertyBuilder().put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(DATA_BASE_PATH, baseDataPath) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(PERSISTENCE_TYPE, ROCKS_DB) .build(); @@ -261,6 +263,7 @@ public void testBatchStore(DaVinciConfig clientConfig) throws Exception { .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(DATA_BASE_PATH, baseDataPath) .put(PERSISTENCE_TYPE, ROCKS_DB) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(DA_VINCI_CURRENT_VERSION_BOOTSTRAPPING_SPEEDUP_ENABLED, true) .put(PUSH_STATUS_STORE_ENABLED, true) .put(DAVINCI_PUSH_STATUS_CHECK_INTERVAL_IN_MS, 1000) @@ -381,6 +384,7 @@ public void testIncrementalPushStatusBatching(boolean isIngestionIsolated) throw .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(DATA_BASE_PATH, Utils.getTempDataDirectory().getAbsolutePath()) .put(PERSISTENCE_TYPE, ROCKS_DB) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(PUSH_STATUS_STORE_ENABLED, true) .put(DAVINCI_PUSH_STATUS_CHECK_INTERVAL_IN_MS, 1000) .build(); @@ -458,6 +462,7 @@ public void testObjectReuse(DaVinciConfig clientConfig) throws Exception { // TODO: Looks like cache = null does not work with fast meta store repository refresh interval // .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(DATA_BASE_PATH, baseDataPath) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(PERSISTENCE_TYPE, ROCKS_DB) .build(); @@ -710,6 +715,7 @@ public void testHybridStoreWithoutIngestionIsolation(DaVinciConfig daVinciConfig VeniceProperties backendConfig = new PropertyBuilder().put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(DATA_BASE_PATH, Utils.getTempDataDirectory().getAbsolutePath()) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(PERSISTENCE_TYPE, ROCKS_DB) .build(); @@ -774,6 +780,7 @@ public void testHybridStore() throws Exception { VeniceProperties backendConfig = new PropertyBuilder().put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(DATA_BASE_PATH, Utils.getTempDataDirectory().getAbsolutePath()) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(PERSISTENCE_TYPE, ROCKS_DB) .build(); @@ -1003,6 +1010,7 @@ public void testBootstrapSubscription(DaVinciConfig daVinciConfig) throws Except .put(DA_VINCI_CURRENT_VERSION_BOOTSTRAPPING_SPEEDUP_ENABLED, true) .put(PUSH_STATUS_STORE_ENABLED, true) .put(DAVINCI_PUSH_STATUS_CHECK_INTERVAL_IN_MS, 1000) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(DA_VINCI_SUBSCRIBE_ON_DISK_PARTITIONS_AUTOMATICALLY, false) .build(); @@ -1064,7 +1072,8 @@ public void testBootstrapSubscription(DaVinciConfig daVinciConfig) throws Except @Test(timeOut = TEST_TIMEOUT, dataProvider = "dv-client-config-provider", dataProviderClass = DataProviderUtils.class) public void testPartialSubscription(DaVinciConfig daVinciConfig) throws Exception { String storeName = createStoreWithMetaSystemStoreAndPushStatusSystemStore(KEY_COUNT); - VeniceProperties backendConfig = new PropertyBuilder().build(); + VeniceProperties backendConfig = + new PropertyBuilder().put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L).build(); Set keySet = new HashSet<>(); for (int i = 0; i < KEY_COUNT; ++i) { @@ -1187,6 +1196,7 @@ public void testCrashedDaVinciWithIngestionIsolation() throws Exception { .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(DATA_BASE_PATH, baseDataPath) .put(PERSISTENCE_TYPE, ROCKS_DB) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(D2_ZK_HOSTS_ADDRESS, zkHosts) .build(); @@ -1265,6 +1275,7 @@ public void testBlobP2PTransferAmongDVC() throws Exception { .put(DAVINCI_P2P_BLOB_TRANSFER_SERVER_PORT, port2) .put(DAVINCI_P2P_BLOB_TRANSFER_CLIENT_PORT, port1) .put(PUSH_STATUS_STORE_ENABLED, true) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(DAVINCI_PUSH_STATUS_SCAN_INTERVAL_IN_SECONDS, 1) .put(BLOB_TRANSFER_MANAGER_ENABLED, true); VeniceProperties backendConfig2 = configBuilder.build(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java index c82e5a30fd..236bfbc522 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java @@ -1,5 +1,6 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; import static com.linkedin.venice.ConfigKeys.CLIENT_USE_SYSTEM_STORE_REPOSITORY; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; @@ -166,6 +167,7 @@ public void testMultiClusterDaVinci() throws Exception { new PropertyBuilder().put(DATA_BASE_PATH, Utils.getTempDataDirectory().getAbsolutePath()) .put(PERSISTENCE_TYPE, PersistenceType.ROCKS_DB) .put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .build(); DaVinciConfig daVinciConfig = new DaVinciConfig(); @@ -320,6 +322,7 @@ public void testDaVinciVersionSwap() throws Exception { new PropertyBuilder().put(DATA_BASE_PATH, Utils.getTempDataDirectory().getAbsolutePath()) .put(PERSISTENCE_TYPE, PersistenceType.ROCKS_DB) .put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .build(); D2Client daVinciD2 = D2TestUtils.getAndStartD2Client(multiClusterVenice.getZkServerWrapper().getAddress()); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java index a4f988fd83..0d379ae64b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java @@ -266,6 +266,7 @@ public void testBatchOnlyMaterializedViewDVCConsumer() throws IOException, Execu new PropertyBuilder().put(DATA_BASE_PATH, Utils.getTempDataDirectory().getAbsolutePath()) .put(PERSISTENCE_TYPE, PersistenceType.ROCKS_DB) .put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .build(); D2Client daVinciD2SourceFabric = D2TestUtils From 306c43e95951ba793cb75bb1f81089ab0e8ddbf0 Mon Sep 17 00:00:00 2001 From: Sourav Maji Date: Tue, 11 Feb 2025 17:14:48 -0800 Subject: [PATCH 08/12] updated tests --- .../endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java b/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java index 3c0c1ff37c..4681cb6b36 100644 --- a/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java +++ b/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java @@ -1,5 +1,6 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; import static com.linkedin.venice.ConfigKeys.CLIENT_USE_SYSTEM_STORE_REPOSITORY; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; @@ -271,6 +272,7 @@ public VeniceProperties buildRecordTransformerBackendConfig(boolean pushStatusSt .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(DATA_BASE_PATH, baseDataPath) .put(PERSISTENCE_TYPE, ROCKS_DB) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(DA_VINCI_CURRENT_VERSION_BOOTSTRAPPING_SPEEDUP_ENABLED, true) .put(PUSH_STATUS_STORE_ENABLED, pushStatusStoreEnabled) .put(DAVINCI_PUSH_STATUS_CHECK_INTERVAL_IN_MS, 1000); From f7e6cddcdb739ef70bd067d187680744a0760e07 Mon Sep 17 00:00:00 2001 From: Sourav Maji Date: Wed, 12 Feb 2025 09:10:33 -0800 Subject: [PATCH 09/12] updated tests --- docker/venice-server/single-dc-configs/server.properties | 1 + .../venice/endToEnd/DaVinciClientRecordTransformerTest.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docker/venice-server/single-dc-configs/server.properties b/docker/venice-server/single-dc-configs/server.properties index 92fba50401..7ab24bcf53 100644 --- a/docker/venice-server/single-dc-configs/server.properties +++ b/docker/venice-server/single-dc-configs/server.properties @@ -17,3 +17,4 @@ system.schema.cluster.name=venice-cluster0 data.base.path=/opt/venice/rocksdb server.ingestion.isolation.application.port=54094 persistence.type=ROCKS_DB +rocksdb.block.cache.size.in.bytes=2147483648 diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java index 2435202317..f8f9d696ef 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java @@ -1,5 +1,6 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED; import static com.linkedin.venice.ConfigKeys.BLOB_TRANSFER_MANAGER_ENABLED; import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; @@ -756,6 +757,7 @@ public VeniceProperties buildRecordTransformerBackendConfig(boolean pushStatusSt .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(DATA_BASE_PATH, baseDataPath) .put(PERSISTENCE_TYPE, ROCKS_DB) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(DA_VINCI_CURRENT_VERSION_BOOTSTRAPPING_SPEEDUP_ENABLED, true); if (pushStatusStoreEnabled) { From 53cef283323a75f01e24394c0e3bee8b9a4c8a1c Mon Sep 17 00:00:00 2001 From: Sourav Maji Date: Wed, 12 Feb 2025 10:05:53 -0800 Subject: [PATCH 10/12] updated tests --- .../com/linkedin/davinci/store/AbstractStorageEngineTest.java | 4 ++++ .../src/test/java/com/linkedin/venice/utils/UtilsTest.java | 2 ++ .../venice/integration/utils/VeniceServerWrapper.java | 4 ++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java index cbe8b0117b..a48d5f005c 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/store/AbstractStorageEngineTest.java @@ -1,5 +1,7 @@ package com.linkedin.davinci.store; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.venice.ConfigKeys.ADMIN_PORT; import static com.linkedin.venice.ConfigKeys.CLUSTER_NAME; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; @@ -38,6 +40,8 @@ public static VeniceProperties getServerProperties(PersistenceType persistenceTy .put(ZOOKEEPER_ADDRESS, "localhost:2181") .put(PERSISTENCE_TYPE, persistenceType.toString()) .put(KAFKA_BOOTSTRAP_SERVERS, "127.0.0.1:9092") + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) + .put(ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES, 1 * 1024 * 1024L) .put(LISTENER_PORT, 7072) .put(ADMIN_PORT, 7073) .put(DATA_BASE_PATH, dataDirectory.getAbsolutePath()) diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java index 892bbbc0f4..3a8d91a507 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java @@ -56,6 +56,8 @@ public void testGetHelixNodeIdentifier() { Utils.getHelixNodeIdentifier(fixedHostname, 1234), fixedHostname + "_" + port, "Identifier is not the valid format required by Helix."); + long memSize = Utils.getOSMemorySize(); + assertTrue(memSize > 0); } @Test diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java index 727fd79fb9..b59ed285ab 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java @@ -266,8 +266,8 @@ static StatefulServiceProvider generateService( .put(SERVER_INGESTION_HEARTBEAT_INTERVAL_MS, 5000) .put(SERVER_LEADER_COMPLETE_STATE_CHECK_IN_FOLLOWER_VALID_INTERVAL_MS, 5000) .put(SERVER_RESUBSCRIPTION_TRIGGERED_BY_VERSION_INGESTION_CONTEXT_CHANGE_ENABLED, true) - .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 8 * 1024 * 1024 * 1024L) - .put(ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES, 1024 * 1024 * 1024L) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 512 * 1024 * 1024L) + .put(ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES, 128 * 1024 * 1024L) .put(SERVER_DELETE_UNASSIGNED_PARTITIONS_ON_STARTUP, serverDeleteUnassignedPartitionsOnStartup); if (sslToKafka) { serverPropsBuilder.put(KAFKA_SECURITY_PROTOCOL, PubSubSecurityProtocol.SSL.name()); From 1a4c5d4acc2936274ab534dc98a3cd1ba32b8675 Mon Sep 17 00:00:00 2001 From: Sourav Maji Date: Wed, 12 Feb 2025 11:26:06 -0800 Subject: [PATCH 11/12] updated tests --- .../venice/endToEnd/DaVinciClientRecordTransformerTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java index f8f9d696ef..82781ada79 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java @@ -568,6 +568,7 @@ public void testBlobTransferRecordTransformer() throws Exception { .put(DAVINCI_P2P_BLOB_TRANSFER_SERVER_PORT, port2) .put(DAVINCI_P2P_BLOB_TRANSFER_CLIENT_PORT, port1) .put(PUSH_STATUS_STORE_ENABLED, true) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(DAVINCI_PUSH_STATUS_SCAN_INTERVAL_IN_SECONDS, 1) .put(BLOB_TRANSFER_MANAGER_ENABLED, true); VeniceProperties backendConfig2 = configBuilder.build(); From b3b0a29c4848cad6aaf7b18e2ba16ccfe3a2834c Mon Sep 17 00:00:00 2001 From: Sourav Maji Date: Wed, 12 Feb 2025 13:21:35 -0800 Subject: [PATCH 12/12] updated tests --- .../linkedin/davinci/ingestion/IsolatedIngestionServerTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/ingestion/IsolatedIngestionServerTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/ingestion/IsolatedIngestionServerTest.java index 6203ebb3ce..9f7dcb9dc5 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/ingestion/IsolatedIngestionServerTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/ingestion/IsolatedIngestionServerTest.java @@ -3,6 +3,7 @@ import static com.linkedin.davinci.ingestion.utils.IsolatedIngestionUtils.INGESTION_ISOLATION_CONFIG_PREFIX; import static com.linkedin.davinci.ingestion.utils.IsolatedIngestionUtils.buildAndSaveConfigsForForkedIngestionProcess; import static com.linkedin.davinci.ingestion.utils.IsolatedIngestionUtils.executeShellCommand; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.venice.ConfigKeys.CLUSTER_NAME; import static com.linkedin.venice.ConfigKeys.D2_ZK_HOSTS_ADDRESS; import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; @@ -127,6 +128,7 @@ private VeniceConfigLoader getConfigLoader(int servicePort) { .put(D2_ZK_HOSTS_ADDRESS, zkServerWrapper.getAddress()) .put(SERVER_PARTITION_GRACEFUL_DROP_DELAY_IN_SECONDS, 100) .put(SERVER_INGESTION_ISOLATION_CONNECTION_TIMEOUT_SECONDS, 10) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024L) .put(INGESTION_ISOLATION_CONFIG_PREFIX + "." + SERVER_PARTITION_GRACEFUL_DROP_DELAY_IN_SECONDS, 10) .put(SERVER_INGESTION_ISOLATION_SERVICE_PORT, servicePort) .build();