From b5cde844d6e7708de7ad765854146247fcc41b57 Mon Sep 17 00:00:00 2001 From: Sourav Maji Date: Fri, 7 Feb 2025 10:25:35 -0800 Subject: [PATCH] addressed comments --- .../davinci/store/rocksdb/RocksDBServerConfig.java | 4 ++-- .../store/rocksdb/RocksDBStorageEngineFactory.java | 5 ++++- .../main/java/com/linkedin/venice/utils/Utils.java | 13 +++++++++++++ .../integration/utils/VeniceServerWrapper.java | 4 ++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java index 988aa196212..bdcb1eb3153 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBServerConfig.java @@ -338,9 +338,9 @@ public RocksDBServerConfig(VeniceProperties props) { } this.rocksDBBlockCacheSizeInBytes = - props.getSizeInBytes(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024 * 1024L); // 2GB + props.getSizeInBytes(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 16 * 1024 * 1024 * 1024L); // 16GB this.rocksDBRMDBlockCacheSizeInBytes = - props.getSizeInBytes(ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES, 1 * 1024 * 1024 * 1024L); // 1GB + props.getSizeInBytes(ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES, 2 * 1024 * 1024 * 1024L); // 2GB this.rocksDBBlockCacheImplementation = RocksDBBlockCacheImplementations .valueOf(props.getString(ROCKSDB_BLOCK_CACHE_IMPLEMENTATION, RocksDBBlockCacheImplementations.LRU.toString())); diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java index 2e122ea6edb..c2a3cfe9b32 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java @@ -1,6 +1,7 @@ package com.linkedin.davinci.store.rocksdb; import static com.linkedin.venice.utils.ByteUtils.generateHumanReadableByteCountString; +import static com.linkedin.venice.utils.Utils.getOSMemorySize; import static org.rocksdb.RateLimiter.DEFAULT_FAIRNESS; import static org.rocksdb.RateLimiter.DEFAULT_MODE; import static org.rocksdb.RateLimiter.DEFAULT_REFILL_PERIOD_MICROS; @@ -144,7 +145,9 @@ public RocksDBStorageEngineFactory( rocksDBServerConfig.getRocksDBBlockCacheSizeInBytes() + (rocksDBServerConfig.isUseSeparateRMDCacheEnabled() ? rocksDBServerConfig.getRocksDBRMDBlockCacheSizeInBytes() : 0); - if (Runtime.getRuntime().maxMemory() * 0.8 < cacheBytesNeeded) { + + long systemMemorySize = getOSMemorySize(); + if (systemMemorySize > 0 && (systemMemorySize * 0.8 < cacheBytesNeeded)) { throw new RuntimeException( "Cannot setup rocksdb instance with block-cache size " + generateHumanReadableByteCountString(cacheBytesNeeded)); diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java b/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java index 29852764347..f4f5a3d557a 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.io.InputStream; import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.file.Files; @@ -1141,4 +1142,16 @@ public static long parseDateTimeToEpoch(String dateTime, String dateTimeFormat, dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone)); return dateFormat.parse(dateTime).getTime(); } + + public static long getOSMemorySize() { + OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); + + if (osBean instanceof com.sun.management.OperatingSystemMXBean) { + com.sun.management.OperatingSystemMXBean extendedOsBean = (com.sun.management.OperatingSystemMXBean) osBean; + return extendedOsBean.getTotalPhysicalMemorySize(); + } else { + System.out.println("OS Bean not available."); + } + return -1; + } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java index 8a36bb03f67..97a9ec3288d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java @@ -1,7 +1,9 @@ package com.linkedin.venice.integration.utils; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_OPTIONS_USE_DIRECT_READS; import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES; import static com.linkedin.venice.ConfigKeys.ADMIN_PORT; import static com.linkedin.venice.ConfigKeys.CLUSTER_DISCOVERY_D2_SERVICE; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; @@ -261,6 +263,8 @@ static StatefulServiceProvider generateService( .put(SERVER_INGESTION_HEARTBEAT_INTERVAL_MS, 5000) .put(SERVER_LEADER_COMPLETE_STATE_CHECK_IN_FOLLOWER_VALID_INTERVAL_MS, 5000) .put(SERVER_RESUBSCRIPTION_TRIGGERED_BY_VERSION_INGESTION_CONTEXT_CHANGE_ENABLED, true) + .put(ROCKSDB_BLOCK_CACHE_SIZE_IN_BYTES, 8 * 1024 * 1024 * 1024L) + .put(ROCKSDB_RMD_BLOCK_CACHE_SIZE_IN_BYTES, 1024 * 1024 * 1024L) .put(SERVER_DELETE_UNASSIGNED_PARTITIONS_ON_STARTUP, serverDeleteUnassignedPartitionsOnStartup); if (sslToKafka) { serverPropsBuilder.put(KAFKA_SECURITY_PROTOCOL, PubSubSecurityProtocol.SSL.name());