Skip to content

Commit cc6e3db

Browse files
arjun4084346arjun4084346
authored
[compat] [controller] add rt topic name in store config (#1345)
* add a new field in store and store-version's hybrid configs and add apis in Utils to get real time topic name using these configs * address review comments
1 parent b2d29b8 commit cc6e3db

File tree

16 files changed

+1862
-12
lines changed

16 files changed

+1862
-12
lines changed

Diff for: ‎internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerApiConstants.java

+1
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ public class ControllerApiConstants {
5555
public static final String TIME_LAG_TO_GO_ONLINE = "time_lag_to_go_online";
5656
public static final String DATA_REPLICATION_POLICY = "data_replication_policy";
5757
public static final String BUFFER_REPLAY_POLICY = "buffer_replay_policy";
58+
public static final String REAL_TIME_TOPIC_NAME = "real_time_topic_name";
5859
public static final String COMPRESSION_STRATEGY = "compression_strategy";
5960
public static final String CLIENT_DECOMPRESSION_ENABLED = "client_decompression_enabled";
6061
public static final String CHUNKING_ENABLED = "chunking_enabled";

Diff for: ‎internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/UpdateStoreQueryParams.java

+9
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import static com.linkedin.venice.controllerapi.ControllerApiConstants.PUSH_STREAM_SOURCE_ADDRESS;
4646
import static com.linkedin.venice.controllerapi.ControllerApiConstants.READ_COMPUTATION_ENABLED;
4747
import static com.linkedin.venice.controllerapi.ControllerApiConstants.READ_QUOTA_IN_CU;
48+
import static com.linkedin.venice.controllerapi.ControllerApiConstants.REAL_TIME_TOPIC_NAME;
4849
import static com.linkedin.venice.controllerapi.ControllerApiConstants.REGIONS_FILTER;
4950
import static com.linkedin.venice.controllerapi.ControllerApiConstants.REGULAR_VERSION_ETL_ENABLED;
5051
import static com.linkedin.venice.controllerapi.ControllerApiConstants.REPLICATE_ALL_CONFIGS;
@@ -359,6 +360,14 @@ public Optional<BufferReplayPolicy> getHybridBufferReplayPolicy() {
359360
return Optional.ofNullable(params.get(BUFFER_REPLAY_POLICY)).map(BufferReplayPolicy::valueOf);
360361
}
361362

363+
public UpdateStoreQueryParams setRealTimeTopicName(String realTimeTopicName) {
364+
return putString(REAL_TIME_TOPIC_NAME, realTimeTopicName);
365+
}
366+
367+
public Optional<String> getRealTimeTopicName() {
368+
return getString(REAL_TIME_TOPIC_NAME);
369+
}
370+
362371
public UpdateStoreQueryParams setAccessControlled(boolean accessControlled) {
363372
return putBoolean(ACCESS_CONTROLLED, accessControlled);
364373
}

Diff for: ‎internal/venice-common/src/main/java/com/linkedin/venice/meta/HybridStoreConfig.java

+4
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,9 @@ public interface HybridStoreConfig extends DataModelBackedStructure<StoreHybridC
2626

2727
BufferReplayPolicy getBufferReplayPolicy();
2828

29+
String getRealTimeTopicName();
30+
31+
void setRealTimeTopicName(String realTimeTopicName);
32+
2933
HybridStoreConfig clone();
3034
}

Diff for: ‎internal/venice-common/src/main/java/com/linkedin/venice/meta/HybridStoreConfigImpl.java

+33-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.linkedin.venice.meta;
22

3+
import com.fasterxml.jackson.annotation.JsonCreator;
34
import com.fasterxml.jackson.annotation.JsonIgnore;
45
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
56
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -17,15 +18,33 @@ public class HybridStoreConfigImpl implements HybridStoreConfig {
1718
public static final long DEFAULT_REWIND_TIME_IN_SECONDS = Time.SECONDS_PER_DAY;
1819
public static final long DEFAULT_HYBRID_TIME_LAG_THRESHOLD = -1L;
1920
public static final long DEFAULT_HYBRID_OFFSET_LAG_THRESHOLD = 1000L;
21+
public static final String DEFAULT_REAL_TIME_TOPIC_NAME = "";
2022

2123
private final StoreHybridConfig hybridConfig;
2224

25+
public HybridStoreConfigImpl(
26+
long rewindTimeInSeconds,
27+
long offsetLagThresholdToGoOnline,
28+
long producerTimestampLagThresholdToGoOnlineInSeconds,
29+
DataReplicationPolicy dataReplicationPolicy,
30+
BufferReplayPolicy bufferReplayPolicy) {
31+
this(
32+
rewindTimeInSeconds,
33+
offsetLagThresholdToGoOnline,
34+
producerTimestampLagThresholdToGoOnlineInSeconds,
35+
dataReplicationPolicy,
36+
bufferReplayPolicy,
37+
DEFAULT_REAL_TIME_TOPIC_NAME);
38+
}
39+
40+
@JsonCreator
2341
public HybridStoreConfigImpl(
2442
@JsonProperty("rewindTimeInSeconds") long rewindTimeInSeconds,
2543
@JsonProperty("offsetLagThresholdToGoOnline") long offsetLagThresholdToGoOnline,
2644
@JsonProperty("producerTimestampLagThresholdToGoOnlineInSeconds") long producerTimestampLagThresholdToGoOnlineInSeconds,
2745
@JsonProperty("dataReplicationPolicy") DataReplicationPolicy dataReplicationPolicy,
28-
@JsonProperty("bufferReplayPolicy") BufferReplayPolicy bufferReplayPolicy) {
46+
@JsonProperty("bufferReplayPolicy") BufferReplayPolicy bufferReplayPolicy,
47+
@JsonProperty("realTimeTopicName") String realTimeTopicName) {
2948
this.hybridConfig = new StoreHybridConfig();
3049
this.hybridConfig.rewindTimeInSeconds = rewindTimeInSeconds;
3150
this.hybridConfig.offsetLagThresholdToGoOnline = offsetLagThresholdToGoOnline;
@@ -37,6 +56,7 @@ public HybridStoreConfigImpl(
3756
: dataReplicationPolicy.getValue();
3857
this.hybridConfig.bufferReplayPolicy =
3958
bufferReplayPolicy == null ? BufferReplayPolicy.REWIND_FROM_EOP.getValue() : bufferReplayPolicy.getValue();
59+
this.hybridConfig.realTimeTopicName = realTimeTopicName;
4060
}
4161

4262
HybridStoreConfigImpl(StoreHybridConfig config) {
@@ -83,6 +103,16 @@ public BufferReplayPolicy getBufferReplayPolicy() {
83103
return BufferReplayPolicy.valueOf(this.hybridConfig.bufferReplayPolicy);
84104
}
85105

106+
@Override
107+
public String getRealTimeTopicName() {
108+
return this.hybridConfig.realTimeTopicName.toString();
109+
}
110+
111+
@Override
112+
public void setRealTimeTopicName(String realTimeTopicName) {
113+
this.hybridConfig.realTimeTopicName = realTimeTopicName;
114+
}
115+
86116
@Override
87117
public StoreHybridConfig dataModel() {
88118
return this.hybridConfig;
@@ -112,6 +142,7 @@ public HybridStoreConfig clone() {
112142
getOffsetLagThresholdToGoOnline(),
113143
getProducerTimestampLagThresholdToGoOnlineInSeconds(),
114144
getDataReplicationPolicy(),
115-
getBufferReplayPolicy());
145+
getBufferReplayPolicy(),
146+
getRealTimeTopicName());
116147
}
117148
}

Diff for: ‎internal/venice-common/src/main/java/com/linkedin/venice/meta/ReadOnlyStore.java

+15
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,16 @@ public BufferReplayPolicy getBufferReplayPolicy() {
142142
return this.delegate.getBufferReplayPolicy();
143143
}
144144

145+
@Override
146+
public String getRealTimeTopicName() {
147+
return this.delegate.getRealTimeTopicName();
148+
}
149+
150+
@Override
151+
public void setRealTimeTopicName(String realTimeTopicName) {
152+
throw new UnsupportedOperationException();
153+
}
154+
145155
@Override
146156
public HybridStoreConfig clone() {
147157
return this.delegate.clone();
@@ -513,6 +523,11 @@ public void setUseVersionLevelIncrementalPushEnabled(boolean versionLevelIncreme
513523
throw new UnsupportedOperationException();
514524
}
515525

526+
@Override
527+
public boolean isHybrid() {
528+
return this.delegate.isHybrid();
529+
}
530+
516531
@Override
517532
public HybridStoreConfig getHybridStoreConfig() {
518533
HybridStoreConfig config = this.delegate.getHybridStoreConfig();

Diff for: ‎internal/venice-common/src/main/java/com/linkedin/venice/meta/Version.java

+2
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,8 @@ default void setLeaderFollowerModelEnabled(boolean leaderFollowerModelEnabled) {
174174

175175
void setUseVersionLevelIncrementalPushEnabled(boolean versionLevelIncrementalPushEnabled);
176176

177+
boolean isHybrid();
178+
177179
HybridStoreConfig getHybridStoreConfig();
178180

179181
void setHybridStoreConfig(HybridStoreConfig hybridConfig);

Diff for: ‎internal/venice-common/src/main/java/com/linkedin/venice/meta/VersionImpl.java

+5
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,11 @@ public void setUseVersionLevelIncrementalPushEnabled(boolean versionLevelIncreme
298298
this.storeVersion.useVersionLevelIncrementalPushEnabled = versionLevelIncrementalPushEnabled;
299299
}
300300

301+
@Override
302+
public boolean isHybrid() {
303+
return getHybridStoreConfig() != null;
304+
}
305+
301306
@Override
302307
public HybridStoreConfig getHybridStoreConfig() {
303308
if (this.storeVersion.hybridConfig == null) {

Diff for: ‎internal/venice-common/src/main/java/com/linkedin/venice/serialization/avro/AvroProtocolDefinition.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public enum AvroProtocolDefinition {
7272
*
7373
* TODO: Move AdminOperation to venice-common module so that we can properly reference it here.
7474
*/
75-
ADMIN_OPERATION(82, SpecificData.get().getSchema(ByteBuffer.class), "AdminOperation"),
75+
ADMIN_OPERATION(83, SpecificData.get().getSchema(ByteBuffer.class), "AdminOperation"),
7676

7777
/**
7878
* Single chunk of a large multi-chunk value. Just a bunch of bytes.
@@ -143,7 +143,7 @@ public enum AvroProtocolDefinition {
143143
/**
144144
* Value schema for metadata system store.
145145
*/
146-
METADATA_SYSTEM_SCHEMA_STORE(25, StoreMetaValue.class),
146+
METADATA_SYSTEM_SCHEMA_STORE(26, StoreMetaValue.class),
147147

148148
/**
149149
* Key schema for push status system store.

Diff for: ‎internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java

+98
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,14 @@
1313
import com.linkedin.venice.helix.HelixState;
1414
import com.linkedin.venice.helix.Replica;
1515
import com.linkedin.venice.helix.ResourceAssignment;
16+
import com.linkedin.venice.meta.HybridStoreConfig;
1617
import com.linkedin.venice.meta.Instance;
1718
import com.linkedin.venice.meta.Partition;
1819
import com.linkedin.venice.meta.PartitionAssignment;
1920
import com.linkedin.venice.meta.ReadOnlyStoreRepository;
2021
import com.linkedin.venice.meta.RoutingDataRepository;
2122
import com.linkedin.venice.meta.Store;
23+
import com.linkedin.venice.meta.StoreInfo;
2224
import com.linkedin.venice.meta.Version;
2325
import com.linkedin.venice.pubsub.api.PubSubTopic;
2426
import com.linkedin.venice.pubsub.api.PubSubTopicPartition;
@@ -66,6 +68,7 @@
6668
import org.apache.http.HttpStatus;
6769
import org.apache.logging.log4j.LogManager;
6870
import org.apache.logging.log4j.Logger;
71+
import org.apache.logging.log4j.util.Strings;
6972

7073

7174
/**
@@ -529,6 +532,101 @@ public static File getTempDataDirectory(String prefix) {
529532
}
530533
}
531534

535+
/** This method should only be used for system stores.
536+
* For other stores, use {@link Utils#getRealTimeTopicName(Store)}, {@link Utils#getRealTimeTopicName(StoreInfo)} or
537+
* {@link Utils#getRealTimeTopicName(Version)}
538+
*/
539+
public static String composeRealTimeTopic(String storeName) {
540+
return storeName + Version.REAL_TIME_TOPIC_SUFFIX;
541+
}
542+
543+
/**
544+
* It follows the following order to search for real time topic name,
545+
* i) current store-version config, ii) store config, iii) other store-version configs, iv) default name
546+
*/
547+
public static String getRealTimeTopicName(Store store) {
548+
return getRealTimeTopicName(
549+
store.getName(),
550+
store.getVersions(),
551+
store.getCurrentVersion(),
552+
store.getHybridStoreConfig());
553+
}
554+
555+
public static String getRealTimeTopicName(StoreInfo storeInfo) {
556+
return getRealTimeTopicName(
557+
storeInfo.getName(),
558+
storeInfo.getVersions(),
559+
storeInfo.getCurrentVersion(),
560+
storeInfo.getHybridStoreConfig());
561+
}
562+
563+
public static String getRealTimeTopicName(Version version) {
564+
HybridStoreConfig hybridStoreConfig = version.getHybridStoreConfig();
565+
if (hybridStoreConfig != null) {
566+
String realTimeTopicName = version.getHybridStoreConfig().getRealTimeTopicName();
567+
return getRealTimeTopicNameIfEmpty(realTimeTopicName, version.getStoreName());
568+
} else {
569+
// if the version is not hybrid, caller should not ask for the real time topic,
570+
// but unfortunately that happens, so instead of throwing exception, we just return a default name.
571+
return composeRealTimeTopic(version.getStoreName());
572+
}
573+
}
574+
575+
static String getRealTimeTopicName(
576+
String storeName,
577+
List<Version> versions,
578+
int currentVersionNumber,
579+
HybridStoreConfig hybridStoreConfig) {
580+
if (currentVersionNumber < 1) {
581+
return composeRealTimeTopic(storeName);
582+
}
583+
584+
Optional<Version> currentVersion =
585+
versions.stream().filter(version -> version.getNumber() == currentVersionNumber).findFirst();
586+
if (currentVersion.isPresent() && currentVersion.get().isHybrid()) {
587+
String realTimeTopicName = currentVersion.get().getHybridStoreConfig().getRealTimeTopicName();
588+
if (Strings.isNotBlank(realTimeTopicName)) {
589+
return realTimeTopicName;
590+
}
591+
}
592+
593+
if (hybridStoreConfig != null) {
594+
String realTimeTopicName = hybridStoreConfig.getRealTimeTopicName();
595+
return getRealTimeTopicNameIfEmpty(realTimeTopicName, storeName);
596+
}
597+
598+
Set<String> realTimeTopicNames = new HashSet<>();
599+
600+
for (Version version: versions) {
601+
try {
602+
if (version.isHybrid()) {
603+
String realTimeTopicName = version.getHybridStoreConfig().getRealTimeTopicName();
604+
if (Strings.isNotBlank(realTimeTopicName)) {
605+
realTimeTopicNames.add(realTimeTopicName);
606+
}
607+
}
608+
} catch (VeniceException e) {
609+
// just try another version
610+
}
611+
}
612+
613+
if (realTimeTopicNames.size() > 1) {
614+
LOGGER.warn(
615+
"Store " + storeName + " and current version are not hybrid, yet " + realTimeTopicNames.size()
616+
+ " older versions are using real time topics. Will return one of them.");
617+
}
618+
619+
if (!realTimeTopicNames.isEmpty()) {
620+
return realTimeTopicNames.iterator().next();
621+
}
622+
623+
return composeRealTimeTopic(storeName);
624+
}
625+
626+
private static String getRealTimeTopicNameIfEmpty(String realTimeTopicName, String storeName) {
627+
return Strings.isBlank(realTimeTopicName) ? composeRealTimeTopic(storeName) : realTimeTopicName;
628+
}
629+
532630
private static class TimeUnitInfo {
533631
String suffix;
534632
int multiplier;

0 commit comments

Comments
 (0)