Skip to content

Commit b68eb16

Browse files
gaojieliugaojieliu
authored
[router][server] ACL optimization (#1521)
* [router][server] ACL optimization High-level idea: 1. For a keep-alive connection, the client cert will never change, so for the same store, it is useless to validate each request on the same connection. 2. In Server, there is an Acl Handler called `ServerAclHandler`, which is used to validate whether the connection is from Venice Router or not via static ACL. For each connection, Server will buffer the ACL check result in this attribute key: `SERVER_ACL_APPROVED_ATTRIBUTE_KEY`. And the ACL check result won't change during the lifetime of the Server instance. 3. In both Router and Server, there is a store-level ACL check, which can change during the lifetime of the Router/Server (ACL added/removed). The caching idea is a little different from #2, and it will maintain a cache map in the original connection, and for all the requests coming from this particular connection, it will check the acl check cache map first, and it will follow the previous result if the cache entry is not expired. If there is no such entry or the cache entry is expired, it will resort to the underlying access control to update the cache map. In theory, the acl check against the access controller will be minimized a lot. New config: acl.in.memory.cache.ttl.ms: 60000 (by default)
1 parent aa7dd9a commit b68eb16

File tree

13 files changed

+328
-32
lines changed

13 files changed

+328
-32
lines changed

‎clients/da-vinci-client/src/main/java/com/linkedin/davinci/config/VeniceServerConfig.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import static com.linkedin.davinci.ingestion.utils.IsolatedIngestionUtils.INGESTION_ISOLATION_CONFIG_PREFIX;
44
import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_TOTAL_MEMTABLE_USAGE_CAP_IN_BYTES;
55
import static com.linkedin.venice.ConfigConstants.DEFAULT_MAX_RECORD_SIZE_BYTES_BACKFILL;
6+
import static com.linkedin.venice.ConfigKeys.ACL_IN_MEMORY_CACHE_TTL_MS;
67
import static com.linkedin.venice.ConfigKeys.AUTOCREATE_DATA_PATH;
78
import static com.linkedin.venice.ConfigKeys.BLOB_TRANSFER_DISABLED_OFFSET_LAG_THRESHOLD;
89
import static com.linkedin.venice.ConfigKeys.BLOB_TRANSFER_MANAGER_ENABLED;
@@ -584,6 +585,7 @@ public class VeniceServerConfig extends VeniceClusterConfig {
584585
private final int zstdDictCompressionLevel;
585586
private final long maxWaitAfterUnsubscribeMs;
586587
private final boolean deleteUnassignedPartitionsOnStartup;
588+
private final int aclInMemoryCacheTTLMs;
587589

588590
public VeniceServerConfig(VeniceProperties serverProperties) throws ConfigurationException {
589591
this(serverProperties, Collections.emptyMap());
@@ -988,6 +990,8 @@ public VeniceServerConfig(VeniceProperties serverProperties, Map<String, Map<Str
988990

989991
deleteUnassignedPartitionsOnStartup =
990992
serverProperties.getBoolean(SERVER_DELETE_UNASSIGNED_PARTITIONS_ON_STARTUP, false);
993+
aclInMemoryCacheTTLMs = serverProperties.getInt(ACL_IN_MEMORY_CACHE_TTL_MS, -1); // acl caching is disabled by
994+
// default
991995
}
992996

993997
long extractIngestionMemoryLimit(
@@ -1801,4 +1805,8 @@ public long getMaxWaitAfterUnsubscribeMs() {
18011805
public boolean isDeleteUnassignedPartitionsOnStartupEnabled() {
18021806
return deleteUnassignedPartitionsOnStartup;
18031807
}
1808+
1809+
public int getAclInMemoryCacheTTLMs() {
1810+
return aclInMemoryCacheTTLMs;
1811+
}
18041812
}

‎internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2442,4 +2442,10 @@ private ConfigKeys() {
24422442
public static final String CONTROLLER_DEFERRED_VERSION_SWAP_SLEEP_MS = "controller.deferred.version.swap.sleep.ms";
24432443
public static final String CONTROLLER_DEFERRED_VERSION_SWAP_SERVICE_ENABLED =
24442444
"controller.deferred.version.swap.service.enabled";
2445+
2446+
/*
2447+
* Both Router and Server will maintain an in-memory cache for connection-level ACLs and the following config
2448+
* controls the TTL of the cache per entry.
2449+
*/
2450+
public static final String ACL_IN_MEMORY_CACHE_TTL_MS = "acl.in.memory.cache.ttl.ms";
24452451
}

‎internal/venice-common/src/main/java/com/linkedin/venice/acl/handler/AbstractStoreAclHandler.java

Lines changed: 80 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,21 @@
77
import com.linkedin.venice.acl.DynamicAccessController;
88
import com.linkedin.venice.authorization.IdentityParser;
99
import com.linkedin.venice.common.VeniceSystemStoreUtils;
10+
import com.linkedin.venice.listener.ServerHandlerUtils;
1011
import com.linkedin.venice.meta.ReadOnlyStoreRepository;
1112
import com.linkedin.venice.meta.Store;
1213
import com.linkedin.venice.utils.NettyUtils;
14+
import com.linkedin.venice.utils.SystemTime;
15+
import com.linkedin.venice.utils.Time;
16+
import com.linkedin.venice.utils.concurrent.VeniceConcurrentHashMap;
17+
import io.netty.channel.Channel;
1318
import io.netty.channel.ChannelHandler;
1419
import io.netty.channel.ChannelHandlerContext;
1520
import io.netty.channel.SimpleChannelInboundHandler;
21+
import io.netty.handler.codec.http.HttpMethod;
1622
import io.netty.handler.codec.http.HttpRequest;
1723
import io.netty.handler.codec.http.HttpResponseStatus;
24+
import io.netty.util.AttributeKey;
1825
import io.netty.util.ReferenceCountUtil;
1926
import java.net.URI;
2027
import java.security.cert.X509Certificate;
@@ -29,7 +36,25 @@
2936
*/
3037
@ChannelHandler.Sharable
3138
public abstract class AbstractStoreAclHandler<REQUEST_TYPE> extends SimpleChannelInboundHandler<HttpRequest> {
39+
private static class CachedAcl {
40+
AccessResult accessResult;
41+
long timestamp;
42+
43+
public CachedAcl(AccessResult accessResult, long timestamp) {
44+
this.accessResult = accessResult;
45+
this.timestamp = timestamp;
46+
}
47+
}
48+
3249
private static final Logger LOGGER = LogManager.getLogger(AbstractStoreAclHandler.class);
50+
public static final String STORE_ACL_CHECK_RESULT = "STORE_ACL_CHECK_RESULT_ATTRIBUTE_KEY";
51+
public static final AttributeKey<VeniceConcurrentHashMap<String, CachedAcl>> STORE_ACL_CHECK_RESULT_ATTRIBUTE_KEY =
52+
AttributeKey.valueOf(STORE_ACL_CHECK_RESULT);
53+
private static final byte[] BAD_REQUEST_RESPONSE = "Unexpected! Original channel should not be null".getBytes();
54+
55+
private final int cacheTTLMs;
56+
private final Time time;
57+
private final boolean aclCacheEnabled;
3358

3459
private final IdentityParser identityParser;
3560
private final ReadOnlyStoreRepository metadataRepository;
@@ -38,12 +63,25 @@ public abstract class AbstractStoreAclHandler<REQUEST_TYPE> extends SimpleChanne
3863
public AbstractStoreAclHandler(
3964
IdentityParser identityParser,
4065
DynamicAccessController accessController,
41-
ReadOnlyStoreRepository metadataRepository) {
66+
ReadOnlyStoreRepository metadataRepository,
67+
int cacheTTLMs) {
68+
this(identityParser, accessController, metadataRepository, cacheTTLMs, new SystemTime());
69+
}
70+
71+
public AbstractStoreAclHandler(
72+
IdentityParser identityParser,
73+
DynamicAccessController accessController,
74+
ReadOnlyStoreRepository metadataRepository,
75+
int cacheTTLMs,
76+
Time time) {
4277
this.identityParser = identityParser;
4378
this.metadataRepository = metadataRepository;
4479
this.accessController = accessController
4580
.init(metadataRepository.getAllStores().stream().map(Store::getName).collect(Collectors.toList()));
4681
this.metadataRepository.registerStoreDataChangedListener(new AclCreationDeletionListener(accessController));
82+
this.cacheTTLMs = cacheTTLMs;
83+
this.time = time;
84+
this.aclCacheEnabled = cacheTTLMs > 0;
4785
}
4886

4987
/**
@@ -55,14 +93,18 @@ public AbstractStoreAclHandler(
5593
*/
5694
@Override
5795
public void channelRead0(ChannelHandlerContext ctx, HttpRequest req) throws SSLPeerUnverifiedException {
58-
if (isAccessAlreadyApproved(ctx)) {
96+
Channel originalChannel = ServerHandlerUtils.getOriginalChannel(ctx);
97+
if (originalChannel == null) {
98+
NettyUtils.setupResponseAndFlush(HttpResponseStatus.BAD_REQUEST, BAD_REQUEST_RESPONSE, false, ctx);
99+
return;
100+
}
101+
if (isAccessAlreadyApproved(originalChannel)) {
59102
ReferenceCountUtil.retain(req);
60103
ctx.fireChannelRead(req);
61104
return;
62105
}
63106

64107
String uri = req.uri();
65-
66108
// Parse resource type and store name
67109
String[] requestParts = URI.create(uri).getPath().split("/");
68110
REQUEST_TYPE requestType = validateRequest(requestParts);
@@ -89,9 +131,41 @@ public void channelRead0(ChannelHandlerContext ctx, HttpRequest req) throws SSLP
89131
return;
90132
}
91133

92-
X509Certificate clientCert = extractClientCert(ctx);
93134
String method = req.method().name();
94-
AccessResult accessResult = checkAccess(uri, clientCert, storeName, method);
135+
136+
if (!method.equals(HttpMethod.GET.name()) && !method.equals(HttpMethod.POST.name())) {
137+
// Neither get nor post method, just let it pass
138+
ReferenceCountUtil.retain(req);
139+
ctx.fireChannelRead(req);
140+
return;
141+
}
142+
AccessResult accessResult;
143+
if (aclCacheEnabled) {
144+
VeniceConcurrentHashMap<String, CachedAcl> storeAclCache =
145+
originalChannel.attr(STORE_ACL_CHECK_RESULT_ATTRIBUTE_KEY).get();
146+
if (storeAclCache == null) {
147+
originalChannel.attr(STORE_ACL_CHECK_RESULT_ATTRIBUTE_KEY).setIfAbsent(new VeniceConcurrentHashMap<>());
148+
storeAclCache = originalChannel.attr(STORE_ACL_CHECK_RESULT_ATTRIBUTE_KEY).get();
149+
}
150+
151+
accessResult = storeAclCache.compute(storeName, (ignored, value) -> {
152+
long currentTimestamp = time.getMilliseconds();
153+
if (value == null || currentTimestamp - value.timestamp > cacheTTLMs) {
154+
try {
155+
return new CachedAcl(
156+
checkAccess(uri, extractClientCert(ctx), storeName, HttpMethod.GET.name()),
157+
currentTimestamp);
158+
} catch (Exception e) {
159+
LOGGER.error("Error while checking access", e);
160+
return new CachedAcl(AccessResult.ERROR_FORBIDDEN, currentTimestamp);
161+
}
162+
} else {
163+
return value;
164+
}
165+
}).accessResult;
166+
} else {
167+
accessResult = checkAccess(uri, extractClientCert(ctx), storeName, HttpMethod.GET.name());
168+
}
95169
switch (accessResult) {
96170
case GRANTED:
97171
ReferenceCountUtil.retain(req);
@@ -109,7 +183,7 @@ public void channelRead0(ChannelHandlerContext ctx, HttpRequest req) throws SSLP
109183
}
110184
}
111185

112-
protected boolean isAccessAlreadyApproved(ChannelHandlerContext ctx) {
186+
protected boolean isAccessAlreadyApproved(Channel originalChannel) {
113187
return false;
114188
}
115189

‎internal/venice-common/src/main/java/com/linkedin/venice/listener/ServerHandlerUtils.java

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.linkedin.venice.exceptions.VeniceException;
44
import com.linkedin.venice.utils.SslUtils;
5+
import io.netty.channel.Channel;
56
import io.netty.channel.ChannelHandlerContext;
67
import io.netty.handler.ssl.SslHandler;
78
import java.security.cert.X509Certificate;
@@ -26,6 +27,27 @@ public static SslHandler extractSslHandler(ChannelHandlerContext ctx) {
2627
return sslHandler;
2728
}
2829

30+
/**
31+
* Return the channel, which contains the ssl handler and it could be the current channel (http/1.x) or the parent channel (http/2).
32+
*/
33+
public static Channel getOriginalChannel(ChannelHandlerContext ctx) {
34+
/**
35+
* Try to extract ssl handler in current channel, which is mostly for http/1.1 request.
36+
*/
37+
SslHandler sslHandler = ctx.pipeline().get(SslHandler.class);
38+
if (sslHandler != null) {
39+
return ctx.channel();
40+
}
41+
/**
42+
* Try to extract ssl handler in parent channel, which is for http/2 request.
43+
*/
44+
if (ctx.channel().parent() != null && ctx.channel().parent().pipeline().get(SslHandler.class) != null) {
45+
return ctx.channel().parent();
46+
}
47+
48+
return null;
49+
}
50+
2951
public static X509Certificate extractClientCert(ChannelHandlerContext ctx) throws SSLPeerUnverifiedException {
3052
SslHandler sslHandler = ServerHandlerUtils.extractSslHandler(ctx);
3153
if (sslHandler != null) {

‎internal/venice-common/src/test/java/com/linkedin/venice/acl/handler/AbstractStoreAclHandlerTest.java

Lines changed: 63 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
package com.linkedin.venice.acl.handler;
22

3+
import static com.linkedin.venice.acl.handler.AbstractStoreAclHandler.STORE_ACL_CHECK_RESULT_ATTRIBUTE_KEY;
34
import static org.mockito.Mockito.any;
45
import static org.mockito.Mockito.argThat;
6+
import static org.mockito.Mockito.doAnswer;
7+
import static org.mockito.Mockito.doReturn;
58
import static org.mockito.Mockito.mock;
69
import static org.mockito.Mockito.never;
710
import static org.mockito.Mockito.spy;
@@ -14,6 +17,9 @@
1417
import com.linkedin.venice.common.VeniceSystemStoreUtils;
1518
import com.linkedin.venice.helix.HelixReadOnlyStoreRepository;
1619
import com.linkedin.venice.meta.Store;
20+
import com.linkedin.venice.utils.TestMockTime;
21+
import com.linkedin.venice.utils.Time;
22+
import com.linkedin.venice.utils.concurrent.VeniceConcurrentHashMap;
1723
import io.netty.channel.Channel;
1824
import io.netty.channel.ChannelHandlerContext;
1925
import io.netty.channel.ChannelPipeline;
@@ -22,6 +28,7 @@
2228
import io.netty.handler.codec.http.HttpRequest;
2329
import io.netty.handler.codec.http.HttpResponseStatus;
2430
import io.netty.handler.ssl.SslHandler;
31+
import io.netty.util.Attribute;
2532
import java.net.SocketAddress;
2633
import java.security.cert.Certificate;
2734
import java.security.cert.X509Certificate;
@@ -33,12 +40,15 @@
3340

3441

3542
public class AbstractStoreAclHandlerTest {
43+
private static int CACHE_TTL_MS = 1000;
3644
private IdentityParser identityParser;
3745
private DynamicAccessController accessController;
3846
private HelixReadOnlyStoreRepository metadataRepo;
3947
private ChannelHandlerContext ctx;
48+
private Channel channel;
4049
private HttpRequest req;
4150
private Store store;
51+
private Time mockTime;
4252
private boolean[] needsAcl = { true };
4353
private boolean[] hasAccess = { false };
4454
private boolean[] hasAcl = { false };
@@ -83,12 +93,18 @@ public void setUp() throws Exception {
8393
when(sslSession.getPeerCertificates()).thenReturn(new Certificate[] { cert });
8494

8595
// Host
86-
Channel channel = mock(Channel.class);
96+
channel = mock(Channel.class);
8797
when(ctx.channel()).thenReturn(channel);
8898
SocketAddress address = mock(SocketAddress.class);
8999
when(channel.remoteAddress()).thenReturn(address);
90100

101+
Attribute<VeniceConcurrentHashMap> aclCacheAttr = mock(Attribute.class);
102+
doAnswer((ignored) -> new VeniceConcurrentHashMap<>()).when(aclCacheAttr).get();
103+
doReturn(aclCacheAttr).when(channel).attr(STORE_ACL_CHECK_RESULT_ATTRIBUTE_KEY);
104+
91105
when(req.method()).thenReturn(HttpMethod.GET);
106+
107+
mockTime = new TestMockTime(0);
92108
}
93109

94110
@Test
@@ -104,6 +120,38 @@ public void noAclNeeded() throws Exception {
104120
verify(ctx, times(32)).fireChannelRead(req);
105121
}
106122

123+
@Test
124+
public void testAclCache() throws Exception {
125+
hasAccess[0] = true;
126+
hasStore[0] = true;
127+
Attribute<VeniceConcurrentHashMap> aclCacheAttr = mock(Attribute.class);
128+
doReturn(new VeniceConcurrentHashMap<>()).when(aclCacheAttr).get();
129+
doReturn(aclCacheAttr).when(channel).attr(STORE_ACL_CHECK_RESULT_ATTRIBUTE_KEY);
130+
enumerate(hasAcl);
131+
hasAccess[0] = true;
132+
hasStore[0] = true;
133+
enumerate(hasAcl);
134+
verify(accessController).hasAccess(any(), any(), any());
135+
// Simulate cache expiration
136+
mockTime.sleep(CACHE_TTL_MS + 1);
137+
hasAccess[0] = true;
138+
hasStore[0] = true;
139+
enumerate(hasAcl);
140+
verify(accessController, times(2)).hasAccess(any(), any(), any());
141+
142+
// Test cache disabled
143+
// New metadataRepo mock and aclHandler every update since thenThrow cannot be re-mocked.
144+
hasAccess[0] = true;
145+
hasStore[0] = true;
146+
metadataRepo = mock(HelixReadOnlyStoreRepository.class);
147+
AbstractStoreAclHandler aclHandler =
148+
spy(new MockStoreAclHandler(identityParser, accessController, metadataRepo, -1, mockTime));
149+
update();
150+
aclHandler.channelRead0(ctx, req);
151+
aclHandler.channelRead0(ctx, req);
152+
verify(accessController, times(4)).hasAccess(any(), any(), any());
153+
}
154+
107155
@Test
108156
public void accessGranted() throws Exception {
109157
hasAccess[0] = true;
@@ -299,7 +347,8 @@ private void enumerate(boolean[]... conditions) throws Exception {
299347
}
300348
// New metadataRepo mock and aclHandler every update since thenThrow cannot be re-mocked.
301349
metadataRepo = mock(HelixReadOnlyStoreRepository.class);
302-
AbstractStoreAclHandler aclHandler = spy(new MockStoreAclHandler(identityParser, accessController, metadataRepo));
350+
AbstractStoreAclHandler aclHandler =
351+
spy(new MockStoreAclHandler(identityParser, accessController, metadataRepo, mockTime));
303352
update();
304353
aclHandler.channelRead0(ctx, req);
305354
}
@@ -334,8 +383,18 @@ private static class MockStoreAclHandler extends AbstractStoreAclHandler<TestReq
334383
public MockStoreAclHandler(
335384
IdentityParser identityParser,
336385
DynamicAccessController accessController,
337-
HelixReadOnlyStoreRepository metadataRepository) {
338-
super(identityParser, accessController, metadataRepository);
386+
HelixReadOnlyStoreRepository metadataRepository,
387+
Time mockTime) {
388+
super(identityParser, accessController, metadataRepository, CACHE_TTL_MS, mockTime);
389+
}
390+
391+
public MockStoreAclHandler(
392+
IdentityParser identityParser,
393+
DynamicAccessController accessController,
394+
HelixReadOnlyStoreRepository metadataRepository,
395+
int cacheTTLMs,
396+
Time mockTime) {
397+
super(identityParser, accessController, metadataRepository, cacheTTLMs, mockTime);
339398
}
340399

341400
@Override

‎services/venice-router/src/main/java/com/linkedin/venice/router/RouterServer.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -681,7 +681,11 @@ public boolean startInner() throws Exception {
681681

682682
RouterSslVerificationHandler routerSslVerificationHandler = new RouterSslVerificationHandler(securityStats);
683683
RouterStoreAclHandler aclHandler = accessController.isPresent()
684-
? new RouterStoreAclHandler(identityParser, accessController.get(), metadataRepository)
684+
? new RouterStoreAclHandler(
685+
identityParser,
686+
accessController.get(),
687+
metadataRepository,
688+
config.getAclInMemoryCacheTTLMs())
685689
: null;
686690
final SslInitializer sslInitializer;
687691
if (sslFactory.isPresent()) {

0 commit comments

Comments
 (0)