Skip to content

Commit 0d7cafe

Browse files
auden-woolfsonauden-woolfson
committed
clean files
1 parent ccaa707 commit 0d7cafe

File tree

22 files changed

+3186
-4288
lines changed

22 files changed

+3186
-4288
lines changed

‎presto-hive-hadoop2/pom.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,12 @@
128128
<artifactId>presto-hive</artifactId>
129129
<type>test-jar</type>
130130
<scope>test</scope>
131+
<exclusions>
132+
<exclusion>
133+
<groupId>org.elasticsearch</groupId>
134+
<artifactId>elasticsearch</artifactId>
135+
</exclusion>
136+
</exclusions>
131137
</dependency>
132138

133139
<dependency>

‎presto-hudi/pom.xml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,16 @@
173173
<artifactId>presto-hive</artifactId>
174174
<type>test-jar</type>
175175
<scope>test</scope>
176+
<exclusions>
177+
<exclusion>
178+
<groupId>org.elasticsearch</groupId>
179+
<artifactId>elasticsearch</artifactId>
180+
</exclusion>
181+
<exclusion>
182+
<groupId>org.apache.httpcomponents</groupId>
183+
<artifactId>httpcore</artifactId>
184+
</exclusion>
185+
</exclusions>
176186
</dependency>
177187

178188
<dependency>

‎presto-password-authenticators/pom.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
<dependency>
4646
<groupId>com.google.inject</groupId>
4747
<artifactId>guice</artifactId>
48+
<version>5.1.0</version>
4849
</dependency>
4950

5051
<dependency>

‎presto-router/pom.xml

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,11 @@
5757
<artifactId>http-server</artifactId>
5858
</dependency>
5959

60+
<dependency>
61+
<groupId>com.facebook.airlift</groupId>
62+
<artifactId>stats</artifactId>
63+
</dependency>
64+
6065
<dependency>
6166
<groupId>com.facebook.airlift</groupId>
6267
<artifactId>jaxrs</artifactId>
@@ -169,12 +174,6 @@
169174
<scope>test</scope>
170175
</dependency>
171176

172-
<dependency>
173-
<groupId>com.facebook.presto</groupId>
174-
<artifactId>presto-main</artifactId>
175-
<scope>test</scope>
176-
</dependency>
177-
178177
<dependency>
179178
<groupId>com.facebook.presto</groupId>
180179
<artifactId>presto-tpch</artifactId>
@@ -192,5 +191,10 @@
192191
<artifactId>mockwebserver</artifactId>
193192
<scope>test</scope>
194193
</dependency>
194+
195+
<dependency>
196+
<groupId>com.facebook.presto</groupId>
197+
<artifactId>presto-main</artifactId>
198+
</dependency>
195199
</dependencies>
196200
</project>

‎presto-router/src/main/java/com/facebook/presto/router/RouterModule.java

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,22 +30,18 @@
3030
import com.facebook.presto.server.PluginManagerConfig;
3131
import com.facebook.presto.server.ServerConfig;
3232
import com.facebook.presto.server.WebUiResource;
33-
import com.facebook.presto.server.security.oauth2.ForOAuth2;
3433
import com.google.inject.Binder;
3534
import com.google.inject.Scopes;
3635
import io.airlift.units.Duration;
3736

3837
import java.lang.annotation.Annotation;
39-
import java.util.concurrent.ExecutorService;
4038
import java.util.concurrent.ScheduledExecutorService;
4139

42-
import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed;
4340
import static com.facebook.airlift.concurrent.Threads.threadsNamed;
4441
import static com.facebook.airlift.configuration.ConfigBinder.configBinder;
4542
import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder;
4643
import static com.facebook.airlift.http.server.HttpServerBinder.httpServerBinder;
4744
import static com.facebook.airlift.jaxrs.JaxrsBinder.jaxrsBinder;
48-
import static java.util.concurrent.Executors.newCachedThreadPool;
4945
import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
5046
import static java.util.concurrent.TimeUnit.SECONDS;
5147

@@ -71,7 +67,7 @@ protected void setup(Binder binder)
7167
configBinder(binder).bindConfig(RouterConfig.class);
7268

7369
configBinder(binder).bindConfig(RemoteStateConfig.class);
74-
configBinder(binder).bindConfigDefaults(RemoteStateConfig.class, config -> config.setSecondsToUnhealthy(60));
70+
configBinder(binder).bindConfigDefaults(RemoteStateConfig.class, config -> config.setSecondsToUnhealthy(30));
7571

7672
// resource for serving static content
7773
jaxrsBinder(binder).bind(WebUiResource.class);
@@ -93,17 +89,13 @@ protected void setup(Binder binder)
9389
binder.bind(PredictorManager.class).in(Scopes.SINGLETON);
9490
binder.bind(RemoteQueryFactory.class).in(Scopes.SINGLETON);
9591

96-
binder.bind(RouterPluginManager.class).in(Scopes.SINGLETON);
9792
configBinder(binder).bindConfig(PluginManagerConfig.class);
9893

9994
bindHttpClient(binder, QUERY_PREDICTOR, ForQueryCpuPredictor.class, IDLE_TIMEOUT_SECOND, PREDICTOR_REQUEST_TIMEOUT_SECOND);
10095
bindHttpClient(binder, QUERY_PREDICTOR, ForQueryMemoryPredictor.class, IDLE_TIMEOUT_SECOND, PREDICTOR_REQUEST_TIMEOUT_SECOND);
10196

10297
jaxrsBinder(binder).bind(RouterResource.class);
10398
jaxrsBinder(binder).bind(ClusterStatusResource.class);
104-
105-
//binder.bind(Executor.class).annotatedWith(ForOAuth2.class).toInstance(executor);
106-
binder.bind(ExecutorService.class).annotatedWith(ForOAuth2.class).toInstance(newCachedThreadPool(daemonThreadsNamed("oauth-executor-%s")));
10799
}
108100

109101
private void bindHttpClient(Binder binder, String name, Class<? extends Annotation> annotation, int idleTimeout, int requestTimeout)

‎presto-router/src/main/java/com/facebook/presto/router/RouterResource.java

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@
1414
package com.facebook.presto.router;
1515

1616
import com.facebook.airlift.log.Logger;
17+
import com.facebook.airlift.stats.CounterStat;
1718
import com.facebook.presto.router.cluster.ClusterManager;
1819
import com.facebook.presto.router.cluster.RequestInfo;
1920
import com.google.inject.Inject;
21+
import org.weakref.jmx.Managed;
22+
import org.weakref.jmx.Nested;
2023

21-
import javax.annotation.security.PermitAll;
2224
import javax.servlet.http.HttpServletRequest;
2325
import javax.ws.rs.POST;
2426
import javax.ws.rs.Path;
@@ -35,12 +37,13 @@
3537
import static javax.ws.rs.core.MediaType.TEXT_PLAIN_TYPE;
3638
import static javax.ws.rs.core.Response.Status.BAD_GATEWAY;
3739

38-
@Path("/")
39-
@PermitAll
40+
@Path("/v1")
4041
public class RouterResource
4142
{
4243
private static final Logger log = Logger.get(RouterResource.class);
4344
private final ClusterManager clusterManager;
45+
private static final CounterStat successRedirectRequests = new CounterStat();
46+
private static final CounterStat failedRedirectRequests = new CounterStat();
4447

4548
@Inject
4649
public RouterResource(ClusterManager clusterManager)
@@ -49,23 +52,39 @@ public RouterResource(ClusterManager clusterManager)
4952
}
5053

5154
@POST
52-
@Path("/v1/statement")
55+
@Path("statement")
5356
@Produces(APPLICATION_JSON)
5457
public Response routeQuery(String statement, @Context HttpServletRequest servletRequest)
5558
{
5659
RequestInfo requestInfo = new RequestInfo(servletRequest, statement);
5760
URI coordinatorUri = clusterManager.getDestination(requestInfo).orElseThrow(() -> badRequest(BAD_GATEWAY, "No Presto cluster available"));
5861
URI statementUri = uriBuilderFrom(coordinatorUri).replacePath("/v1/statement").build();
62+
successRedirectRequests.update(1);
5963
log.info("route query to %s", statementUri);
6064
return Response.temporaryRedirect(statementUri).build();
6165
}
6266

6367
private static WebApplicationException badRequest(Response.Status status, String message)
6468
{
69+
failedRedirectRequests.update(1);
6570
throw new WebApplicationException(
6671
Response.status(status)
6772
.type(TEXT_PLAIN_TYPE)
6873
.entity(message)
6974
.build());
7075
}
76+
77+
@Managed
78+
@Nested
79+
public CounterStat getFailedRedirectRequests()
80+
{
81+
return failedRedirectRequests;
82+
}
83+
84+
@Managed
85+
@Nested
86+
public CounterStat getSuccessRedirectRequests()
87+
{
88+
return successRedirectRequests;
89+
}
7190
}

‎presto-router/src/main/java/com/facebook/presto/router/cluster/ClusterManager.java

Lines changed: 52 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
*/
1414
package com.facebook.presto.router.cluster;
1515

16+
import com.facebook.airlift.log.Logger;
1617
import com.facebook.presto.router.RouterConfig;
1718
import com.facebook.presto.router.scheduler.Scheduler;
1819
import com.facebook.presto.router.scheduler.SchedulerFactory;
@@ -24,8 +25,10 @@
2425
import com.google.common.collect.ImmutableList;
2526
import com.google.common.collect.ImmutableMap;
2627

28+
import javax.annotation.PostConstruct;
2729
import javax.inject.Inject;
2830

31+
import java.io.File;
2932
import java.net.URI;
3033
import java.util.HashMap;
3134
import java.util.List;
@@ -39,6 +42,7 @@
3942

4043
import static com.facebook.presto.router.RouterUtil.parseRouterConfig;
4144
import static com.facebook.presto.router.scheduler.SchedulerType.WEIGHTED_RANDOM_CHOICE;
45+
import static com.facebook.presto.router.scheduler.SchedulerType.WEIGHTED_ROUND_ROBIN;
4246
import static com.facebook.presto.spi.StandardErrorCode.CONFIGURATION_INVALID;
4347
import static com.google.common.base.Preconditions.checkArgument;
4448
import static com.google.common.collect.ImmutableList.toImmutableList;
@@ -47,15 +51,17 @@
4751

4852
public class ClusterManager
4953
{
50-
private final Map<String, GroupSpec> groups;
51-
private final List<SelectorRuleSpec> groupSelectors;
52-
private final SchedulerType schedulerType;
53-
private final Scheduler scheduler;
54-
private final HashMap<String, HashMap<URI, Integer>> serverWeights = new HashMap<>();
54+
private Map<String, GroupSpec> groups;
55+
private List<SelectorRuleSpec> groupSelectors;
56+
private SchedulerType schedulerType;
57+
private Scheduler scheduler;
58+
private HashMap<String, HashMap<URI, Integer>> serverWeights = new HashMap<>();
59+
private HashMap<URI, URI> discoveryURIs = new HashMap<>();
5560
private final RouterConfig routerConfig;
5661
private final ScheduledExecutorService scheduledExecutorService;
5762
private final AtomicLong lastConfigUpdate = new AtomicLong();
5863
private final RemoteInfoFactory remoteInfoFactory;
64+
private final Logger log = Logger.get(ClusterManager.class);
5965

6066
// Cluster status
6167
private final ConcurrentHashMap<URI, RemoteClusterInfo> remoteClusterInfos = new ConcurrentHashMap<>();
@@ -68,13 +74,20 @@ public ClusterManager(RouterConfig config, @ForClusterManager ScheduledExecutorS
6874
this.scheduledExecutorService = scheduledExecutorService;
6975
RouterSpec routerSpec = parseRouterConfig(config)
7076
.orElseThrow(() -> new PrestoException(CONFIGURATION_INVALID, "Failed to load router config"));
71-
7277
this.groups = ImmutableMap.copyOf(routerSpec.getGroups().stream().collect(toMap(GroupSpec::getName, group -> group)));
7378
this.groupSelectors = ImmutableList.copyOf(routerSpec.getSelectors());
7479
this.schedulerType = routerSpec.getSchedulerType();
7580
this.scheduler = new SchedulerFactory(routerSpec.getSchedulerType()).create();
7681
this.remoteInfoFactory = requireNonNull(remoteInfoFactory, "remoteInfoFactory is null");
7782
this.initializeServerWeights();
83+
this.initializeMembersDiscoveryURI();
84+
List<URI> allClusters = getAllClusters();
85+
allClusters.forEach(uri -> {
86+
log.info("Attaching cluster %s to the router", uri.getHost());
87+
remoteClusterInfos.put(uri, remoteInfoFactory.createRemoteClusterInfo(discoveryURIs.get(uri)));
88+
remoteQueryInfos.put(uri, remoteInfoFactory.createRemoteQueryInfo(discoveryURIs.get(uri)));
89+
log.info("Successfully attached cluster %s to the router. Queries will be routed to cluster after successful health check", uri.getHost());
90+
});
7891
}
7992

8093
@PostConstruct
@@ -92,13 +105,28 @@ public void startConfigReloadTask()
92105
this.schedulerType = routerSpec.getSchedulerType();
93106
this.scheduler = new SchedulerFactory(routerSpec.getSchedulerType()).create();
94107
this.initializeServerWeights();
108+
this.initializeMembersDiscoveryURI();
109+
List<URI> allClusters = getAllClusters();
110+
allClusters.forEach(uri -> {
111+
if (!remoteClusterInfos.containsKey(uri)) {
112+
log.info("Attaching cluster %s to the router", uri.getHost());
113+
remoteClusterInfos.put(uri, remoteInfoFactory.createRemoteClusterInfo(discoveryURIs.get(uri)));
114+
remoteQueryInfos.put(uri, remoteInfoFactory.createRemoteQueryInfo(discoveryURIs.get(uri)));
115+
log.info("Successfully attached cluster %s to the router. Queries will be routed to cluster after successful health check", uri.getHost());
116+
}
117+
});
118+
for (URI uri : remoteClusterInfos.keySet()) {
119+
if (!allClusters.contains(uri)) {
120+
log.info("Removing cluster %s from the router", uri.getHost());
121+
remoteClusterInfos.remove(uri);
122+
remoteQueryInfos.remove(uri);
123+
discoveryURIs.remove(uri);
124+
log.info("Successfully removed cluster %s from the router", uri.getHost());
125+
}
126+
}
95127
lastConfigUpdate.set(newConfigUpdateTime);
96128
}
97-
}, 0L, (long) 30, TimeUnit.SECONDS);
98-
getAllClusters().forEach(uri -> {
99-
remoteClusterInfos.put(uri, remoteInfoFactory.createRemoteClusterInfo(uri));
100-
remoteQueryInfos.put(uri, remoteInfoFactory.createRemoteQueryInfo(uri));
101-
});
129+
}, 0L, (long) 5, TimeUnit.SECONDS);
102130
}
103131

104132
public List<URI> getAllClusters()
@@ -123,14 +151,14 @@ public Optional<URI> getDestination(RequestInfo requestInfo)
123151
.collect(Collectors.toList());
124152

125153
if (healthyClusterURIs.isEmpty()) {
154+
log.info("Healthy cluster not found!");
126155
return Optional.empty();
127156
}
128157

129158
scheduler.setCandidates(healthyClusterURIs);
130-
if (schedulerType == WEIGHTED_RANDOM_CHOICE) {
159+
if (schedulerType == WEIGHTED_RANDOM_CHOICE || schedulerType == WEIGHTED_ROUND_ROBIN) {
131160
scheduler.setWeights(serverWeights.get(groupSpec.getName()));
132161
}
133-
134162
return scheduler.getDestination(requestInfo.getUser());
135163
}
136164

@@ -155,6 +183,17 @@ private void initializeServerWeights()
155183
});
156184
}
157185

186+
private void initializeMembersDiscoveryURI()
187+
{
188+
groups.forEach((name, groupSpec) -> {
189+
List<URI> members = groupSpec.getMembers();
190+
List<URI> membersDiscoveryURI = groupSpec.getMembersDiscoveryURI();
191+
for (int i = 0; i < members.size(); i++) {
192+
discoveryURIs.put(members.get(i), membersDiscoveryURI.get(i));
193+
}
194+
});
195+
}
196+
158197
public ConcurrentHashMap<URI, RemoteClusterInfo> getRemoteClusterInfos()
159198
{
160199
return remoteClusterInfos;

‎presto-router/src/main/java/com/facebook/presto/router/cluster/ClusterStatusResource.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
import static java.util.Objects.requireNonNull;
3535
import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
3636

37-
@Path("/")
37+
@Path("/v1")
3838
public class ClusterStatusResource
3939
{
4040
private final String environment;
@@ -51,15 +51,15 @@ public ClusterStatusResource(NodeInfo nodeInfo, ClusterStatusTracker clusterStat
5151

5252
// The web UI depend on the following service endpoints.
5353
@GET
54-
@Path("/v1/info")
54+
@Path("info")
5555
@Produces(APPLICATION_JSON)
5656
public ServerInfo getInfo()
5757
{
5858
return new ServerInfo(UNKNOWN, environment, true, false, Optional.empty());
5959
}
6060

6161
@GET
62-
@Path("/v1/cluster")
62+
@Path("cluster")
6363
@Produces(APPLICATION_JSON)
6464
public ClusterStats getClusterStats()
6565
{
@@ -73,14 +73,14 @@ public ClusterStats getClusterStats()
7373
}
7474

7575
@GET
76-
@Path("/v1/query")
76+
@Path("query")
7777
public List<JsonNode> getAllQueryInfo(@QueryParam("state") String stateFilter)
7878
{
7979
return clusterStatusTracker.getAllQueryInfos();
8080
}
8181

8282
@GET
83-
@Path("/v1/all")
83+
@Path("all")
8484
@Produces(APPLICATION_JSON)
8585
public List<URI> getAllClusters()
8686
{

0 commit comments

Comments
 (0)