Skip to content

Commit 4c51d65

Browse files
zhicwuzhicwu
committed
Merge branch 'ch_develop' into new-sql-parser
2 parents b0eea78 + 41f3e34 commit 4c51d65

File tree

6 files changed

+132
-32
lines changed

6 files changed

+132
-32
lines changed

‎src/main/java/ru/yandex/clickhouse/response/ClickHouseResultSet.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -646,7 +646,7 @@ private int asColNum(String column) {
646646
}
647647
}
648648
// TODO Java8
649-
throw new RuntimeException("no column " + column + " in columns list " + getColumnNames());
649+
throw new RuntimeException("no column " + column + " in columns list " + getColumnNamesString());
650650
}
651651

652652
private ByteFragment getValue(int colNum) {
@@ -700,6 +700,14 @@ public BigDecimal getBigDecimal(int columnIndex, int scale) {
700700
return result.setScale(scale, RoundingMode.HALF_UP);
701701
}
702702

703+
public String[] getColumnNames() {
704+
String[] columnNames = new String[columns.size()];
705+
for (int i = 0; i < columns.size(); ++i) {
706+
columnNames[i] = columns.get(i).getColumnName();
707+
}
708+
return columnNames;
709+
}
710+
703711
@Override
704712
public void setFetchDirection(int direction) throws SQLException {
705713
// ignore perfomance hint
@@ -719,7 +727,7 @@ public String toString() {
719727
", bis=" + bis +
720728
", db='" + db + '\'' +
721729
", table='" + table + '\'' +
722-
", columns=" + getColumnNames() +
730+
", columns=" + getColumnNamesString() +
723731
", maxRows=" + maxRows +
724732
", values=" + Arrays.toString(values) +
725733
", lastReadColumn=" + lastReadColumn +
@@ -729,7 +737,7 @@ public String toString() {
729737
'}';
730738
}
731739

732-
private String getColumnNames() {
740+
private String getColumnNamesString() {
733741
StringBuilder sb = new StringBuilder();
734742
for (ClickHouseColumnInfo info : columns) {
735743
sb.append(info.getColumnName()).append(' ');

‎src/main/java/ru/yandex/clickhouse/settings/ClickHouseConnectionSettings.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public enum ClickHouseConnectionSettings implements DriverPropertyCreator {
3636
TIME_TO_LIVE_MILLIS("timeToLiveMillis", 60 * 1000, ""),
3737
DEFAULT_MAX_PER_ROUTE("defaultMaxPerRoute", 500, ""),
3838
MAX_TOTAL("maxTotal", 10000, ""),
39+
MAX_RETRIES("maxRetries", 3, "Maximum retries(default to 3) for idempotent operation. Set 0 to disable retry."),
3940

4041
/**
4142
* additional

‎src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ public class ClickHouseProperties {
2222
private int timeToLiveMillis;
2323
private int defaultMaxPerRoute;
2424
private int maxTotal;
25+
private int maxRetries;
2526
private String host;
2627
private int port;
2728
private boolean usePathAsDb;
@@ -115,6 +116,7 @@ public ClickHouseProperties(Properties info) {
115116
this.timeToLiveMillis = (Integer)getSetting(info, ClickHouseConnectionSettings.TIME_TO_LIVE_MILLIS);
116117
this.defaultMaxPerRoute = (Integer)getSetting(info, ClickHouseConnectionSettings.DEFAULT_MAX_PER_ROUTE);
117118
this.maxTotal = (Integer)getSetting(info, ClickHouseConnectionSettings.MAX_TOTAL);
119+
this.maxRetries = (Integer)getSetting(info, ClickHouseConnectionSettings.MAX_RETRIES);
118120
this.maxCompressBufferSize = (Integer) getSetting(info, ClickHouseConnectionSettings.MAX_COMPRESS_BUFFER_SIZE);
119121
this.ssl = (Boolean) getSetting(info, ClickHouseConnectionSettings.SSL);
120122
this.sslRootCertificate = (String) getSetting(info, ClickHouseConnectionSettings.SSL_ROOT_CERTIFICATE);
@@ -182,6 +184,7 @@ public Properties asProperties() {
182184
ret.put(ClickHouseConnectionSettings.TIME_TO_LIVE_MILLIS.getKey(), String.valueOf(timeToLiveMillis));
183185
ret.put(ClickHouseConnectionSettings.DEFAULT_MAX_PER_ROUTE.getKey(), String.valueOf(defaultMaxPerRoute));
184186
ret.put(ClickHouseConnectionSettings.MAX_TOTAL.getKey(), String.valueOf(maxTotal));
187+
ret.put(ClickHouseConnectionSettings.MAX_RETRIES.getKey(), String.valueOf(maxRetries));
185188
ret.put(ClickHouseConnectionSettings.MAX_COMPRESS_BUFFER_SIZE.getKey(), String.valueOf(maxCompressBufferSize));
186189
ret.put(ClickHouseConnectionSettings.SSL.getKey(), String.valueOf(ssl));
187190
ret.put(ClickHouseConnectionSettings.SSL_ROOT_CERTIFICATE.getKey(), String.valueOf(sslRootCertificate));
@@ -252,6 +255,7 @@ public ClickHouseProperties(ClickHouseProperties properties) {
252255
setTimeToLiveMillis(properties.timeToLiveMillis);
253256
setDefaultMaxPerRoute(properties.defaultMaxPerRoute);
254257
setMaxTotal(properties.maxTotal);
258+
setMaxRetries(properties.maxRetries);
255259
setMaxCompressBufferSize(properties.maxCompressBufferSize);
256260
setSsl(properties.ssl);
257261
setSslRootCertificate(properties.sslRootCertificate);
@@ -601,6 +605,14 @@ public void setMaxTotal(int maxTotal) {
601605
this.maxTotal = maxTotal;
602606
}
603607

608+
public int getMaxRetries() {
609+
return maxRetries;
610+
}
611+
612+
public void setMaxRetries(int maxRetries) {
613+
this.maxRetries = maxRetries;
614+
}
615+
604616
public int getMaxCompressBufferSize() {
605617
return maxCompressBufferSize;
606618
}

‎src/main/java/ru/yandex/clickhouse/util/ClickHouseHttpClientBuilder.java‎

Lines changed: 20 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -27,20 +27,19 @@
2727

2828
import org.apache.http.ConnectionReuseStrategy;
2929
import org.apache.http.Header;
30-
import org.apache.http.HeaderElement;
31-
import org.apache.http.HeaderElementIterator;
3230
import org.apache.http.HttpHeaders;
3331
import org.apache.http.HttpHost;
3432
import org.apache.http.HttpResponse;
33+
import org.apache.http.NoHttpResponseException;
3534
import org.apache.http.auth.AuthScope;
3635
import org.apache.http.auth.UsernamePasswordCredentials;
3736
import org.apache.http.client.AuthCache;
3837
import org.apache.http.client.CredentialsProvider;
38+
import org.apache.http.client.HttpRequestRetryHandler;
3939
import org.apache.http.client.config.RequestConfig;
4040
import org.apache.http.client.protocol.HttpClientContext;
4141
import org.apache.http.config.ConnectionConfig;
4242
import org.apache.http.config.RegistryBuilder;
43-
import org.apache.http.conn.ConnectionKeepAliveStrategy;
4443
import org.apache.http.conn.socket.ConnectionSocketFactory;
4544
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
4645
import org.apache.http.conn.ssl.NoopHostnameVerifier;
@@ -50,11 +49,10 @@
5049
import org.apache.http.impl.client.BasicAuthCache;
5150
import org.apache.http.impl.client.BasicCredentialsProvider;
5251
import org.apache.http.impl.client.CloseableHttpClient;
52+
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
5353
import org.apache.http.impl.client.HttpClientBuilder;
5454
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
5555
import org.apache.http.message.BasicHeader;
56-
import org.apache.http.message.BasicHeaderElementIterator;
57-
import org.apache.http.protocol.HTTP;
5856
import org.apache.http.protocol.HttpContext;
5957

6058
import ru.yandex.clickhouse.settings.ClickHouseProperties;
@@ -71,16 +69,32 @@ public ClickHouseHttpClientBuilder(ClickHouseProperties properties) {
7169
public CloseableHttpClient buildClient() throws Exception {
7270
return HttpClientBuilder.create()
7371
.setConnectionManager(getConnectionManager())
72+
.setRetryHandler(getRequestRetryHandler())
7473
.setConnectionReuseStrategy(getConnectionReuseStrategy())
7574
.setDefaultConnectionConfig(getConnectionConfig())
7675
.setDefaultRequestConfig(getRequestConfig())
7776
.setDefaultHeaders(getDefaultHeaders())
7877
.setDefaultCredentialsProvider(getDefaultCredentialsProvider())
79-
.disableContentCompression() // gzip здесь ни к чему. Используется lz4 при compress=1
78+
.disableContentCompression() // gzip is not needed. Use lz4 when compress=1
8079
.disableRedirectHandling()
8180
.build();
8281
}
8382

83+
private HttpRequestRetryHandler getRequestRetryHandler() {
84+
final int maxRetries = properties.getMaxRetries();
85+
return new DefaultHttpRequestRetryHandler(maxRetries, false) {
86+
@Override
87+
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
88+
if (executionCount > maxRetries || context == null
89+
|| !Boolean.TRUE.equals(context.getAttribute("is_idempotent"))) {
90+
return false;
91+
}
92+
93+
return (exception instanceof NoHttpResponseException) || super.retryRequest(exception, executionCount, context);
94+
}
95+
};
96+
}
97+
8498
public static HttpClientContext createClientContext(ClickHouseProperties props) {
8599
if (props == null
86100
|| !isConfigurationValidForAuth(props))
@@ -155,29 +169,6 @@ private Collection<Header> getDefaultHeaders() {
155169
return headers;
156170
}
157171

158-
private ConnectionKeepAliveStrategy createKeepAliveStrategy() {
159-
return new ConnectionKeepAliveStrategy() {
160-
@Override
161-
public long getKeepAliveDuration(HttpResponse httpResponse, HttpContext httpContext) {
162-
// in case of errors keep-alive not always works. close connection just in case
163-
if (httpResponse.getStatusLine().getStatusCode() != HttpURLConnection.HTTP_OK) {
164-
return -1;
165-
}
166-
HeaderElementIterator it = new BasicHeaderElementIterator(
167-
httpResponse.headerIterator(HTTP.CONN_DIRECTIVE));
168-
while (it.hasNext()) {
169-
HeaderElement he = it.nextElement();
170-
String param = he.getName();
171-
//String value = he.getValue();
172-
if (param != null && param.equalsIgnoreCase(HTTP.CONN_KEEP_ALIVE)) {
173-
return properties.getKeepAliveTimeout();
174-
}
175-
}
176-
return -1;
177-
}
178-
};
179-
}
180-
181172
private SSLContext getSSLContext()
182173
throws CertificateException, NoSuchAlgorithmException, KeyStoreException, IOException, KeyManagementException {
183174
SSLContext ctx = SSLContext.getInstance("TLS");

‎src/test/java/ru/yandex/clickhouse/response/ClickHouseResultSetTest.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -448,6 +448,25 @@ public void testClassNamesObjects() throws Exception {
448448
}
449449
}
450450

451+
@Test
452+
public void testGetColumnNames() throws Exception {
453+
String response = "SiteName\tCountry\n" +
454+
"String\tString\n" +
455+
"hello.com\tPoland\n" +
456+
"there.com\tUSA\n" +
457+
"\t\n" +
458+
"other.com\t\n" +
459+
"\n" +
460+
"\t\n";
461+
462+
ByteArrayInputStream is = new ByteArrayInputStream(response.getBytes("UTF-8"));
463+
464+
ClickHouseResultSet rs = buildResultSet(is, 1024, "db", "table", false, null, null, props);
465+
String[] columnNames = rs.getColumnNames();
466+
assertEquals(2, columnNames.length);
467+
assertEquals("SiteName", columnNames[0]);
468+
assertEquals("Country", columnNames[1]);
469+
}
451470

452471
/**
453472
* By jdbc specification

‎src/test/java/ru/yandex/clickhouse/util/ClickHouseHttpClientBuilderTest.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
package ru.yandex.clickhouse.util;
22

33
import org.apache.http.HttpHost;
4+
import org.apache.http.NoHttpResponseException;
45
import org.apache.http.client.methods.HttpPost;
6+
import org.apache.http.conn.HttpHostConnectException;
57
import org.apache.http.impl.client.CloseableHttpClient;
8+
import org.apache.http.protocol.BasicHttpContext;
9+
import org.apache.http.protocol.HttpContext;
610
import org.testng.annotations.AfterClass;
711
import org.testng.annotations.AfterMethod;
812
import org.testng.annotations.BeforeClass;
@@ -141,7 +145,72 @@ private static Object[][] provideAuthUserPasswordTestData() {
141145
null, null, "baz", "Basic ZGVmYXVsdDpiYXo=" // default:baz
142146
},
143147
};
148+
}
149+
150+
private static WireMockServer newServer() {
151+
WireMockServer server = new WireMockServer(
152+
WireMockConfiguration.wireMockConfig().dynamicPort());
153+
server.start();
154+
server.stubFor(WireMock.post(WireMock.urlPathMatching("/*"))
155+
.willReturn(WireMock.aResponse().withStatus(200).withHeader("Connection", "Keep-Alive")
156+
.withHeader("Content-Type", "text/plain; charset=UTF-8")
157+
.withHeader("Transfer-Encoding", "chunked").withHeader("Keep-Alive", "timeout=3")
158+
.withBody("OK.........................").withFixedDelay(2)));
159+
return server;
160+
}
144161

162+
private static void shutDownServerWithDelay(final WireMockServer server, final long delayMs) {
163+
new Thread() {
164+
public void run() {
165+
try {
166+
Thread.sleep(delayMs);
167+
} catch (InterruptedException e) {
168+
e.printStackTrace();
169+
}
170+
171+
server.shutdownServer();
172+
server.stop();
173+
}
174+
}.start();
145175
}
146176

177+
// @Test(dependsOnMethods = { "testWithRetry" }, expectedExceptions = { NoHttpResponseException.class })
178+
public void testWithoutRetry() throws Exception {
179+
final WireMockServer server = newServer();
180+
181+
ClickHouseProperties props = new ClickHouseProperties();
182+
props.setMaxRetries(0);
183+
ClickHouseHttpClientBuilder builder = new ClickHouseHttpClientBuilder(props);
184+
CloseableHttpClient client = builder.buildClient();
185+
HttpPost post = new HttpPost("http://localhost:" + server.port() + "/?db=system&query=select%201");
186+
187+
shutDownServerWithDelay(server, 100);
188+
189+
try {
190+
client.execute(post);
191+
} finally {
192+
client.close();
193+
}
194+
}
195+
196+
// @Test(expectedExceptions = { HttpHostConnectException.class })
197+
public void testWithRetry() throws Exception {
198+
final WireMockServer server = newServer();
199+
200+
ClickHouseProperties props = new ClickHouseProperties();
201+
// props.setMaxRetries(3);
202+
ClickHouseHttpClientBuilder builder = new ClickHouseHttpClientBuilder(props);
203+
CloseableHttpClient client = builder.buildClient();
204+
HttpContext context = new BasicHttpContext();
205+
context.setAttribute("is_idempotent", Boolean.TRUE);
206+
HttpPost post = new HttpPost("http://localhost:" + server.port() + "/?db=system&query=select%202");
207+
208+
shutDownServerWithDelay(server, 100);
209+
210+
try {
211+
client.execute(post, context);
212+
} finally {
213+
client.close();
214+
}
215+
}
147216
}

0 commit comments

Comments
 (0)