Skip to content

Commit bf61fc2

Browse files
jmuscireumjmuscireum
authored
Merge pull request #677 from scireum/GA-13
GA 13.0.0
2 parents fde66e9 + a2ef7eb commit bf61fc2

File tree

68 files changed

+1242
-577
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+1242
-577
lines changed

‎.drone.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ clone:
1717

1818
steps:
1919
- name: compile
20-
image: scireum/sirius-build-jdk21
20+
image: scireum/sirius-build-jdk23
2121
commands:
2222
- mvn clean compile
2323
volumes: *scireum_volumes
@@ -28,7 +28,7 @@ steps:
2828
- push
2929

3030
- name: cron_unit_tests
31-
image: scireum/sirius-build-jdk21
31+
image: scireum/sirius-build-jdk23
3232
commands:
3333
- mvn clean test
3434
volumes: *scireum_volumes
@@ -48,7 +48,7 @@ steps:
4848
- cron
4949

5050
- name: test
51-
image: scireum/sirius-build-jdk21
51+
image: scireum/sirius-build-jdk23
5252
commands:
5353
- mvn clean test -Dtest.excluded.groups=nightly
5454
volumes: *scireum_volumes
@@ -57,7 +57,7 @@ steps:
5757
- pull_request
5858

5959
- name: deploy
60-
image: scireum/sirius-build-jdk21
60+
image: scireum/sirius-build-jdk23
6161
commands:
6262
- sed -i 's/DEVELOPMENT-SNAPSHOT/${DRONE_TAG}/g' pom.xml
6363
- mvn clean deploy -DskipTests
@@ -77,7 +77,7 @@ steps:
7777
- tag
7878

7979
- name: sonar
80-
image: scireum/sirius-build-jdk21
80+
image: scireum/sirius-build-jdk23
8181
commands:
8282
- sed -i 's/DEVELOPMENT-SNAPSHOT/${DRONE_TAG}/g' pom.xml
8383
- mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent test org.jacoco:jacoco-maven-plugin:report sonar:sonar -Dsonar.projectKey=${DRONE_REPO_NAME}

‎.github/PULL_REQUEST_TEMPLATE.md

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
### Description
2+
3+
<!-- Describe your changes in detail. Also mention how to handle breaking changes if there are any -->
4+
5+
### Additional Notes
6+
7+
- This PR fixes or works on following ticket(s): [SIRI-](https://scireum.myjetbrains.com/youtrack/issue/SIRI-)
8+
- This PR is related to PR: <!-- URL of PR if applicable, remove otherwise -->
9+
10+
### Checklist
11+
12+
- [ ] Code change has been tested and works locally
13+
- [ ] Code was formatted via IntelliJ and follows SonarLint & [best practices](https://scireum.myjetbrains.com/youtrack/articles/MISC-A-16/CodeStyle-JavaDoc)
14+
- [ ] Patch Tasks: Is local execution of Patch Tasks necessary? If so, please also mark the PR with the tag.

‎pom.xml

+8-12
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<parent>
77
<groupId>com.scireum</groupId>
88
<artifactId>sirius-parent</artifactId>
9-
<version>11.5.1</version>
9+
<version>13.1.0</version>
1010
</parent>
1111
<artifactId>sirius-db</artifactId>
1212
<version>DEVELOPMENT-SNAPSHOT</version>
@@ -19,7 +19,7 @@
1919
<url>http://www.sirius-lib.net</url>
2020

2121
<properties>
22-
<sirius.kernel>ga-11.0.0</sirius.kernel>
22+
<sirius.kernel>ga-12.0.0</sirius.kernel>
2323
</properties>
2424

2525
<repositories>
@@ -45,7 +45,7 @@
4545
<dependency>
4646
<groupId>org.apache.commons</groupId>
4747
<artifactId>commons-dbcp2</artifactId>
48-
<version>2.11.0</version>
48+
<version>2.12.0</version>
4949
</dependency>
5050
<dependency>
5151
<groupId>org.mongodb</groupId>
@@ -55,7 +55,7 @@
5555
<dependency>
5656
<groupId>redis.clients</groupId>
5757
<artifactId>jedis</artifactId>
58-
<version>4.2.3</version>
58+
<version>5.1.5</version>
5959
<exclusions>
6060
<exclusion>
6161
<groupId>org.json</groupId>
@@ -67,25 +67,21 @@
6767
<dependency>
6868
<groupId>org.elasticsearch.client</groupId>
6969
<artifactId>elasticsearch-rest-client</artifactId>
70-
<version>8.11.2</version>
70+
<version>8.15.3</version>
7171
</dependency>
7272
<dependency>
7373
<groupId>org.mariadb.jdbc</groupId>
7474
<artifactId>mariadb-java-client</artifactId>
75-
<version>3.3.1</version>
75+
<version>3.4.1</version>
7676
<scope>test</scope>
7777
</dependency>
7878
<dependency>
7979
<groupId>com.clickhouse</groupId>
8080
<artifactId>clickhouse-jdbc</artifactId>
81-
<version>0.3.2-patch11</version>
81+
<version>0.6.5</version>
82+
<classifier>http</classifier>
8283
<scope>test</scope>
8384
</dependency>
84-
<!-- Required as the version brought by clickhouse-jdbc contains security issues -->
85-
<dependency>
86-
<groupId>com.fasterxml.jackson.core</groupId>
87-
<artifactId>jackson-databind</artifactId>
88-
</dependency>
8985
</dependencies>
9086

9187
</project>

‎src/main/java/sirius/db/es/BulkContext.java

+34-3
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
public class BulkContext implements Closeable {
4040

4141
private static final int MAX_BATCH_SIZE = 1024;
42+
private static final int MAX_REQUEST_SIZE = 100_000_000;
4243
private static final int RECOMMENDED_BATCH_SIZE = 256;
4344

4445
private static final String KEY_INDEX = "_index";
@@ -128,17 +129,45 @@ private void update(ElasticEntity entity, boolean force) {
128129
return;
129130
}
130131

131-
commands.add(Json.createObject().set(COMMAND_INDEX, meta));
132-
commands.add(data);
132+
ObjectNode metaCommand = Json.createObject().set(COMMAND_INDEX, meta);
133+
commitIfNeededAndAdd(metaCommand, data);
133134
autocommit();
134135
}
135136

137+
/**
138+
* Commits all commands if the batch size exceeds {@link #MAX_BATCH_SIZE}.
139+
*/
136140
private void autocommit() {
137141
if (commands.size() >= MAX_BATCH_SIZE) {
138142
commit().throwFailures();
139143
}
140144
}
141145

146+
/**
147+
* Commits queued commands if needed and adds the given nodes to the queue.
148+
* <p>
149+
* If the new nodes would exceed {@link #MAX_REQUEST_SIZE}, we commit them before adding new commands to the list. This is important
150+
* for bulk requests since Elasticsearch establishes a default limit of 100MB per request.
151+
*
152+
* @param nodes the nodes which size should be checked
153+
*/
154+
private void commitIfNeededAndAdd(ObjectNode... nodes) {
155+
int currentSize = commands.stream().map(this::calculateNodeLength).reduce(0, Integer::sum);
156+
for (ObjectNode node : nodes) {
157+
currentSize += calculateNodeLength(node);
158+
}
159+
if (currentSize >= MAX_REQUEST_SIZE) {
160+
// The given commands will exceed the maximum request size, so we commit the current queued commands
161+
// before adding new ones
162+
commit().throwFailures();
163+
}
164+
commands.addAll(List.of(nodes));
165+
}
166+
167+
private int calculateNodeLength(ObjectNode node) {
168+
return node.toString().length();
169+
}
170+
142171
private ObjectNode builtMetadata(ElasticEntity entity, boolean force, EntityDescriptor ed) {
143172
ObjectNode meta = Json.createObject();
144173

@@ -167,7 +196,9 @@ private void delete(ElasticEntity entity, boolean force) {
167196
entityDescriptor.beforeDelete(entity);
168197

169198
ObjectNode meta = builtMetadata(entity, force, entityDescriptor);
170-
commands.add(Json.createObject().set(COMMAND_DELETE, meta));
199+
200+
ObjectNode metaCommand = Json.createObject().set(COMMAND_DELETE, meta);
201+
commitIfNeededAndAdd(metaCommand);
171202
autocommit();
172203
}
173204

‎src/main/java/sirius/db/es/Elastic.java

+1-7
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import org.apache.http.HttpHost;
1313
import org.elasticsearch.client.Request;
1414
import org.elasticsearch.client.RestClient;
15-
import org.elasticsearch.client.RestClientBuilder;
1615
import sirius.db.KeyGenerator;
1716
import sirius.db.es.constraints.ElasticConstraint;
1817
import sirius.db.es.constraints.ElasticFilterFactory;
@@ -169,19 +168,14 @@ private synchronized void initializeClient() {
169168
if (client == null) {
170169
Elastic.LOG.INFO("Initializing Elasticsearch client against: %s", hosts);
171170

172-
// Fixes an Elastic bug that results in TimeoutExceptions
173-
// Remove this, once ES is updated to at least 6.3.1
174-
RestClientBuilder.RequestConfigCallback configCallback =
175-
requestConfigBuilder -> requestConfigBuilder.setConnectionRequestTimeout(0);
176-
177171
HttpHost[] httpHosts = Arrays.stream(this.hosts.split(","))
178172
.map(String::trim)
179173
.map(host -> Strings.splitAtLast(host, ":"))
180174
.map(this::parsePort)
181175
.map(this::mapPort)
182176
.map(this::makeHttpHost)
183177
.toArray(size -> new HttpHost[size]);
184-
client = new LowLevelClient(RestClient.builder(httpHosts).setRequestConfigCallback(configCallback).build());
178+
client = new LowLevelClient(RestClient.builder(httpHosts).build());
185179

186180
// If we're using a docker container (most probably for testing), we give ES some time
187181
// to fully boot up. Otherwise, strange connection issues might arise.

‎src/main/java/sirius/db/es/ElasticMetricsProvider.java

+46-1
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,25 @@
88

99
package sirius.db.es;
1010

11+
import com.fasterxml.jackson.core.JsonPointer;
12+
import com.fasterxml.jackson.databind.JsonNode;
1113
import com.fasterxml.jackson.databind.node.ObjectNode;
1214
import sirius.kernel.di.std.Part;
1315
import sirius.kernel.di.std.Register;
16+
import sirius.kernel.health.metrics.Metric;
1417
import sirius.kernel.health.metrics.MetricProvider;
1518
import sirius.kernel.health.metrics.MetricsCollector;
1619

20+
import java.util.Map;
21+
1722
/**
1823
* Provides metrics for Elasticsearch (if configured).
1924
*/
20-
@Register
25+
@Register(classes = {ElasticMetricsProvider.class, MetricProvider.class})
2126
public class ElasticMetricsProvider implements MetricProvider {
2227

28+
private static final JsonPointer OLD_GEN_STATS_POINTER = JsonPointer.compile("/jvm/mem/pools/old");
29+
2330
@Part
2431
private Elastic elastic;
2532

@@ -43,6 +50,44 @@ public void gather(MetricsCollector collector) {
4350
"ES Unassigned Shards",
4451
health.path("unassigned_shards").asInt(),
4552
null);
53+
collector.metric("es_memory_pressure",
54+
"es-memory-pressure",
55+
"Elasticsearch Memory Pressure",
56+
getCurrentMaxMemoryPressure(),
57+
Metric.UNIT_PERCENT);
4658
}
4759
}
60+
61+
/**
62+
* Determines the current maximum memory pressure of all ES nodes.
63+
* <p>
64+
* The memory pressure is defined as the usage in percent of the old gen memory pool.
65+
*
66+
* @return the current maximum memory pressure
67+
* @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/high-jvm-memory-pressure.html">
68+
* ElasticSearch reference page for JVM memory pressure</a>
69+
*/
70+
public int getCurrentMaxMemoryPressure() {
71+
return elastic.getLowLevelClient()
72+
.memoryStats()
73+
.path("nodes")
74+
.properties()
75+
.stream()
76+
.map(Map.Entry::getValue)
77+
.mapToInt(this::calculateMemoryPressure)
78+
.max()
79+
.orElse(0);
80+
}
81+
82+
private int calculateMemoryPressure(JsonNode memoryStats) {
83+
JsonNode oldGenStats = memoryStats.at(OLD_GEN_STATS_POINTER);
84+
long usedInBytes = oldGenStats.path("used_in_bytes").asLong();
85+
long maxInBytes = oldGenStats.path("max_in_bytes").asLong();
86+
87+
if (maxInBytes == 0) {
88+
return 0;
89+
}
90+
91+
return (int) (100f * usedInBytes / maxInBytes);
92+
}
4893
}

‎src/main/java/sirius/db/es/ElasticQuery.java

+16-7
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,15 @@
5050
* @param <E> the type of entities to be queried
5151
*/
5252
public class ElasticQuery<E extends ElasticEntity> extends Query<ElasticQuery<E>, E, ElasticConstraint> {
53+
54+
/**
55+
* Defines the fallback value used when no last sort value is present.
56+
*
57+
* @see #searchAfter(String)
58+
* @see #getLastSortValue()
59+
*/
60+
public static final String NO_LAST_SORT_VALUE = "-";
61+
5362
/**
5463
* If we only fetch from a single shard (as we use a routing), we fetch up to {@link #BLOCK_SIZE_FOR_SINGLE_SHARD}
5564
* entities at once and hope to process them within {@link #STREAM_BLOCKWISE_PIT_TTL}.
@@ -448,15 +457,15 @@ public ElasticQuery<E> searchAfter(List<String> searchAfter) {
448457
* Sets a singular "last sort value" to continue a previous query.
449458
* <p>
450459
* It is recommended to sort by {@link ElasticEntity#ID} when using this. Note that this gracefully
451-
* handles empty values as well as "-" which is properly generated by {@link #getLastSortValue()} if no more
452-
* results are expected.
460+
* handles empty values as well as {@link #NO_LAST_SORT_VALUE} which is properly generated by {@link #getLastSortValue()}
461+
* if no more results are expected.
453462
*
454463
* @param searchAfter the last sort value of the previous result
455464
* @return the query itself for fluent method calls
456465
*/
457466
public ElasticQuery<E> searchAfter(String searchAfter) {
458467
if (Strings.isFilled(searchAfter)) {
459-
if ("-".equals(searchAfter)) {
468+
if (NO_LAST_SORT_VALUE.equals(searchAfter)) {
460469
return this.fail();
461470
}
462471

@@ -494,17 +503,17 @@ public List<String> getLastSortValues() {
494503
/**
495504
* Obtains the last sort value of this result.
496505
* <p>
497-
* Note that this will automatically return "-" if no more results are expected. Use this in conjunction with
498-
* {@link #searchAfter(String)} and sort queries by {@link ElasticEntity#ID}.
506+
* Note that this will automatically return {@link #NO_LAST_SORT_VALUE} if no more results are expected.
507+
* Use this in conjunction with {@link #searchAfter(String)} and sort queries by {@link ElasticEntity#ID}.
499508
*
500509
* @return the last sort value within this result
501510
*/
502511
public String getLastSortValue() {
503512
if (limit > 0 && Json.getArrayAt(getRawResponse(), HITS_POINTER).size() < limit) {
504-
return "-";
513+
return NO_LAST_SORT_VALUE;
505514
}
506515

507-
return getLastSortValues().stream().filter(Strings::isFilled).findFirst().orElse("-");
516+
return getLastSortValues().stream().filter(Strings::isFilled).findFirst().orElse(NO_LAST_SORT_VALUE);
508517
}
509518

510519
/**

‎src/main/java/sirius/db/es/LowLevelClient.java

+10
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ public class LowLevelClient {
5050
private static final String API_REFRESH = "/_refresh";
5151
private static final String API_SETTINGS = "/_settings";
5252
private static final String API_CLUSTER_HEALTH = "/_cluster/health";
53+
private static final String API_JVM_MEMORY_STATS = "/_nodes/stats?pretty&filter_path=nodes.*.jvm.mem";
5354
private static final String API_STATS = "/_stats";
5455
private static final String API_MAPPING = "/_mapping";
5556
private static final String API_BULK = "_bulk";
@@ -579,6 +580,15 @@ public ObjectNode clusterHealth() {
579580
return performGet().execute(API_CLUSTER_HEALTH).response();
580581
}
581582

583+
/**
584+
* Fetches the JVM memory statistics.
585+
*
586+
* @return a JSON object as returned by <tt>/_nodes/stats?pretty&filter_path=nodes.*.jvm.mem</tt>
587+
*/
588+
public ObjectNode memoryStats() {
589+
return performGet().execute(API_JVM_MEMORY_STATS).response();
590+
}
591+
582592
/**
583593
* Fetches statistics for all indices.
584594
*

‎src/main/java/sirius/db/es/RequestBuilder.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,11 @@ private HandledException handleAsyncFailure(Exception exception, String uri) {
246246

247247
protected ObjectNode extractErrorJSON(ResponseException e) {
248248
try {
249-
ObjectNode response = Json.parseObject(EntityUtils.toString(e.getResponse().getEntity()));
249+
HttpEntity httpEntity = e.getResponse().getEntity();
250+
if (e.getResponse().getEntity().getContentLength() == 0) {
251+
return null;
252+
}
253+
ObjectNode response = Json.parseObject(EntityUtils.toString(httpEntity));
250254
return Json.getObject(response, PARAM_ERROR);
251255
} catch (IOException ex) {
252256
Exceptions.handle(Elastic.LOG, ex);

0 commit comments

Comments
 (0)