From 6099ed99fe68a739e60bf0c13c9954ec5c890fac Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Wed, 14 Feb 2024 15:30:33 -0500 Subject: [PATCH 01/41] Update BouncyCastle dependencies from jdk15to18 to jdk18on (#12317) Signed-off-by: Stephen Crawford --- CHANGELOG.md | 3 +++ buildSrc/version.properties | 2 +- plugins/identity-shiro/build.gradle | 2 +- .../identity-shiro/licenses/bcprov-jdk15to18-1.76.jar.sha1 | 1 - .../identity-shiro/licenses/bcprov-jdk18on-1.77.jar.sha1 | 1 + ...rov-jdk15to18-LICENSE.txt => bcprov-jdk18on-LICENSE.txt} | 0 ...cprov-jdk15to18-NOTICE.txt => bcprov-jdk18on-NOTICE.txt} | 0 plugins/ingest-attachment/build.gradle | 6 +++--- .../licenses/bcmail-jdk15to18-1.76.jar.sha1 | 1 - .../ingest-attachment/licenses/bcmail-jdk18on-1.77.jar.sha1 | 1 + ...ail-jdk15to18-LICENSE.txt => bcmail-jdk18on-LICENSE.txt} | 0 ...cmail-jdk15to18-NOTICE.txt => bcmail-jdk18on-NOTICE.txt} | 0 .../licenses/bcpkix-jdk15to18-1.76.jar.sha1 | 1 - .../ingest-attachment/licenses/bcpkix-jdk18on-1.77.jar.sha1 | 1 + ...kix-jdk15to18-LICENSE.txt => bcpkix-jdk18on-LICENSE.txt} | 0 ...cpkix-jdk15to18-NOTICE.txt => bcpkix-jdk18on-NOTICE.txt} | 0 .../licenses/bcprov-jdk15to18-1.76.jar.sha1 | 1 - .../ingest-attachment/licenses/bcprov-jdk18on-1.77.jar.sha1 | 1 + ...rov-jdk15to18-LICENSE.txt => bcprov-jdk18on-LICENSE.txt} | 0 ...cprov-jdk15to18-NOTICE.txt => bcprov-jdk18on-NOTICE.txt} | 0 20 files changed, 12 insertions(+), 9 deletions(-) delete mode 100644 plugins/identity-shiro/licenses/bcprov-jdk15to18-1.76.jar.sha1 create mode 100644 plugins/identity-shiro/licenses/bcprov-jdk18on-1.77.jar.sha1 rename plugins/identity-shiro/licenses/{bcprov-jdk15to18-LICENSE.txt => bcprov-jdk18on-LICENSE.txt} (100%) rename plugins/identity-shiro/licenses/{bcprov-jdk15to18-NOTICE.txt => bcprov-jdk18on-NOTICE.txt} (100%) delete mode 100644 plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.76.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/bcmail-jdk18on-1.77.jar.sha1 rename plugins/ingest-attachment/licenses/{bcmail-jdk15to18-LICENSE.txt => bcmail-jdk18on-LICENSE.txt} (100%) rename plugins/ingest-attachment/licenses/{bcmail-jdk15to18-NOTICE.txt => bcmail-jdk18on-NOTICE.txt} (100%) delete mode 100644 plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.76.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/bcpkix-jdk18on-1.77.jar.sha1 rename plugins/ingest-attachment/licenses/{bcpkix-jdk15to18-LICENSE.txt => bcpkix-jdk18on-LICENSE.txt} (100%) rename plugins/ingest-attachment/licenses/{bcpkix-jdk15to18-NOTICE.txt => bcpkix-jdk18on-NOTICE.txt} (100%) delete mode 100644 plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.76.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/bcprov-jdk18on-1.77.jar.sha1 rename plugins/ingest-attachment/licenses/{bcprov-jdk15to18-LICENSE.txt => bcprov-jdk18on-LICENSE.txt} (100%) rename plugins/ingest-attachment/licenses/{bcprov-jdk15to18-NOTICE.txt => bcprov-jdk18on-NOTICE.txt} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5b3742af80ea..de8a041da3b73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.eclipse.jgit` from 6.5.0 to 6.7.0 ([#10147](https://github.com/opensearch-project/OpenSearch/pull/10147)) - Bump OpenTelemetry from 1.30.1 to 1.31.0 ([#10617](https://github.com/opensearch-project/OpenSearch/pull/10617)) - Bump OpenTelemetry from 1.31.0 to 1.32.0 and OpenTelemetry Semconv from 1.21.0-alpha to 1.23.1-alpha ([#11305](https://github.com/opensearch-project/OpenSearch/pull/11305)) +- Bump `org.bouncycastle:bcprov-jdk15to18` to `org.bouncycastle:bcprov-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317)) +- Bump `org.bouncycastle:bcmail-jdk15to18` to `org.bouncycastle:bcmail-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317)) +- Bump `org.bouncycastle:bcpkix-jdk15to18` to `org.bouncycastle:bcpkix-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317)) ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 95ae1ddb578a1..c7af58ab6b0b9 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -50,7 +50,7 @@ reactivestreams = 1.0.4 # when updating this version, you need to ensure compatibility with: # - plugins/ingest-attachment (transitive dependency, check the upstream POM) # - distribution/tools/plugin-cli -bouncycastle=1.76 +bouncycastle=1.77 # test dependencies randomizedrunner = 2.7.1 junit = 4.13.2 diff --git a/plugins/identity-shiro/build.gradle b/plugins/identity-shiro/build.gradle index 1548780aaa566..222443efcb214 100644 --- a/plugins/identity-shiro/build.gradle +++ b/plugins/identity-shiro/build.gradle @@ -28,7 +28,7 @@ dependencies { implementation 'org.passay:passay:1.6.3' - implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}" + implementation "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}" testImplementation project(path: ':modules:transport-netty4') // for http testImplementation project(path: ':plugins:transport-nio') // for http diff --git a/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.76.jar.sha1 b/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.76.jar.sha1 deleted file mode 100644 index 2e96c404bef98..0000000000000 --- a/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.76.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0cb53f10290a634808555bc4b34328fdab1001f2 \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/bcprov-jdk18on-1.77.jar.sha1 b/plugins/identity-shiro/licenses/bcprov-jdk18on-1.77.jar.sha1 new file mode 100644 index 0000000000000..3e780df9559a9 --- /dev/null +++ b/plugins/identity-shiro/licenses/bcprov-jdk18on-1.77.jar.sha1 @@ -0,0 +1 @@ +2cc971b6c20949c1ff98d1a4bc741ee848a09523 \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/bcprov-jdk15to18-LICENSE.txt b/plugins/identity-shiro/licenses/bcprov-jdk18on-LICENSE.txt similarity index 100% rename from plugins/identity-shiro/licenses/bcprov-jdk15to18-LICENSE.txt rename to plugins/identity-shiro/licenses/bcprov-jdk18on-LICENSE.txt diff --git a/plugins/identity-shiro/licenses/bcprov-jdk15to18-NOTICE.txt b/plugins/identity-shiro/licenses/bcprov-jdk18on-NOTICE.txt similarity index 100% rename from plugins/identity-shiro/licenses/bcprov-jdk15to18-NOTICE.txt rename to plugins/identity-shiro/licenses/bcprov-jdk18on-NOTICE.txt diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index 22db73ad86796..6da34c4c9caf2 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -71,9 +71,9 @@ dependencies { api "org.apache.pdfbox:fontbox:${versions.pdfbox}" api "org.apache.pdfbox:jempbox:1.8.17" api "commons-logging:commons-logging:${versions.commonslogging}" - api "org.bouncycastle:bcmail-jdk15to18:${versions.bouncycastle}" - api "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}" - api "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}" + api "org.bouncycastle:bcmail-jdk18on:${versions.bouncycastle}" + api "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}" + api "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}" // OpenOffice api "org.apache.poi:poi-ooxml:${versions.poi}" api "org.apache.poi:poi:${versions.poi}" diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.76.jar.sha1 b/plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.76.jar.sha1 deleted file mode 100644 index 46010d64015ad..0000000000000 --- a/plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.76.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -23d8bcad6b57912e4633ca9955926ffcdf3c5c71 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk18on-1.77.jar.sha1 b/plugins/ingest-attachment/licenses/bcmail-jdk18on-1.77.jar.sha1 new file mode 100644 index 0000000000000..f71659316b8cd --- /dev/null +++ b/plugins/ingest-attachment/licenses/bcmail-jdk18on-1.77.jar.sha1 @@ -0,0 +1 @@ +f2bb8aa55dc901ee8b8aae7d1007c03592d65e03 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk15to18-LICENSE.txt b/plugins/ingest-attachment/licenses/bcmail-jdk18on-LICENSE.txt similarity index 100% rename from plugins/ingest-attachment/licenses/bcmail-jdk15to18-LICENSE.txt rename to plugins/ingest-attachment/licenses/bcmail-jdk18on-LICENSE.txt diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk15to18-NOTICE.txt b/plugins/ingest-attachment/licenses/bcmail-jdk18on-NOTICE.txt similarity index 100% rename from plugins/ingest-attachment/licenses/bcmail-jdk15to18-NOTICE.txt rename to plugins/ingest-attachment/licenses/bcmail-jdk18on-NOTICE.txt diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.76.jar.sha1 b/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.76.jar.sha1 deleted file mode 100644 index a843d972ac681..0000000000000 --- a/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.76.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3ee440dfa1c557c1cc0c46b5dadf5ef3896ccebb \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk18on-1.77.jar.sha1 b/plugins/ingest-attachment/licenses/bcpkix-jdk18on-1.77.jar.sha1 new file mode 100644 index 0000000000000..05a8b2d5729bd --- /dev/null +++ b/plugins/ingest-attachment/licenses/bcpkix-jdk18on-1.77.jar.sha1 @@ -0,0 +1 @@ +ed953791ba0229747dd0fd9911e3d76a462acfd3 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-LICENSE.txt b/plugins/ingest-attachment/licenses/bcpkix-jdk18on-LICENSE.txt similarity index 100% rename from plugins/ingest-attachment/licenses/bcpkix-jdk15to18-LICENSE.txt rename to plugins/ingest-attachment/licenses/bcpkix-jdk18on-LICENSE.txt diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-NOTICE.txt b/plugins/ingest-attachment/licenses/bcpkix-jdk18on-NOTICE.txt similarity index 100% rename from plugins/ingest-attachment/licenses/bcpkix-jdk15to18-NOTICE.txt rename to plugins/ingest-attachment/licenses/bcpkix-jdk18on-NOTICE.txt diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.76.jar.sha1 b/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.76.jar.sha1 deleted file mode 100644 index 2e96c404bef98..0000000000000 --- a/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.76.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0cb53f10290a634808555bc4b34328fdab1001f2 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk18on-1.77.jar.sha1 b/plugins/ingest-attachment/licenses/bcprov-jdk18on-1.77.jar.sha1 new file mode 100644 index 0000000000000..3e780df9559a9 --- /dev/null +++ b/plugins/ingest-attachment/licenses/bcprov-jdk18on-1.77.jar.sha1 @@ -0,0 +1 @@ +2cc971b6c20949c1ff98d1a4bc741ee848a09523 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15to18-LICENSE.txt b/plugins/ingest-attachment/licenses/bcprov-jdk18on-LICENSE.txt similarity index 100% rename from plugins/ingest-attachment/licenses/bcprov-jdk15to18-LICENSE.txt rename to plugins/ingest-attachment/licenses/bcprov-jdk18on-LICENSE.txt diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15to18-NOTICE.txt b/plugins/ingest-attachment/licenses/bcprov-jdk18on-NOTICE.txt similarity index 100% rename from plugins/ingest-attachment/licenses/bcprov-jdk15to18-NOTICE.txt rename to plugins/ingest-attachment/licenses/bcprov-jdk18on-NOTICE.txt From ad7f00f1d673a055ce7293531423d69ab9f6b0e8 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Wed, 14 Feb 2024 12:34:34 -0800 Subject: [PATCH 02/41] Add 'Related component' dropdown to meta template (#12321) Similar to feature requests and bugs, this dropdown will ensure that every meta issue gets assigned to a component. Signed-off-by: Andrew Ross --- .github/ISSUE_TEMPLATE/meta.yml | 36 ++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/meta.yml b/.github/ISSUE_TEMPLATE/meta.yml index 0ef42688474c3..b766a26bc3ff2 100644 --- a/.github/ISSUE_TEMPLATE/meta.yml +++ b/.github/ISSUE_TEMPLATE/meta.yml @@ -23,6 +23,36 @@ body: placeholder: Ex. - [ ] https://github.com/opensearch-project/security/issues/3888 Add views to the cluster metadata schema validations: required: true - - - + - type: dropdown + attributes: + label: Related component + description: Choose a specific OpenSearch component your project belongs to. If you are unsure of which component to select or if the component is not present, select "Other". + multiple: false + options: + - # Empty first option to force selection + - Build + - Clients + - Cluster Manager + - Extensions + - Indexing:Performance + - Indexing:Replication + - Indexing + - Libraries + - Other + - Plugins + - Search:Aggregations + - Search:Performance + - Search:Query Capabilities + - Search:Query Insights + - Search:Relevance + - Search:Remote Search + - Search:Resiliency + - Search:Searchable Snapshots + - Search + - Storage:Durability + - Storage:Performance + - Storage:Remote + - Storage:Snapshots + - Storage + validations: + required: true From bb0b4b0eac34b9d942351445764a7eb0ad97edf1 Mon Sep 17 00:00:00 2001 From: Paras Jain Date: Thu, 15 Feb 2024 05:39:30 +0530 Subject: [PATCH 03/41] Fixes ByteArrayIndexInput::validatePos and adds UT (#10551) * Prevent read beyond slice boundary in ByteArrayIndexInput Signed-off-by: Paras Jain * Fix spotless errors Signed-off-by: Andrew Ross --------- Signed-off-by: Paras Jain Signed-off-by: Andrew Ross Co-authored-by: Paras Jain Co-authored-by: Andrew Ross --- CHANGELOG.md | 1 + .../lucene/store/ByteArrayIndexInput.java | 2 +- .../store/ByteArrayIndexInputTests.java | 32 +++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de8a041da3b73..05427db6f7fad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -115,6 +115,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix for deserilization bug in weighted round-robin metadata ([#11679](https://github.com/opensearch-project/OpenSearch/pull/11679)) - Add a system property to configure YamlParser codepoint limits ([#12298](https://github.com/opensearch-project/OpenSearch/pull/12298)) +- Prevent read beyond slice boundary in ByteArrayIndexInput ([#10481](https://github.com/opensearch-project/OpenSearch/issues/10481)) ### Security diff --git a/server/src/main/java/org/opensearch/common/lucene/store/ByteArrayIndexInput.java b/server/src/main/java/org/opensearch/common/lucene/store/ByteArrayIndexInput.java index bb273b14c42e2..1804a9ac05a29 100644 --- a/server/src/main/java/org/opensearch/common/lucene/store/ByteArrayIndexInput.java +++ b/server/src/main/java/org/opensearch/common/lucene/store/ByteArrayIndexInput.java @@ -144,7 +144,7 @@ public long readLong(long pos) throws IOException { } private void validatePos(long pos, int len) throws EOFException { - if (pos < 0 || pos + len > length + offset) { + if (pos < 0 || pos + len > length) { throw new EOFException("seek past EOF"); } } diff --git a/server/src/test/java/org/opensearch/common/lucene/store/ByteArrayIndexInputTests.java b/server/src/test/java/org/opensearch/common/lucene/store/ByteArrayIndexInputTests.java index 827f9dd992294..ee71cfef7d925 100644 --- a/server/src/test/java/org/opensearch/common/lucene/store/ByteArrayIndexInputTests.java +++ b/server/src/test/java/org/opensearch/common/lucene/store/ByteArrayIndexInputTests.java @@ -32,6 +32,8 @@ package org.opensearch.common.lucene.store; +import org.apache.lucene.store.IndexInput; + import java.io.EOFException; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -153,4 +155,34 @@ public void testRandomAccessReads() throws IOException { // 10001001 00100101 10001001 00110000 11100111 00100100 10110001 00101110 assertEquals(-8564288273245753042L, indexInput.readLong(1)); } + + public void testReadBytesWithSlice() throws IOException { + int inputLength = randomIntBetween(100, 1000); + + byte[] input = randomUnicodeOfLength(inputLength).getBytes(StandardCharsets.UTF_8); + ByteArrayIndexInput indexInput = new ByteArrayIndexInput("test", input); + + int sliceOffset = randomIntBetween(1, inputLength - 10); + int sliceLength = randomIntBetween(2, inputLength - sliceOffset); + IndexInput slice = indexInput.slice("slice", sliceOffset, sliceLength); + + // read a byte from sliced index input and verify if the read value is correct + assertEquals(input[sliceOffset], slice.readByte()); + + // read few more bytes into a byte array + int bytesToRead = randomIntBetween(1, sliceLength - 1); + slice.readBytes(new byte[bytesToRead], 0, bytesToRead); + + // now try to read beyond the boundary of the slice, but within the + // boundary of the original IndexInput. We've already read few bytes + // so this is expected to fail + assertThrows(EOFException.class, () -> slice.readBytes(new byte[sliceLength], 0, sliceLength)); + + // seek to EOF and then try to read + slice.seek(sliceLength); + assertThrows(EOFException.class, () -> slice.readBytes(new byte[1], 0, 1)); + + slice.close(); + indexInput.close(); + } } From f73c82aa466cf3354b5d2e3e3f46f0613cacbb0f Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Thu, 15 Feb 2024 07:39:10 -0800 Subject: [PATCH 04/41] Allow security manager in IntelliJ unit tests (#12330) If you configure IntelliJ to use JDK 21 then all unit tests will fail to run in the IDE when attempting to set the security manager in the test bootstrap code. This change configures all tests to include the required VM argument to enable the security manager. Signed-off-by: Andrew Ross --- gradle/ide.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gradle/ide.gradle b/gradle/ide.gradle index bc442a081adf0..14d6b2982ccd0 100644 --- a/gradle/ide.gradle +++ b/gradle/ide.gradle @@ -82,6 +82,9 @@ if (System.getProperty('idea.active') == 'true') { runConfigurations { defaults(JUnit) { vmParameters = '-ea -Djava.locale.providers=SPI,COMPAT' + if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) { + vmParameters += ' -Djava.security.manager=allow' + } } } copyright { From 03e415e8956dbc63b014f6b49c10650d353c90ca Mon Sep 17 00:00:00 2001 From: bowenlan-amzn Date: Thu, 15 Feb 2024 09:43:57 -0800 Subject: [PATCH 05/41] Fix test logic error in FilterRewriteIT (#12327) Signed-off-by: bowenlan-amzn --- .../search/aggregations/bucket/FilterRewriteIT.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterRewriteIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterRewriteIT.java index 35e024603f0f2..e051265d4b3bc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterRewriteIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterRewriteIT.java @@ -59,19 +59,19 @@ public static Collection parameters() { @Override protected void setupSuiteScopeCluster() throws Exception { assertAcked(client().admin().indices().prepareCreate("idx").get()); + expected.clear(); - final int segmentCount = randomIntBetween(2, 10); - final Set longTerms = new HashSet(); + final int repeat = randomIntBetween(2, 10); + final Set longTerms = new HashSet<>(); - final Map dateTerms = new HashMap<>(); - for (int i = 0; i < segmentCount; i++) { + for (int i = 0; i < repeat; i++) { final List indexRequests = new ArrayList<>(); long longTerm; do { - longTerm = randomInt(segmentCount * 2); + longTerm = randomInt(repeat * 2); } while (!longTerms.add(longTerm)); - ZonedDateTime time = ZonedDateTime.of(2024, 1, ((int) longTerm % 20) + 1, 0, 0, 0, 0, ZoneOffset.UTC); + ZonedDateTime time = ZonedDateTime.of(2024, 1, ((int) longTerm) + 1, 0, 0, 0, 0, ZoneOffset.UTC); String dateTerm = DateFormatter.forPattern("yyyy-MM-dd").format(time); final int frequency = randomBoolean() ? 1 : randomIntBetween(2, 20); From 8fe15e6110986b05b65afe6c811deb4e347c985c Mon Sep 17 00:00:00 2001 From: Austin Lee Date: Thu, 15 Feb 2024 15:51:39 -0800 Subject: [PATCH 06/41] =?UTF-8?q?Fix=20a=20flaky=20test=20that=20relies=20?= =?UTF-8?q?on=20terminate=5Fafter=20for=20an=20exact=20count=20ma=E2=80=A6?= =?UTF-8?q?=20(#12179)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix a flaky test that relies on terminate_after for an exact count match. (Issue # 10435) Signed-off-by: Austin Lee * Make comments HTML Javadoc friendly. Signed-off-by: Austin Lee * Fix javadoc syntax Signed-off-by: Andrew Ross --------- Signed-off-by: Austin Lee Signed-off-by: Andrew Ross Co-authored-by: Andrew Ross --- .../search/simple/SimpleSearchIT.java | 81 +++++++++++++------ .../test/hamcrest/OpenSearchAssertions.java | 16 ++++ 2 files changed, 73 insertions(+), 24 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java index 0606bc1dbbdb6..1c1587a3be600 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java @@ -299,7 +299,15 @@ public void dotestSimpleTerminateAfterCountWithSize(int size, int max) throws Ex .setSize(size) .setTrackTotalHits(true) .get(); - assertHitCount(searchResponse, i); + + // Do not expect an exact match as an optimization introduced by https://issues.apache.org/jira/browse/LUCENE-10620 + // can produce a total hit count > terminated_after, but this only kicks in + // when size = 0 which is when TotalHitCountCollector is used. + if (size == 0) { + assertHitCount(searchResponse, i, max); + } else { + assertHitCount(searchResponse, i); + } assertTrue(searchResponse.isTerminatedEarly()); assertEquals(Math.min(i, size), searchResponse.getHits().getHits().length); } @@ -313,7 +321,6 @@ public void dotestSimpleTerminateAfterCountWithSize(int size, int max) throws Ex assertFalse(searchResponse.isTerminatedEarly()); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/10435") public void testSimpleTerminateAfterCountSize0() throws Exception { int max = randomIntBetween(3, 29); dotestSimpleTerminateAfterCountWithSize(0, max); @@ -324,6 +331,24 @@ public void testSimpleTerminateAfterCountRandomSize() throws Exception { dotestSimpleTerminateAfterCountWithSize(randomIntBetween(1, max), max); } + /** + * Special cases when size = 0: + * + * If track_total_hits = true: + * Weight#count optimization can cause totalHits in the response to be up to the total doc count regardless of terminate_after. + * So, we will have to do a range check, not an equality check. + * + * If track_total_hits != true, but set to a value AND terminate_after is set: + * Again, due to the optimization, any count can be returned. + * Up to terminate_after, relation == EQUAL_TO. + * But if track_total_hits_up_to ≥ terminate_after, relation can be EQ _or_ GTE. + * This ambiguity is due to the fact that totalHits == track_total_hits_up_to + * or totalHits > track_total_hits_up_to and SearchPhaseController sets totalHits = track_total_hits_up_to when returning results + * in which case relation = GTE. + * + * @param size + * @throws Exception + */ public void doTestSimpleTerminateAfterTrackTotalHitsUpTo(int size) throws Exception { prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)).get(); ensureGreen(); @@ -340,6 +365,7 @@ public void doTestSimpleTerminateAfterTrackTotalHitsUpTo(int size) throws Except refresh(); SearchResponse searchResponse; + searchResponse = client().prepareSearch("test") .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) .setTerminateAfter(10) @@ -350,25 +376,28 @@ public void doTestSimpleTerminateAfterTrackTotalHitsUpTo(int size) throws Except assertEquals(5, searchResponse.getHits().getTotalHits().value); assertEquals(GREATER_THAN_OR_EQUAL_TO, searchResponse.getHits().getTotalHits().relation); - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) - .setTerminateAfter(5) - .setSize(size) - .setTrackTotalHitsUpTo(10) - .get(); - assertTrue(searchResponse.isTerminatedEarly()); - assertEquals(5, searchResponse.getHits().getTotalHits().value); - assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); + // For size = 0, the following queries terminate early, but hits and relation can vary. + if (size > 0) { + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) + .setTerminateAfter(5) + .setSize(size) + .setTrackTotalHitsUpTo(10) + .get(); + assertTrue(searchResponse.isTerminatedEarly()); + assertEquals(5, searchResponse.getHits().getTotalHits().value); + assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) - .setTerminateAfter(5) - .setSize(size) - .setTrackTotalHitsUpTo(5) - .get(); - assertTrue(searchResponse.isTerminatedEarly()); - assertEquals(5, searchResponse.getHits().getTotalHits().value); - assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) + .setTerminateAfter(5) + .setSize(size) + .setTrackTotalHitsUpTo(5) + .get(); + assertTrue(searchResponse.isTerminatedEarly()); + assertEquals(5, searchResponse.getHits().getTotalHits().value); + assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); + } searchResponse = client().prepareSearch("test") .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) @@ -377,7 +406,12 @@ public void doTestSimpleTerminateAfterTrackTotalHitsUpTo(int size) throws Except .setTrackTotalHits(true) .get(); assertTrue(searchResponse.isTerminatedEarly()); - assertEquals(5, searchResponse.getHits().getTotalHits().value); + if (size == 0) { + // Since terminate_after < track_total_hits, we need to do a range check. + assertHitCount(searchResponse, 5, numDocs); + } else { + assertEquals(5, searchResponse.getHits().getTotalHits().value); + } assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); searchResponse = client().prepareSearch("test") @@ -399,12 +433,11 @@ public void doTestSimpleTerminateAfterTrackTotalHitsUpTo(int size) throws Except assertEquals(GREATER_THAN_OR_EQUAL_TO, searchResponse.getHits().getTotalHits().relation); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/10435") - public void testSimpleTerminateAfterTrackTotalHitsUpToRandomSize() throws Exception { + public void testSimpleTerminateAfterTrackTotalHitsUpToRandomSize0() throws Exception { doTestSimpleTerminateAfterTrackTotalHitsUpTo(0); } - public void testSimpleTerminateAfterTrackTotalHitsUpToSize0() throws Exception { + public void testSimpleTerminateAfterTrackTotalHitsUpToSize() throws Exception { doTestSimpleTerminateAfterTrackTotalHitsUpTo(randomIntBetween(1, 29)); } diff --git a/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java b/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java index 9b0de13c35ec8..650558aaa97a6 100644 --- a/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java +++ b/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java @@ -304,6 +304,22 @@ public static void assertHitCount(SearchResponse countResponse, long expectedHit } } + public static void assertHitCount(SearchResponse countResponse, long minHitCount, long maxHitCount) { + final TotalHits totalHits = countResponse.getHits().getTotalHits(); + if (!(totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value >= minHitCount && totalHits.value <= maxHitCount)) { + fail( + "Count is " + + totalHits + + " not between " + + minHitCount + + " and " + + maxHitCount + + " inclusive. " + + formatShardStatus(countResponse) + ); + } + } + public static void assertExists(GetResponse response) { String message = String.format(Locale.ROOT, "Expected %s/%s to exist, but does not", response.getIndex(), response.getId()); assertThat(message, response.isExists(), is(true)); From 006ba0e6068d18d5523f8975de7c4e1e5e797490 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 15 Feb 2024 20:13:55 -0600 Subject: [PATCH 07/41] Fix error in RemoteSegmentStoreDirectory when debug logging is enabled (#12328) Signed-off-by: Peter Nied --- CHANGELOG.md | 1 + .../store/RemoteSegmentStoreDirectory.java | 8 +-- .../RemoteSegmentStoreDirectoryTests.java | 47 ++++++++++----- .../org/opensearch/test/MockLogAppender.java | 57 +++++++++++++++++++ .../opensearch/test/OpenSearchTestCase.java | 28 +++++++++ 5 files changed, 124 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05427db6f7fad..d93d404e0c6dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,6 +90,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Update supported version for must_exist parameter in update aliases API ([#11872](https://github.com/opensearch-project/OpenSearch/pull/11872)) - [Bug] Check phase name before SearchRequestOperationsListener onPhaseStart ([#12035](https://github.com/opensearch-project/OpenSearch/pull/12035)) - Fix Span operation names generated from RestActions ([#12005](https://github.com/opensearch-project/OpenSearch/pull/12005)) +- Fix error in RemoteSegmentStoreDirectory when debug logging is enabled ([#12328](https://github.com/opensearch-project/OpenSearch/pull/12328)) ### Security diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index dab99fd25b192..bfab9f8c18aa2 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -45,6 +45,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.NoSuchFileException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -745,9 +746,8 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException return; } - List metadataFilesEligibleToDelete = sortedMetadataFileList.subList( - lastNMetadataFilesToKeep, - sortedMetadataFileList.size() + List metadataFilesEligibleToDelete = new ArrayList<>( + sortedMetadataFileList.subList(lastNMetadataFilesToKeep, sortedMetadataFileList.size()) ); Set allLockFiles; try { @@ -764,7 +764,7 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException logger.debug( "metadataFilesEligibleToDelete={} metadataFilesToBeDeleted={}", metadataFilesEligibleToDelete, - metadataFilesEligibleToDelete + metadataFilesToBeDeleted ); Map activeSegmentFilesMetadataMap = new HashMap<>(); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 7944ee681f5fc..e2ebb2e642bfe 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -8,6 +8,7 @@ package org.opensearch.index.store; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.CodecUtil; @@ -41,6 +42,8 @@ import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.MockLogAppender; +import org.opensearch.test.junit.annotations.TestLogging; import org.opensearch.threadpool.ThreadPool; import org.junit.After; import org.junit.Before; @@ -58,6 +61,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.mockito.Mockito; @@ -971,21 +975,38 @@ public void testDeleteStaleCommitsWithinThreshold() throws Exception { verify(remoteMetadataDirectory, times(0)).openInput(any(String.class), eq(IOContext.DEFAULT)); } + @TestLogging(value = "_root:debug", reason = "Validate logging output") public void testDeleteStaleCommitsActualDelete() throws Exception { - Map> metadataFilenameContentMapping = populateMetadata(); - remoteSegmentStoreDirectory.init(); - - // popluateMetadata() adds stub to return 3 metadata files - // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted - remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); - - for (String metadata : metadataFilenameContentMapping.get(metadataFilename3).values()) { - String uploadedFilename = metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]; - verify(remoteDataDirectory).deleteFile(uploadedFilename); + try (final MockLogAppender appender = MockLogAppender.createForLoggers(LogManager.getRootLogger())) { + appender.addExpectation( + new MockLogAppender.PatternSeenWithLoggerPrefixExpectation( + "Metadata files to delete message", + "org.opensearch.index.store.RemoteSegmentStoreDirectory", + Level.DEBUG, + "metadataFilesEligibleToDelete=\\[" + metadataFilename3 + "\\] metadataFilesToBeDeleted=\\[" + metadataFilename3 + "\\]" + ) + ); + + final Map> metadataFilenameContentMapping = populateMetadata(); + final List filesToBeDeleted = metadataFilenameContentMapping.get(metadataFilename3) + .values() + .stream() + .map(metadata -> metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]) + .collect(Collectors.toList()); + + remoteSegmentStoreDirectory.init(); + + // popluateMetadata() adds stub to return 3 metadata files + // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted + remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); + + for (final String file : filesToBeDeleted) { + verify(remoteDataDirectory).deleteFile(file); + } + assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); + verify(remoteMetadataDirectory).deleteFile(metadataFilename3); + appender.assertAllExpectationsMatched(); } - ; - assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); - verify(remoteMetadataDirectory).deleteFile(metadataFilename3); } public void testDeleteStaleCommitsActualDeleteWithLocks() throws Exception { diff --git a/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java b/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java index 59eda7a665d4c..6d6199833b25b 100644 --- a/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java +++ b/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java @@ -39,15 +39,19 @@ import org.apache.logging.log4j.core.filter.RegexFilter; import org.opensearch.common.logging.Loggers; import org.opensearch.common.regex.Regex; +import org.opensearch.test.junit.annotations.TestLogging; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.fail; /** * Test appender that can be used to verify that certain events were logged correctly @@ -259,6 +263,59 @@ public void assertMatched() { } + /** + * Used for cases when the logger is dynamically named such as to include an index name or shard id + * + * Best used in conjunction with the root logger: + * {@code @TestLogging(value = "_root:debug", reason = "Validate logging output");} + * @see TestLogging + * */ + public static class PatternSeenWithLoggerPrefixExpectation implements LoggingExpectation { + private final String expectationName; + private final String loggerPrefix; + private final Level level; + private final String messageMatchingRegex; + + private final List loggerMatches = new ArrayList<>(); + private final AtomicBoolean eventSeen = new AtomicBoolean(false); + + public PatternSeenWithLoggerPrefixExpectation( + final String expectationName, + final String loggerPrefix, + final Level level, + final String messageMatchingRegex + ) { + this.expectationName = expectationName; + this.loggerPrefix = loggerPrefix; + this.level = level; + this.messageMatchingRegex = messageMatchingRegex; + } + + @Override + public void match(final LogEvent event) { + if (event.getLevel() == level && event.getLoggerName().startsWith(loggerPrefix)) { + final String formattedMessage = event.getMessage().getFormattedMessage(); + loggerMatches.add(formattedMessage); + if (formattedMessage.matches(messageMatchingRegex)) { + eventSeen.set(true); + } + } + } + + @Override + public void assertMatched() { + if (!eventSeen.get()) { + final StringBuilder failureMessage = new StringBuilder(); + failureMessage.append(expectationName + " was not seen, found " + loggerMatches.size() + " messages matching the logger."); + failureMessage.append("\r\nMessage matching regex: " + messageMatchingRegex); + if (!loggerMatches.isEmpty()) { + failureMessage.append("\r\nMessage details:\r\n" + String.join("\r\n", loggerMatches)); + } + fail(failureMessage.toString()); + } + } + } + private static String getLoggerName(String name) { if (name.startsWith("org.opensearch.")) { name = name.substring("org.opensearch.".length()); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index 96bffcf2d3692..aac3fca9e1e16 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -145,6 +145,8 @@ import java.io.IOException; import java.io.InputStream; +import java.io.PrintWriter; +import java.io.StringWriter; import java.math.BigInteger; import java.net.InetAddress; import java.net.UnknownHostException; @@ -170,6 +172,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BooleanSupplier; import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.IntFunction; import java.util.function.Predicate; import java.util.function.Supplier; @@ -639,7 +642,32 @@ protected static void checkStaticState(boolean afterClass) throws Exception { try { // ensure that there are no status logger messages which would indicate a problem with our Log4j usage; we map the // StatusData instances to Strings as otherwise their toString output is useless + + final Function statusToString = (statusData) -> { + try (final StringWriter sw = new StringWriter(); final PrintWriter pw = new PrintWriter(sw)) { + + pw.print(statusData.getLevel()); + pw.print(":"); + pw.print(statusData.getMessage().getFormattedMessage()); + + if (statusData.getStackTraceElement() != null) { + final var messageSource = statusData.getStackTraceElement(); + pw.println("Source:"); + pw.println(messageSource.getFileName() + "@" + messageSource.getLineNumber()); + } + + if (statusData.getThrowable() != null) { + pw.println("Throwable:"); + statusData.getThrowable().printStackTrace(pw); + } + return sw.toString(); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + }; + assertThat( + statusData.stream().map(statusToString::apply).collect(Collectors.joining("\r\n")), statusData.stream().map(status -> status.getMessage().getFormattedMessage()).collect(Collectors.toList()), empty() ); From 95a181914e3035d604db0ef24f56a3920cbeaa87 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Fri, 16 Feb 2024 15:21:53 +0530 Subject: [PATCH 08/41] [Remote Store] Introducing mixed mode support for remote store migration (#11986) * Introducing mixed mode support for remote store migration Signed-off-by: Gaurav Bafna * Adding changelog Signed-off-by: Gaurav Bafna * Refactoring IT and naming changes Signed-off-by: Gaurav Bafna * Missing java doc Signed-off-by: Gaurav Bafna --------- Signed-off-by: Gaurav Bafna --- CHANGELOG.md | 1 + .../DocRepMigrationTestCase.java | 38 ++++++++++ .../MigrationBaseTestCase.java | 50 +++++++++++++ .../RemoteStoreMigrationTestCase.java | 74 +++++++++++++++++++ .../coordination/JoinTaskExecutor.java | 66 +++++++++++------ .../common/settings/ClusterSettings.java | 1 + .../common/settings/FeatureFlagSettings.java | 3 +- .../opensearch/common/util/FeatureFlags.java | 11 +++ .../remotestore/RemoteStoreNodeService.java | 61 ++++++++++++++- .../coordination/JoinTaskExecutorTests.java | 61 +++++++++++++++ 10 files changed, 341 insertions(+), 25 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotemigration/DocRepMigrationTestCase.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d93d404e0c6dc..99f63c596c72e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625)) - [Admission Control] Integrate CPU AC with ResourceUsageCollector and add CPU AC stats to nodes/stats ([#10887](https://github.com/opensearch-project/OpenSearch/pull/10887)) - [S3 Repository] Add setting to control connection count for sync client ([#12028](https://github.com/opensearch-project/OpenSearch/pull/12028)) +- Add Remote Store Migration Experimental flag and allow mixed mode clusters under same ([#11986](https://github.com/opensearch-project/OpenSearch/pull/11986)) ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/DocRepMigrationTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/DocRepMigrationTestCase.java new file mode 100644 index 0000000000000..5240949ff87b9 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/DocRepMigrationTestCase.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotemigration; + +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.List; + +import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) +public class DocRepMigrationTestCase extends MigrationBaseTestCase { + + public void testMixedModeAddDocRep() throws Exception { + internalCluster().setBootstrapClusterManagerNodeIndex(0); + List cmNodes = internalCluster().startNodes(1); + + Client client = internalCluster().client(cmNodes.get(0)); + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + addRemote = false; + internalCluster().startNode(); + String[] allNodes = internalCluster().getNodeNames(); + assertBusy(() -> { assertEquals(client.admin().cluster().prepareClusterStats().get().getNodes().size(), allNodes.length); }); + } + +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java new file mode 100644 index 0000000000000..88d6f6897ee68 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotemigration; + +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.nio.file.Path; + +import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; + +public class MigrationBaseTestCase extends OpenSearchIntegTestCase { + protected static final String REPOSITORY_NAME = "test-remote-store-repo"; + protected static final String REPOSITORY_2_NAME = "test-remote-store-repo-2"; + + protected Path segmentRepoPath; + protected Path translogRepoPath; + + boolean addRemote = false; + + protected Settings nodeSettings(int nodeOrdinal) { + if (segmentRepoPath == null || translogRepoPath == null) { + segmentRepoPath = randomRepoPath().toAbsolutePath(); + translogRepoPath = randomRepoPath().toAbsolutePath(); + } + if (addRemote) { + logger.info("Adding remote store node"); + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, segmentRepoPath, REPOSITORY_2_NAME, translogRepoPath)) + .put("discovery.initial_state_timeout", "500ms") + .build(); + } else { + logger.info("Adding docrep node"); + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put("discovery.initial_state_timeout", "500ms").build(); + } + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build(); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java new file mode 100644 index 0000000000000..a31d203058565 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotemigration; + +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.List; + +import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; +import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) +public class RemoteStoreMigrationTestCase extends MigrationBaseTestCase { + public void testMixedModeAddRemoteNodes() throws Exception { + internalCluster().setBootstrapClusterManagerNodeIndex(0); + List cmNodes = internalCluster().startNodes(1); + Client client = internalCluster().client(cmNodes.get(0)); + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + // add remote node in mixed mode cluster + addRemote = true; + internalCluster().startNode(); + internalCluster().startNode(); + internalCluster().validateClusterFormed(); + + // assert repo gets registered + GetRepositoriesRequest gr = new GetRepositoriesRequest(new String[] { REPOSITORY_NAME }); + GetRepositoriesResponse getRepositoriesResponse = client.admin().cluster().getRepositories(gr).actionGet(); + assertEquals(1, getRepositoriesResponse.repositories().size()); + + // add docrep mode in mixed mode cluster + addRemote = true; + internalCluster().startNode(); + assertBusy(() -> { + assertEquals(client.admin().cluster().prepareClusterStats().get().getNodes().size(), internalCluster().getNodeNames().length); + }); + + // add incompatible remote node in remote mixed cluster + Settings.Builder badSettings = Settings.builder() + .put(remoteStoreClusterSettings(REPOSITORY_NAME, segmentRepoPath, "REPOSITORY_2_NAME", translogRepoPath)) + .put("discovery.initial_state_timeout", "500ms"); + String badNode = internalCluster().startNode(badSettings); + assertTrue(client.admin().cluster().prepareClusterStats().get().getNodes().size() < internalCluster().getNodeNames().length); + internalCluster().stopRandomNode(settings -> settings.get("node.name").equals(badNode)); + } + + public void testMigrationDirections() { + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + // add remote node in docrep cluster + updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), "docrep")); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), "random")); + assertThrows(IllegalArgumentException.class, () -> client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + } +} diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index f701a2f52277d..bc365b9872037 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -58,6 +58,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.BiConsumer; import java.util.stream.Collectors; @@ -176,12 +177,13 @@ public ClusterTasksResult execute(ClusterState currentState, List jo DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(newState.nodes()); - // TODO: We are using one of the existing node to build the repository metadata, this will need to be updated - // once we start supporting mixed compatibility mode. An optimization can be done as this will get invoked + // An optimization can be done as this will get invoked // for every set of node join task which we can optimize to not compute if cluster state already has // repository information. + Optional remoteDN = currentNodes.getNodes().values().stream().filter(DiscoveryNode::isRemoteStoreNode).findFirst(); + DiscoveryNode dn = remoteDN.orElseGet(() -> (currentNodes.getNodes().values()).stream().findFirst().get()); RepositoriesMetadata repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( - (currentNodes.getNodes().values()).stream().findFirst().get(), + dn, currentState.getMetadata().custom(RepositoriesMetadata.TYPE) ); @@ -212,6 +214,16 @@ public ClusterTasksResult execute(ClusterState currentState, List jo // would guarantee that a decommissioned node would never be able to join the cluster and ensures correctness ensureNodeCommissioned(node, currentState.metadata()); nodesBuilder.add(node); + + if (remoteDN.isEmpty()) { + // This is hit only on cases where we encounter first remote node + logger.info("Updating system repository now for remote store"); + repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( + node, + currentState.getMetadata().custom(RepositoriesMetadata.TYPE) + ); + } + nodesChanged = true; minClusterNodeVersion = Version.min(minClusterNodeVersion, node.getVersion()); maxClusterNodeVersion = Version.max(maxClusterNodeVersion, node.getVersion()); @@ -495,36 +507,46 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod assert existingNodes.isEmpty() == false; - // TODO: The below check is valid till we don't support migration, once we start supporting migration a remote - // store node will be able to join a non remote store cluster and vice versa. #7986 CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(metadata.settings()); if (STRICT.equals(remoteStoreCompatibilityMode)) { + DiscoveryNode existingNode = existingNodes.get(0); if (joiningNode.isRemoteStoreNode()) { + ensureRemoteStoreNodesCompatibility(joiningNode, existingNode); + } else { if (existingNode.isRemoteStoreNode()) { - RemoteStoreNodeAttribute joiningRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(joiningNode); - RemoteStoreNodeAttribute existingRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(existingNode); - if (existingRemoteStoreNodeAttribute.equals(joiningRemoteStoreNodeAttribute) == false) { - throw new IllegalStateException( - "a remote store node [" - + joiningNode - + "] is trying to join a remote store cluster with incompatible node attributes in " - + "comparison with existing node [" - + existingNode - + "]" - ); - } - } else { throw new IllegalStateException( - "a remote store node [" + joiningNode + "] is trying to join a non remote store cluster" + "a non remote store node [" + joiningNode + "] is trying to join a remote store cluster" ); } - } else { - if (existingNode.isRemoteStoreNode()) { + } + } else { + if (remoteStoreCompatibilityMode == CompatibilityMode.MIXED) { + if (joiningNode.isRemoteStoreNode()) { + Optional remoteDN = existingNodes.stream().filter(DiscoveryNode::isRemoteStoreNode).findFirst(); + remoteDN.ifPresent(discoveryNode -> ensureRemoteStoreNodesCompatibility(joiningNode, discoveryNode)); + } + } + } + } + + private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, DiscoveryNode existingNode) { + if (joiningNode.isRemoteStoreNode()) { + if (existingNode.isRemoteStoreNode()) { + RemoteStoreNodeAttribute joiningRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(joiningNode); + RemoteStoreNodeAttribute existingRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(existingNode); + if (existingRemoteStoreNodeAttribute.equals(joiningRemoteStoreNodeAttribute) == false) { throw new IllegalStateException( - "a non remote store node [" + joiningNode + "] is trying to join a remote store cluster" + "a remote store node [" + + joiningNode + + "] is trying to join a remote store cluster with incompatible node attributes in " + + "comparison with existing node [" + + existingNode + + "]" ); } + } else { + throw new IllegalStateException("a remote store node [" + joiningNode + "] is trying to join a non remote store cluster"); } } } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 0c97d62c44a5e..896a234c115b6 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -698,6 +698,7 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, RemoteClusterStateService.METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING, + RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING, IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING, IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING, AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE, diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index e6d7ba0c60772..47da53b52c325 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -35,6 +35,7 @@ protected FeatureFlagSettings( FeatureFlags.TELEMETRY_SETTING, FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING, FeatureFlags.WRITEABLE_REMOTE_INDEX_SETTING, - FeatureFlags.DOC_ID_FUZZY_SET_SETTING + FeatureFlags.DOC_ID_FUZZY_SET_SETTING, + FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 075dc9934e130..b51efeab21254 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -20,6 +20,11 @@ * @opensearch.internal */ public class FeatureFlags { + /** + * Gates the visibility of the remote store migration support from docrep . + */ + public static final String REMOTE_STORE_MIGRATION_EXPERIMENTAL = "opensearch.experimental.feature.remote_store.migration.enabled"; + /** * Gates the ability for Searchable Snapshots to read snapshots that are older than the * guaranteed backward compatibility for OpenSearch (one prior major version) on a best effort basis. @@ -98,6 +103,12 @@ public static boolean isEnabled(Setting featureFlag) { } } + public static final Setting REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting( + REMOTE_STORE_MIGRATION_EXPERIMENTAL, + false, + Property.NodeScope + ); + public static final Setting EXTENSIONS_SETTING = Setting.boolSetting(EXTENSIONS, false, Property.NodeScope); public static final Setting IDENTITY_SETTING = Setting.boolSetting(IDENTITY, false, Property.NodeScope); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index ca2413a057a6b..33b182dd3cc97 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -15,18 +15,22 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Setting; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.RepositoryException; import org.opensearch.threadpool.ThreadPool; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.function.Supplier; +import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL; + /** * Contains all the method needed for a remote store backed node lifecycle. */ @@ -39,6 +43,33 @@ public class RemoteStoreNodeService { "remote_store.compatibility_mode", CompatibilityMode.STRICT.name(), CompatibilityMode::parseString, + value -> { + if (value == CompatibilityMode.MIXED + && FeatureFlags.isEnabled(FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING) == false) { + throw new IllegalArgumentException( + " mixed mode is under an experimental feature and can be activated only by enabling " + + REMOTE_STORE_MIGRATION_EXPERIMENTAL + + " feature flag in the JVM options " + ); + } + }, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + public static final Setting MIGRATION_DIRECTION_SETTING = new Setting<>( + "migration.direction", + Direction.NONE.name(), + Direction::parseString, + value -> { + if (value != Direction.NONE && FeatureFlags.isEnabled(FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING) == false) { + throw new IllegalArgumentException( + " migration.direction is under an experimental feature and can be activated only by enabling " + + REMOTE_STORE_MIGRATION_EXPERIMENTAL + + " feature flag in the JVM options " + ); + } + }, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -49,7 +80,8 @@ public class RemoteStoreNodeService { * @opensearch.internal */ public enum CompatibilityMode { - STRICT("strict"); + STRICT("strict"), + MIXED("mixed"); public final String mode; @@ -66,13 +98,38 @@ public static CompatibilityMode parseString(String compatibilityMode) { + compatibilityMode + "] compatibility mode is not supported. " + "supported modes are [" - + CompatibilityMode.values().toString() + + Arrays.toString(CompatibilityMode.values()) + "]" ); } } } + /** + * Migration Direction intended for docrep to remote store migration and vice versa + * + * @opensearch.internal + */ + public enum Direction { + REMOTE_STORE("remote_store"), + NONE("none"), + DOCREP("docrep"); + + public final String direction; + + Direction(String d) { + this.direction = d; + } + + public static Direction parseString(String direction) { + try { + return Direction.valueOf(direction.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("[" + direction + "] migration.direction is not supported."); + } + } + } + public RemoteStoreNodeService(Supplier repositoriesService, ThreadPool threadPool) { this.repositoriesService = repositoriesService; this.threadPool = threadPool; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index 5952cc1bcaac2..be25bee5fe7b1 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -52,6 +52,7 @@ import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -67,11 +68,14 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; import static org.opensearch.test.VersionUtils.allVersions; import static org.opensearch.test.VersionUtils.maxCompatibleVersion; import static org.opensearch.test.VersionUtils.randomCompatibleVersion; @@ -393,6 +397,7 @@ public void testJoinClusterWithNonRemoteStoreNodeJoiningNonRemoteStoreCluster() } public void testPreventJoinClusterWithRemoteStoreNodeJoiningNonRemoteStoreCluster() { + final DiscoveryNode existingNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) @@ -406,6 +411,62 @@ public void testPreventJoinClusterWithRemoteStoreNodeJoiningNonRemoteStoreCluste assertTrue(e.getMessage().equals("a remote store node [" + joiningNode + "] is trying to join a non remote " + "store cluster")); } + public void testRemoteStoreNodeJoiningNonRemoteStoreClusterMixedMode() { + final DiscoveryNode existingNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); + final Settings settings = Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.REMOTE_STORE) + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + .build(); + final Settings nodeSettings = Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + Metadata metadata = Metadata.builder().persistentSettings(settings).build(); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .metadata(metadata) + .build(); + + DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); + } + + public void testAllTypesNodeJoiningRemoteStoreClusterMixedMode() { + final DiscoveryNode docrepNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNode remoteNode = newDiscoveryNode(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); + final Settings settings = Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.REMOTE_STORE) + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + .build(); + final Settings nodeSettings = Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + Metadata metadata = Metadata.builder().persistentSettings(settings).build(); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes( + DiscoveryNodes.builder() + .add(docrepNode) + .localNodeId(docrepNode.getId()) + .add(remoteNode) + .localNodeId(remoteNode.getId()) + .build() + ) + .metadata(metadata) + .build(); + + // compatible remote node should not be able to join a mixed mode having a remote node + DiscoveryNode goodRemoteNode = newDiscoveryNode(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); + JoinTaskExecutor.ensureNodesCompatibility(goodRemoteNode, currentState.getNodes(), currentState.metadata()); + + // incompatible node should not be able to join a mixed mode + DiscoveryNode badRemoteNode = newDiscoveryNode(remoteStoreNodeAttributes(TRANSLOG_REPO, TRANSLOG_REPO)); + assertThrows( + IllegalStateException.class, + () -> JoinTaskExecutor.ensureNodesCompatibility(badRemoteNode, currentState.getNodes(), currentState.metadata()) + ); + + // DocRep node should be able to join a mixed mode + DiscoveryNode docrepNode2 = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); + JoinTaskExecutor.ensureNodesCompatibility(docrepNode2, currentState.getNodes(), currentState.metadata()); + } + public void testJoinClusterWithRemoteStoreNodeJoiningRemoteStoreCluster() { final DiscoveryNode existingNode = new DiscoveryNode( UUIDs.base64UUID(), From 4b51a855d9bbe67239c93078af435962acf62f10 Mon Sep 17 00:00:00 2001 From: rishavz_sagar Date: Fri, 16 Feb 2024 20:13:24 +0530 Subject: [PATCH 09/41] Fixing flaky test cases (#12320) Signed-off-by: RS146BIJAY --- .../decider/DiskThresholdDeciderIT.java | 4 +- .../cluster/shards/ClusterShardLimitIT.java | 15 ++++--- .../index/engine/InternalEngineTests.java | 41 +++++++++++++++++-- 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index 089a91a30dd17..cc8747e5f5666 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -245,8 +245,10 @@ public void testIndexCreateBlockIsRemovedWhenAnyNodesNotExceedHighWatermarkWithA (discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, TOTAL_SPACE_BYTES, TOTAL_SPACE_BYTES) ); - // Validate if index create block is removed on the cluster + // Validate if index create block is removed on the cluster. Need to refresh this periodically as well to remove + // the node from high watermark breached list. assertBusy(() -> { + clusterInfoService.refresh(); ClusterState state1 = client().admin().cluster().prepareState().setLocal(true).get().getState(); assertFalse(state1.blocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id())); }, 30L, TimeUnit.SECONDS); diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java index fb97ae59aae91..5eef7074e1dd6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java @@ -245,23 +245,22 @@ public void testIndexCreationOverLimitForDotIndexesFail() { assertFalse(clusterState.getMetadata().hasIndex(".test-index")); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6287") public void testCreateIndexWithMaxClusterShardSetting() { - int dataNodes = client().admin().cluster().prepareState().get().getState().getNodes().getDataNodes().size(); - ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); - setMaxShardLimit(dataNodes, shardsPerNodeKey); + int maxAllowedShardsPerNode = client().admin().cluster().prepareState().get().getState().getNodes().getDataNodes().size(); + setMaxShardLimit(maxAllowedShardsPerNode, shardsPerNodeKey); - int maxAllowedShards = dataNodes + 1; - int extraShardCount = maxAllowedShards + 1; + // Always keep + int maxAllowedShardsPerCluster = maxAllowedShardsPerNode * 1000; + int extraShardCount = 1; // Getting total active shards in the cluster. int currentActiveShards = client().admin().cluster().prepareHealth().get().getActiveShards(); try { - setMaxShardLimit(maxAllowedShards, SETTING_MAX_SHARDS_PER_CLUSTER_KEY); + setMaxShardLimit(maxAllowedShardsPerCluster, SETTING_MAX_SHARDS_PER_CLUSTER_KEY); prepareCreate("test_index_with_cluster_shard_limit").setSettings( Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, extraShardCount).put(SETTING_NUMBER_OF_REPLICAS, 0).build() ).get(); } catch (final IllegalArgumentException ex) { - verifyException(Math.min(maxAllowedShards, dataNodes * dataNodes), currentActiveShards, extraShardCount, ex); + verifyException(maxAllowedShardsPerCluster, currentActiveShards, extraShardCount, ex); } finally { setMaxShardLimit(-1, SETTING_MAX_SHARDS_PER_CLUSTER_KEY); } diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index 5b586524d0bfc..cc927a19fd01a 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -78,6 +78,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; +import org.apache.lucene.tests.index.ForceMergePolicy; import org.apache.lucene.tests.mockfile.ExtrasFS; import org.apache.lucene.tests.store.MockDirectoryWrapper; import org.apache.lucene.util.Bits; @@ -152,6 +153,7 @@ import org.opensearch.index.translog.TranslogDeletionPolicyFactory; import org.opensearch.index.translog.TranslogException; import org.opensearch.index.translog.listener.TranslogEventListener; +import org.opensearch.test.DummyShardLock; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.MockLogAppender; import org.opensearch.test.VersionUtils; @@ -3278,12 +3280,15 @@ public void onFailedEngine(String reason, Exception e) { final AtomicReference retentionLeasesHolder = new AtomicReference<>( new RetentionLeases(primaryTerm, retentionLeasesVersion.get(), Collections.emptyList()) ); + + // Just allow force merge so that regular merge does not close the shard first before any any other operation + // InternalEngine engine = createEngine( config( defaultSettings, store, createTempDir(), - newMergePolicy(), + newForceMergePolicy(), null, null, null, @@ -3377,7 +3382,7 @@ public void onFailedEngine(String reason, Exception e) { defaultSettings, store, createTempDir(), - newMergePolicy(), + newForceMergePolicy(), null, null, null, @@ -3446,7 +3451,8 @@ public void eval(MockDirectoryWrapper dir) throws IOException { wrapper.failOn(fail); MockLogAppender mockAppender = MockLogAppender.createForLoggers(Loggers.getLogger(Engine.class, shardId)); try { - Store store = createStore(wrapper); + // Create a store where directory is closed during unreferenced file cleanup. + Store store = createFailingDirectoryStore(wrapper); final Engine.EventListener eventListener = new Engine.EventListener() { @Override public void onFailedEngine(String reason, Exception e) { @@ -3473,7 +3479,7 @@ public void onFailedEngine(String reason, Exception e) { defaultSettings, store, createTempDir(), - newMergePolicy(), + newForceMergePolicy(), null, null, null, @@ -3534,6 +3540,33 @@ public void testSettings() { assertEquals(currentIndexWriterConfig.getCodec().getName(), codecService.codec(codecName).getName()); } + /** + * Creates a merge policy which only supports force merge. + * @return returns a merge policy which only supports force merge. + */ + private MergePolicy newForceMergePolicy() { + return new ForceMergePolicy(new TieredMergePolicy()); + } + + /** + * Create a store where directory is closed when referenced while unreferenced file cleanup. + * + * @param directory directory used for creating the store. + * @return a store where directory is closed when referenced while unreferenced file cleanup. + */ + private Store createFailingDirectoryStore(final Directory directory) { + return new Store(shardId, INDEX_SETTINGS, directory, new DummyShardLock(shardId)) { + @Override + public Directory directory() { + if (callStackContainsAnyOf("cleanUpUnreferencedFiles")) { + throw new AlreadyClosedException("store is already closed"); + } + + return super.directory(); + } + }; + } + public void testCurrentTranslogUUIIDIsCommitted() throws IOException { final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); try (Store store = createStore()) { From b19e4270f07e80b7d4fcc9473ff46deb61e4719c Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 16 Feb 2024 11:54:13 -0500 Subject: [PATCH 10/41] Add support of special WrappingSearchAsyncActionPhase so the onPhaseStart() will always be followed by onPhaseEnd() within AbstractSearchAsyncAction (#12293) * Add support of special WrappingSearchAsyncActionPhase so the onPhaseStart() will always be followed by onPhaseEnd() within AbstractSearchAsyncAction Signed-off-by: Andriy Redko * Address code review comments Signed-off-by: Andriy Redko --------- Signed-off-by: Andriy Redko --- CHANGELOG.md | 2 + .../search/AbstractSearchAsyncAction.java | 18 +- .../action/search/SearchPhaseName.java | 13 - .../action/search/TransportSearchAction.java | 14 +- .../WrappingSearchAsyncActionPhase.java | 35 +++ .../AbstractSearchAsyncActionTests.java | 72 +++-- .../CanMatchPreFilterSearchPhaseTests.java | 249 +++++++++++++++++- .../action/search/MockSearchPhaseContext.java | 12 +- 8 files changed, 345 insertions(+), 70 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/search/WrappingSearchAsyncActionPhase.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 99f63c596c72e..60e68edbfffbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix for deserilization bug in weighted round-robin metadata ([#11679](https://github.com/opensearch-project/OpenSearch/pull/11679)) +- [Revert] [Bug] Check phase name before SearchRequestOperationsListener onPhaseStart ([#12035](https://github.com/opensearch-project/OpenSearch/pull/12035)) +- Add support of special WrappingSearchAsyncActionPhase so the onPhaseStart() will always be followed by onPhaseEnd() within AbstractSearchAsyncAction ([#12293](https://github.com/opensearch-project/OpenSearch/pull/12293)) - Add a system property to configure YamlParser codepoint limits ([#12298](https://github.com/opensearch-project/OpenSearch/pull/12298)) - Prevent read beyond slice boundary in ByteArrayIndexInput ([#10481](https://github.com/opensearch-project/OpenSearch/issues/10481)) diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java index 3c27d3ce59e4c..9e1d065c96dd6 100644 --- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java @@ -118,6 +118,7 @@ abstract class AbstractSearchAsyncAction exten private final SearchRequestContext searchRequestContext; private SearchPhase currentPhase; + private boolean currentPhaseHasLifecycle; private final List releasables = new ArrayList<>(); @@ -432,16 +433,18 @@ public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPha } private void onPhaseEnd(SearchRequestContext searchRequestContext) { - if (getCurrentPhase() != null && SearchPhaseName.isValidName(getName())) { + if (getCurrentPhase() != null) { long tookInNanos = System.nanoTime() - getCurrentPhase().getStartTimeInNanos(); searchRequestContext.updatePhaseTookMap(getCurrentPhase().getName(), TimeUnit.NANOSECONDS.toMillis(tookInNanos)); + } + if (currentPhaseHasLifecycle) { this.searchRequestContext.getSearchRequestOperationsListener().onPhaseEnd(this, searchRequestContext); } } - void onPhaseStart(SearchPhase phase) { + private void onPhaseStart(SearchPhase phase) { setCurrentPhase(phase); - if (SearchPhaseName.isValidName(phase.getName())) { + if (currentPhaseHasLifecycle) { this.searchRequestContext.getSearchRequestOperationsListener().onPhaseStart(this); } } @@ -458,6 +461,7 @@ private void executePhase(SearchPhase phase) { if (logger.isDebugEnabled()) { logger.debug(new ParameterizedMessage("Failed to execute [{}] while moving to [{}] phase", request, phase.getName()), e); } + onPhaseFailure(phase, "", e); } } @@ -637,6 +641,12 @@ public SearchPhase getCurrentPhase() { private void setCurrentPhase(SearchPhase phase) { currentPhase = phase; + // The WrappingSearchAsyncActionPhase (see please CanMatchPreFilterSearchPhase as one example) is a special case + // of search phase that wraps SearchAsyncActionPhase as SearchPhase. The AbstractSearchAsyncAction manages own + // onPhaseStart / onPhaseFailure / OnPhaseDone callbacks and the wrapping SearchPhase is being abandoned + // (fe, has no onPhaseEnd callbacks called ever). To fix that, we would not send any notifications for this + // phase. + currentPhaseHasLifecycle = ((phase instanceof WrappingSearchAsyncActionPhase) == false); } @Override @@ -716,7 +726,7 @@ public void sendSearchResponse(InternalSearchResponse internalSearchResponse, At @Override public final void onPhaseFailure(SearchPhase phase, String msg, Throwable cause) { - if (SearchPhaseName.isValidName(phase.getName())) { + if (currentPhaseHasLifecycle) { this.searchRequestContext.getSearchRequestOperationsListener().onPhaseFailure(this); } raisePhaseFailure(new SearchPhaseExecutionException(phase.getName(), msg, cause, buildShardFailures())); diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java index c6f3d4c70632d..8cf92934c8a52 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java @@ -10,9 +10,6 @@ import org.opensearch.common.annotation.PublicApi; -import java.util.HashSet; -import java.util.Set; - /** * Enum for different Search Phases in OpenSearch * @@ -28,12 +25,6 @@ public enum SearchPhaseName { CAN_MATCH("can_match"); private final String name; - private static final Set PHASE_NAMES = new HashSet<>(); - static { - for (SearchPhaseName phaseName : SearchPhaseName.values()) { - PHASE_NAMES.add(phaseName.name); - } - } SearchPhaseName(final String name) { this.name = name; @@ -42,8 +33,4 @@ public enum SearchPhaseName { public String getName() { return name; } - - public static boolean isValidName(String phaseName) { - return PHASE_NAMES.contains(phaseName); - } } diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java index 79e599ec9387b..3d1a25a8aa01f 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java @@ -1220,8 +1220,8 @@ private AbstractSearchAsyncAction searchAsyncAction timeProvider, clusterState, task, - (iter) -> { - AbstractSearchAsyncAction action = searchAsyncAction( + (iter) -> new WrappingSearchAsyncActionPhase( + searchAsyncAction( task, searchRequest, executor, @@ -1237,14 +1237,8 @@ private AbstractSearchAsyncAction searchAsyncAction threadPool, clusters, searchRequestContext - ); - return new SearchPhase("none") { - @Override - public void run() { - action.start(); - } - }; - }, + ) + ), clusters, searchRequestContext ); diff --git a/server/src/main/java/org/opensearch/action/search/WrappingSearchAsyncActionPhase.java b/server/src/main/java/org/opensearch/action/search/WrappingSearchAsyncActionPhase.java new file mode 100644 index 0000000000000..3c1ad52a1fe6a --- /dev/null +++ b/server/src/main/java/org/opensearch/action/search/WrappingSearchAsyncActionPhase.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.search; + +import org.opensearch.search.SearchPhaseResult; + +/** + * The WrappingSearchAsyncActionPhase (see please {@link CanMatchPreFilterSearchPhase} as one example) is a special case + * of search phase that wraps SearchAsyncActionPhase as {@link SearchPhase}. The {@link AbstractSearchAsyncAction} manages own + * onPhaseStart / onPhaseFailure / OnPhaseDone callbacks and but just wrapping it with the SearchPhase causes + * only some callbacks being called. The {@link AbstractSearchAsyncAction} has special treatment of {@link WrappingSearchAsyncActionPhase}. + */ +class WrappingSearchAsyncActionPhase extends SearchPhase { + private final AbstractSearchAsyncAction action; + + protected WrappingSearchAsyncActionPhase(AbstractSearchAsyncAction action) { + super(action.getName()); + this.action = action; + } + + @Override + public void run() { + action.start(); + } + + SearchPhase getSearchPhase() { + return action; + } +} diff --git a/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java index a7cbbffc51ed4..601aa9dc1856e 100644 --- a/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java @@ -85,6 +85,8 @@ import java.util.function.BiFunction; import java.util.stream.IntStream; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.instanceOf; @@ -95,6 +97,7 @@ public class AbstractSearchAsyncActionTests extends OpenSearchTestCase { private final List> resolvedNodes = new ArrayList<>(); private final Set releasedContexts = new CopyOnWriteArraySet<>(); private ExecutorService executor; + private SearchRequestOperationsListener assertingListener; ThreadPool threadPool; @Before @@ -103,6 +106,27 @@ public void setUp() throws Exception { super.setUp(); executor = Executors.newFixedThreadPool(1); threadPool = new TestThreadPool(getClass().getName()); + assertingListener = new SearchRequestOperationsListener() { + private volatile SearchPhase phase; + + @Override + protected void onPhaseStart(SearchPhaseContext context) { + assertThat(phase, is(nullValue())); + phase = context.getCurrentPhase(); + } + + @Override + protected void onPhaseEnd(SearchPhaseContext context, SearchRequestContext searchRequestContext) { + assertThat(phase, is(context.getCurrentPhase())); + phase = null; + } + + @Override + protected void onPhaseFailure(SearchPhaseContext context) { + assertThat(phase, is(context.getCurrentPhase())); + phase = null; + } + }; } @After @@ -178,7 +202,10 @@ private AbstractSearchAsyncAction createAction( results, request.getMaxConcurrentShardRequests(), SearchResponse.Clusters.EMPTY, - new SearchRequestContext(new SearchRequestOperationsListener.CompositeListener(List.of(), LogManager.getLogger()), request) + new SearchRequestContext( + new SearchRequestOperationsListener.CompositeListener(List.of(assertingListener), LogManager.getLogger()), + request + ) ) { @Override protected SearchPhase getNextPhase(final SearchPhaseResults results, SearchPhaseContext context) { @@ -334,18 +361,11 @@ public void testOnPhaseFailureAndVerifyListeners() { ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); SearchRequestStats testListener = new SearchRequestStats(clusterSettings); - final List requestOperationListeners = new ArrayList<>(List.of(testListener)); + final List requestOperationListeners = List.of(testListener); SearchQueryThenFetchAsyncAction action = createSearchQueryThenFetchAsyncAction(requestOperationListeners); action.start(); assertEquals(1, testListener.getPhaseCurrent(action.getSearchPhaseName())); - action.onPhaseFailure(new SearchPhase("none") { - @Override - public void run() { - - } - }, "message", null); - assertEquals(1, testListener.getPhaseCurrent(action.getSearchPhaseName())); - action.onPhaseFailure(new SearchPhase(action.getName()) { + action.onPhaseFailure(new SearchPhase("test") { @Override public void run() { @@ -359,14 +379,14 @@ public void run() { ); searchDfsQueryThenFetchAsyncAction.start(); assertEquals(1, testListener.getPhaseCurrent(searchDfsQueryThenFetchAsyncAction.getSearchPhaseName())); - searchDfsQueryThenFetchAsyncAction.onPhaseFailure(new SearchPhase(searchDfsQueryThenFetchAsyncAction.getName()) { + searchDfsQueryThenFetchAsyncAction.onPhaseFailure(new SearchPhase("test") { @Override public void run() { } }, "message", null); - assertEquals(0, testListener.getPhaseCurrent(searchDfsQueryThenFetchAsyncAction.getSearchPhaseName())); - assertEquals(0, testListener.getPhaseTotal(searchDfsQueryThenFetchAsyncAction.getSearchPhaseName())); + assertEquals(0, testListener.getPhaseCurrent(action.getSearchPhaseName())); + assertEquals(0, testListener.getPhaseTotal(action.getSearchPhaseName())); FetchSearchPhase fetchPhase = createFetchSearchPhase(); ShardId shardId = new ShardId(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLength(10), randomInt()); @@ -375,7 +395,7 @@ public void run() { action.skipShard(searchShardIterator); action.executeNextPhase(action, fetchPhase); assertEquals(1, testListener.getPhaseCurrent(fetchPhase.getSearchPhaseName())); - action.onPhaseFailure(new SearchPhase(fetchPhase.getName()) { + action.onPhaseFailure(new SearchPhase("test") { @Override public void run() { @@ -410,30 +430,6 @@ public void run() { assertEquals(requestIds, releasedContexts); } - public void testOnPhaseStart() { - ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - SearchRequestStats testListener = new SearchRequestStats(clusterSettings); - - final List requestOperationListeners = new ArrayList<>(List.of(testListener)); - SearchQueryThenFetchAsyncAction action = createSearchQueryThenFetchAsyncAction(requestOperationListeners); - - action.onPhaseStart(new SearchPhase("test") { - @Override - public void run() {} - }); - action.onPhaseStart(new SearchPhase("none") { - @Override - public void run() {} - }); - assertEquals(0, testListener.getPhaseCurrent(action.getSearchPhaseName())); - - action.onPhaseStart(new SearchPhase(action.getName()) { - @Override - public void run() {} - }); - assertEquals(1, testListener.getPhaseCurrent(action.getSearchPhaseName())); - } - public void testShardNotAvailableWithDisallowPartialFailures() { SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(false); AtomicReference exception = new AtomicReference<>(); diff --git a/server/src/test/java/org/opensearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/opensearch/action/search/CanMatchPreFilterSearchPhaseTests.java index 56dcf66d5607d..30fc50f91dabd 100644 --- a/server/src/test/java/org/opensearch/action/search/CanMatchPreFilterSearchPhaseTests.java +++ b/server/src/test/java/org/opensearch/action/search/CanMatchPreFilterSearchPhaseTests.java @@ -32,6 +32,7 @@ package org.opensearch.action.search; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.action.OriginalIndices; @@ -41,37 +42,84 @@ import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchService; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.dfs.DfsSearchResult; import org.opensearch.search.internal.AliasFilter; import org.opensearch.search.internal.ShardSearchRequest; import org.opensearch.search.sort.MinAndMax; import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; +import org.opensearch.test.InternalAggregationTestCase; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.Transport; +import org.junit.After; +import org.junit.Before; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.stream.IntStream; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.collection.IsEmptyCollection.empty; public class CanMatchPreFilterSearchPhaseTests extends OpenSearchTestCase { + private SearchRequestOperationsListener assertingListener; + private Set phases; + + @Before + public void setUp() throws Exception { + super.setUp(); + + phases = Collections.newSetFromMap(new IdentityHashMap<>()); + assertingListener = new SearchRequestOperationsListener() { + @Override + protected void onPhaseStart(SearchPhaseContext context) { + assertThat(phases.contains(context.getCurrentPhase()), is(false)); + phases.add(context.getCurrentPhase()); + } + + @Override + protected void onPhaseEnd(SearchPhaseContext context, SearchRequestContext searchRequestContext) { + assertThat(phases.contains(context.getCurrentPhase()), is(true)); + phases.remove(context.getCurrentPhase()); + } + + @Override + protected void onPhaseFailure(SearchPhaseContext context) { + assertThat(phases.contains(context.getCurrentPhase()), is(true)); + phases.remove(context.getCurrentPhase()); + } + }; + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + assertBusy(() -> assertThat(phases, empty()), 5, TimeUnit.SECONDS); + } public void testFilterShards() throws InterruptedException { @@ -135,11 +183,12 @@ public void sendCanMatch( public void run() throws IOException { result.set(iter); latch.countDown(); + assertingListener.onPhaseEnd(new MockSearchPhaseContext(1, searchRequest, this), null); } }, SearchResponse.Clusters.EMPTY, new SearchRequestContext( - new SearchRequestOperationsListener.CompositeListener(List.of(), LogManager.getLogger()), + new SearchRequestOperationsListener.CompositeListener(List.of(assertingListener), LogManager.getLogger()), searchRequest ) ); @@ -230,11 +279,12 @@ public void sendCanMatch( public void run() throws IOException { result.set(iter); latch.countDown(); + assertingListener.onPhaseEnd(new MockSearchPhaseContext(1, searchRequest, this), null); } }, SearchResponse.Clusters.EMPTY, new SearchRequestContext( - new SearchRequestOperationsListener.CompositeListener(List.of(), LogManager.getLogger()), + new SearchRequestOperationsListener.CompositeListener(List.of(assertingListener), LogManager.getLogger()), searchRequest ) ); @@ -366,6 +416,7 @@ protected void executePhaseOnShard( canMatchPhase.start(); latch.await(); + executor.shutdown(); } @@ -443,17 +494,19 @@ public void sendCanMatch( public void run() { result.set(iter); latch.countDown(); + assertingListener.onPhaseEnd(new MockSearchPhaseContext(1, searchRequest, this), null); } }, SearchResponse.Clusters.EMPTY, new SearchRequestContext( - new SearchRequestOperationsListener.CompositeListener(List.of(), LogManager.getLogger()), + new SearchRequestOperationsListener.CompositeListener(List.of(assertingListener), LogManager.getLogger()), searchRequest ) ); canMatchPhase.start(); latch.await(); + ShardId[] expected = IntStream.range(0, shardIds.size()) .boxed() .sorted(Comparator.comparing(minAndMaxes::get, MinAndMax.getComparator(order)).thenComparing(shardIds::get)) @@ -546,17 +599,19 @@ public void sendCanMatch( public void run() { result.set(iter); latch.countDown(); + assertingListener.onPhaseEnd(new MockSearchPhaseContext(1, searchRequest, this), null); } }, SearchResponse.Clusters.EMPTY, new SearchRequestContext( - new SearchRequestOperationsListener.CompositeListener(List.of(), LogManager.getLogger()), + new SearchRequestOperationsListener.CompositeListener(List.of(assertingListener), LogManager.getLogger()), searchRequest ) ); canMatchPhase.start(); latch.await(); + int shardId = 0; for (SearchShardIterator i : result.get()) { assertThat(i.shardId().id(), equalTo(shardId++)); @@ -565,4 +620,190 @@ public void run() { assertThat(result.get().size(), equalTo(numShards)); } } + + public void testAsyncAction() throws InterruptedException { + + final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider( + 0, + System.nanoTime(), + System::nanoTime + ); + + Map lookup = new ConcurrentHashMap<>(); + DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNode replicaNode = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT); + lookup.put("node_1", new SearchAsyncActionTests.MockConnection(primaryNode)); + lookup.put("node_2", new SearchAsyncActionTests.MockConnection(replicaNode)); + final boolean shard1 = randomBoolean(); + final boolean shard2 = randomBoolean(); + + SearchTransportService searchTransportService = new SearchTransportService(null, null) { + @Override + public void sendCanMatch( + Transport.Connection connection, + ShardSearchRequest request, + SearchTask task, + ActionListener listener + ) { + new Thread( + () -> listener.onResponse(new SearchService.CanMatchResponse(request.shardId().id() == 0 ? shard1 : shard2, null)) + ).start(); + } + }; + + AtomicReference> result = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + GroupShardsIterator shardsIter = SearchAsyncActionTests.getShardsIter( + "idx", + new OriginalIndices(new String[] { "idx" }, SearchRequest.DEFAULT_INDICES_OPTIONS), + 2, + randomBoolean(), + primaryNode, + replicaNode + ); + final SearchRequest searchRequest = new SearchRequest(); + searchRequest.allowPartialSearchResults(true); + + SearchTask task = new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()); + ExecutorService executor = OpenSearchExecutors.newDirectExecutorService(); + SearchRequestContext searchRequestContext = new SearchRequestContext( + new SearchRequestOperationsListener.CompositeListener(List.of(assertingListener), LogManager.getLogger()), + searchRequest + ); + + SearchPhaseController controller = new SearchPhaseController( + writableRegistry(), + r -> InternalAggregationTestCase.emptyReduceContextBuilder() + ); + + QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer( + searchRequest, + executor, + new NoopCircuitBreaker(CircuitBreaker.REQUEST), + controller, + task.getProgressListener(), + writableRegistry(), + shardsIter.size(), + exc -> {} + ); + + CanMatchPreFilterSearchPhase canMatchPhase = new CanMatchPreFilterSearchPhase( + logger, + searchTransportService, + (clusterAlias, node) -> lookup.get(node), + Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)), + Collections.emptyMap(), + Collections.emptyMap(), + executor, + searchRequest, + null, + shardsIter, + timeProvider, + ClusterState.EMPTY_STATE, + null, + (iter) -> { + AbstractSearchAsyncAction action = new SearchDfsQueryAsyncAction( + logger, + searchTransportService, + (clusterAlias, node) -> lookup.get(node), + Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)), + Collections.emptyMap(), + Collections.emptyMap(), + controller, + executor, + resultConsumer, + searchRequest, + null, + shardsIter, + timeProvider, + ClusterState.EMPTY_STATE, + task, + SearchResponse.Clusters.EMPTY, + searchRequestContext + ); + return new WrappingSearchAsyncActionPhase(action) { + @Override + public void run() { + super.run(); + latch.countDown(); + } + }; + }, + SearchResponse.Clusters.EMPTY, + searchRequestContext + ); + + canMatchPhase.start(); + latch.await(); + + assertThat(result.get(), is(nullValue())); + } + + private static final class SearchDfsQueryAsyncAction extends AbstractSearchAsyncAction { + private final SearchRequestOperationsListener listener; + + SearchDfsQueryAsyncAction( + final Logger logger, + final SearchTransportService searchTransportService, + final BiFunction nodeIdToConnection, + final Map aliasFilter, + final Map concreteIndexBoosts, + final Map> indexRoutings, + final SearchPhaseController searchPhaseController, + final Executor executor, + final QueryPhaseResultConsumer queryPhaseResultConsumer, + final SearchRequest request, + final ActionListener listener, + final GroupShardsIterator shardsIts, + final TransportSearchAction.SearchTimeProvider timeProvider, + final ClusterState clusterState, + final SearchTask task, + SearchResponse.Clusters clusters, + SearchRequestContext searchRequestContext + ) { + super( + SearchPhaseName.DFS_PRE_QUERY.getName(), + logger, + searchTransportService, + nodeIdToConnection, + aliasFilter, + concreteIndexBoosts, + indexRoutings, + executor, + request, + listener, + shardsIts, + timeProvider, + clusterState, + task, + new ArraySearchPhaseResults<>(shardsIts.size()), + request.getMaxConcurrentShardRequests(), + clusters, + searchRequestContext + ); + this.listener = searchRequestContext.getSearchRequestOperationsListener(); + } + + @Override + protected void executePhaseOnShard( + final SearchShardIterator shardIt, + final SearchShardTarget shard, + final SearchActionListener listener + ) { + final DfsSearchResult response = new DfsSearchResult(shardIt.getSearchContextId(), shard, null); + response.setShardIndex(shard.getShardId().getId()); + listener.innerOnResponse(response); + } + + @Override + protected SearchPhase getNextPhase(SearchPhaseResults results, SearchPhaseContext context) { + return new SearchPhase("last") { + @Override + public void run() throws IOException { + listener.onPhaseEnd(context, null); + } + }; + } + } + } diff --git a/server/src/test/java/org/opensearch/action/search/MockSearchPhaseContext.java b/server/src/test/java/org/opensearch/action/search/MockSearchPhaseContext.java index 04a00a09dcbc4..cc10da8fc1f12 100644 --- a/server/src/test/java/org/opensearch/action/search/MockSearchPhaseContext.java +++ b/server/src/test/java/org/opensearch/action/search/MockSearchPhaseContext.java @@ -67,17 +67,27 @@ public final class MockSearchPhaseContext implements SearchPhaseContext { final Set releasedSearchContexts = new HashSet<>(); final SearchRequest searchRequest; final AtomicReference searchResponse = new AtomicReference<>(); + final SearchPhase currentPhase; public MockSearchPhaseContext(int numShards) { this(numShards, new SearchRequest()); } public MockSearchPhaseContext(int numShards, SearchRequest searchRequest) { + this(numShards, searchRequest, null); + } + + public MockSearchPhaseContext(int numShards, SearchRequest searchRequest, SearchPhase currentPhase) { this.numShards = numShards; this.searchRequest = searchRequest; + this.currentPhase = currentPhase; numSuccess = new AtomicInteger(numShards); } + public MockSearchPhaseContext(int numShards, SearchPhase currentPhase) { + this(numShards, new SearchRequest(), currentPhase); + } + public void assertNoFailure() { if (phaseFailure.get() != null) { throw new AssertionError(phaseFailure.get()); @@ -106,7 +116,7 @@ public SearchRequest getRequest() { @Override public SearchPhase getCurrentPhase() { - return null; + return currentPhase; } @Override From 181b9854a6bac16cf93dff54fa2c3a48ea76906d Mon Sep 17 00:00:00 2001 From: Poojita Raj Date: Fri, 16 Feb 2024 16:44:49 -0800 Subject: [PATCH 11/41] [Segment Replication] Enable test suites to run with segrep (#12307) * Enable test suites to run with segrep Signed-off-by: Poojita Raj * Refactor static settings into ParameterizedStaticSettingsOpenSearchIntegTestCase Signed-off-by: Poojita Raj --------- Signed-off-by: Poojita Raj --- .../opensearch/indexing/IndexActionIT.java | 18 +++++++++++-- .../org/opensearch/ingest/IngestClientIT.java | 14 +++++++++- ...gestProcessorNotInstalledOnAllNodesIT.java | 20 ++++++++++---- .../recovery/FullRollingRestartIT.java | 24 +++++++++++++---- .../recovery/RecoveryWhileUnderLoadIT.java | 7 +---- .../org/opensearch/recovery/RelocationIT.java | 26 ++++++++++++++----- .../opensearch/recovery/SimpleRecoveryIT.java | 21 ++++++++++++--- .../recovery/TruncatedRecoveryIT.java | 14 +++++++++- ...StaticSettingsOpenSearchIntegTestCase.java | 11 ++++++++ 9 files changed, 124 insertions(+), 31 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indexing/IndexActionIT.java b/server/src/internalClusterTest/java/org/opensearch/indexing/IndexActionIT.java index 73d6d9aff7b72..8cb54631b593f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indexing/IndexActionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indexing/IndexActionIT.java @@ -32,19 +32,23 @@ package org.opensearch.indexing; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.DocWriteResponse; import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.metadata.MetadataCreateIndexService; +import org.opensearch.common.settings.Settings; import org.opensearch.index.VersionType; import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.indices.InvalidIndexNameException; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; import org.opensearch.test.hamcrest.OpenSearchAssertions; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Locale; import java.util.Random; @@ -57,7 +61,17 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; -public class IndexActionIT extends OpenSearchIntegTestCase { +public class IndexActionIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { + + public IndexActionIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return replicationSettings; + } + /** * This test tries to simulate load while creating an index and indexing documents * while the index is being created. diff --git a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java index e2cedea331412..9481a6116cdbc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java @@ -32,6 +32,8 @@ package org.opensearch.ingest; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchParseException; @@ -57,6 +59,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; import java.util.Arrays; import java.util.Collection; @@ -73,7 +76,16 @@ import static org.hamcrest.core.Is.is; @OpenSearchIntegTestCase.ClusterScope(minNumDataNodes = 2) -public class IngestClientIT extends OpenSearchIntegTestCase { +public class IngestClientIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { + + public IngestClientIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return replicationSettings; + } @Override protected Settings nodeSettings(int nodeOrdinal) { diff --git a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java index 38f1375bc7504..4c949e11459ab 100644 --- a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java @@ -32,13 +32,17 @@ package org.opensearch.ingest; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchParseException; import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.settings.Settings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.node.NodeService; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; import java.io.IOException; import java.util.Arrays; @@ -52,12 +56,10 @@ import static org.hamcrest.Matchers.notNullValue; @OpenSearchIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0, scope = OpenSearchIntegTestCase.Scope.TEST) -public class IngestProcessorNotInstalledOnAllNodesIT extends OpenSearchIntegTestCase { +public class IngestProcessorNotInstalledOnAllNodesIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { - private final BytesReference pipelineSource; - private volatile boolean installPlugin; - - public IngestProcessorNotInstalledOnAllNodesIT() throws IOException { + public IngestProcessorNotInstalledOnAllNodesIT(Settings settings) throws IOException { + super(settings); pipelineSource = BytesReference.bytes( jsonBuilder().startObject() .startArray("processors") @@ -70,6 +72,14 @@ public IngestProcessorNotInstalledOnAllNodesIT() throws IOException { ); } + @ParametersFactory + public static Collection parameters() { + return replicationSettings; + } + + private final BytesReference pipelineSource; + private volatile boolean installPlugin; + @Override protected Collection> nodePlugins() { return installPlugin ? Arrays.asList(IngestTestPlugin.class) : Collections.emptyList(); diff --git a/server/src/internalClusterTest/java/org/opensearch/recovery/FullRollingRestartIT.java b/server/src/internalClusterTest/java/org/opensearch/recovery/FullRollingRestartIT.java index d28df90216beb..0752ab7c9d0f1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/recovery/FullRollingRestartIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/recovery/FullRollingRestartIT.java @@ -32,6 +32,8 @@ package org.opensearch.recovery; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.indices.recovery.RecoveryResponse; @@ -44,15 +46,27 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.indices.recovery.RecoveryState; -import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; + +import java.util.Collection; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) -public class FullRollingRestartIT extends OpenSearchIntegTestCase { +public class FullRollingRestartIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { + + public FullRollingRestartIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return replicationSettings; + } + protected void assertTimeout(ClusterHealthRequestBuilder requestBuilder) { ClusterHealthResponse clusterHealth = requestBuilder.get(); if (clusterHealth.isTimedOut()) { @@ -121,7 +135,7 @@ public void testFullRollingRestart() throws Exception { ); logger.info("--> refreshing and checking data"); - refresh(); + refreshAndWaitForReplication(); for (int i = 0; i < 10; i++) { assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 2000L); } @@ -154,7 +168,7 @@ public void testFullRollingRestart() throws Exception { ); logger.info("--> stopped two nodes, verifying data"); - refresh(); + refreshAndWaitForReplication(); for (int i = 0; i < 10; i++) { assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 2000L); } @@ -188,7 +202,7 @@ public void testFullRollingRestart() throws Exception { ); logger.info("--> one node left, verifying data"); - refresh(); + refreshAndWaitForReplication(); for (int i = 0; i < 10; i++) { assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 2000L); } diff --git a/server/src/internalClusterTest/java/org/opensearch/recovery/RecoveryWhileUnderLoadIT.java b/server/src/internalClusterTest/java/org/opensearch/recovery/RecoveryWhileUnderLoadIT.java index eb293aeb6d490..988aeac7da541 100644 --- a/server/src/internalClusterTest/java/org/opensearch/recovery/RecoveryWhileUnderLoadIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/recovery/RecoveryWhileUnderLoadIT.java @@ -54,7 +54,6 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.DocsStats; import org.opensearch.index.translog.Translog; -import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.plugins.Plugin; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.BackgroundIndexer; @@ -72,7 +71,6 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; -import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAllSuccessful; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @@ -86,10 +84,7 @@ public RecoveryWhileUnderLoadIT(Settings settings) { @ParametersFactory public static Collection parameters() { - return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT).build() }, - new Object[] { Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT).build() } - ); + return replicationSettings; } private final Logger logger = LogManager.getLogger(RecoveryWhileUnderLoadIT.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/recovery/RelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/recovery/RelocationIT.java index 8c69424939b57..8d8aea7fc1f3b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/recovery/RelocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/recovery/RelocationIT.java @@ -32,6 +32,8 @@ package org.opensearch.recovery; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.tests.util.English; import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; @@ -73,9 +75,9 @@ import org.opensearch.test.BackgroundIndexer; import org.opensearch.test.InternalSettingsPlugin; import org.opensearch.test.MockIndexEventListener; -import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.test.transport.StubbableTransport; import org.opensearch.transport.Transport; @@ -114,7 +116,17 @@ import static org.hamcrest.Matchers.startsWith; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) -public class RelocationIT extends OpenSearchIntegTestCase { +public class RelocationIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { + + public RelocationIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return replicationSettings; + } + private final TimeValue ACCEPTABLE_RELOCATION_TIME = new TimeValue(5, TimeUnit.MINUTES); @Override @@ -158,7 +170,7 @@ public void testSimpleRelocationNoIndexing() { } logger.info("--> verifying count"); - client().admin().indices().prepareRefresh().execute().actionGet(); + refreshAndWaitForReplication(); assertThat(client().prepareSearch("test").setSize(0).execute().actionGet().getHits().getTotalHits().value, equalTo(20L)); logger.info("--> start another node"); @@ -186,7 +198,7 @@ public void testSimpleRelocationNoIndexing() { assertThat(clusterHealthResponse.isTimedOut(), equalTo(false)); logger.info("--> verifying count again..."); - client().admin().indices().prepareRefresh().execute().actionGet(); + refreshAndWaitForReplication(); assertThat(client().prepareSearch("test").setSize(0).execute().actionGet().getHits().getTotalHits().value, equalTo(20L)); } @@ -265,7 +277,7 @@ public void testRelocationWhileIndexingRandom() throws Exception { logger.info("--> indexing threads stopped"); logger.info("--> refreshing the index"); - client().admin().indices().prepareRefresh("test").execute().actionGet(); + refreshAndWaitForReplication("test"); logger.info("--> searching the index"); boolean ranOnce = false; for (int i = 0; i < 10; i++) { @@ -650,7 +662,7 @@ public void testRelocateWhileWaitingForRefresh() { assertThat(clusterHealthResponse.isTimedOut(), equalTo(false)); logger.info("--> verifying count"); - client().admin().indices().prepareRefresh().execute().actionGet(); + refreshAndWaitForReplication(); assertThat(client().prepareSearch("test").setSize(0).execute().actionGet().getHits().getTotalHits().value, equalTo(20L)); } @@ -726,7 +738,7 @@ public void testRelocateWhileContinuouslyIndexingAndWaitingForRefresh() throws E logger.info("--> verifying count"); assertBusy(() -> { - client().admin().indices().prepareRefresh().execute().actionGet(); + refreshAndWaitForReplication(); assertTrue(pendingIndexResponses.stream().allMatch(ActionFuture::isDone)); }, 1, TimeUnit.MINUTES); diff --git a/server/src/internalClusterTest/java/org/opensearch/recovery/SimpleRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/recovery/SimpleRecoveryIT.java index 85f90738b19ce..1f5fbae6e58e9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/recovery/SimpleRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/recovery/SimpleRecoveryIT.java @@ -32,21 +32,34 @@ package org.opensearch.recovery; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.indices.flush.FlushResponse; import org.opensearch.action.admin.indices.refresh.RefreshResponse; import org.opensearch.action.get.GetResponse; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.MediaTypeRegistry; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; + +import java.util.Collection; import static org.opensearch.client.Requests.flushRequest; import static org.opensearch.client.Requests.getRequest; import static org.opensearch.client.Requests.indexRequest; -import static org.opensearch.client.Requests.refreshRequest; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; -public class SimpleRecoveryIT extends OpenSearchIntegTestCase { +public class SimpleRecoveryIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { + + public SimpleRecoveryIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return replicationSettings; + } + @Override public Settings indexSettings() { return Settings.builder().put(super.indexSettings()).put(recoverySettings()).build(); @@ -72,7 +85,7 @@ public void testSimpleRecovery() throws Exception { assertThat(flushResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries)); assertThat(flushResponse.getFailedShards(), equalTo(0)); client().index(indexRequest("test").id("2").source(source("2", "test"), MediaTypeRegistry.JSON)).actionGet(); - RefreshResponse refreshResponse = client().admin().indices().refresh(refreshRequest("test")).actionGet(); + RefreshResponse refreshResponse = refreshAndWaitForReplication("test"); assertThat(refreshResponse.getTotalShards(), equalTo(numShards.totalNumShards)); assertThat(refreshResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries)); assertThat(refreshResponse.getFailedShards(), equalTo(0)); diff --git a/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java index 5f0922615a557..bf0533143cf91 100644 --- a/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java @@ -32,6 +32,8 @@ package org.opensearch.recovery; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.tests.util.English; import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs; import org.opensearch.action.admin.cluster.node.stats.NodeStats; @@ -47,6 +49,7 @@ import org.opensearch.node.RecoverySettingsChunkSizePlugin; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.transport.TransportService; @@ -65,7 +68,16 @@ @OpenSearchIntegTestCase.ClusterScope(numDataNodes = 2, numClientNodes = 0, scope = OpenSearchIntegTestCase.Scope.TEST) @SuppressCodecs("*") // test relies on exact file extensions -public class TruncatedRecoveryIT extends OpenSearchIntegTestCase { +public class TruncatedRecoveryIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { + + public TruncatedRecoveryIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return replicationSettings; + } @Override protected Collection> nodePlugins() { diff --git a/test/framework/src/main/java/org/opensearch/test/ParameterizedStaticSettingsOpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/ParameterizedStaticSettingsOpenSearchIntegTestCase.java index 2e97228326314..defcedfac1c76 100644 --- a/test/framework/src/main/java/org/opensearch/test/ParameterizedStaticSettingsOpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/ParameterizedStaticSettingsOpenSearchIntegTestCase.java @@ -9,9 +9,14 @@ package org.opensearch.test; import org.opensearch.common.settings.Settings; +import org.opensearch.indices.replication.common.ReplicationType; +import java.util.Arrays; +import java.util.List; import java.util.Objects; +import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; + /** * Base class for running the tests with parameterization with static settings: the cluster will be pre-created with the settings at startup, the method * {@link #hasSameParametersAs(ParameterizedOpenSearchIntegTestCase)} is being used by the test scaffolding to detect when the test suite is instantiated with @@ -24,6 +29,12 @@ * */ public abstract class ParameterizedStaticSettingsOpenSearchIntegTestCase extends ParameterizedOpenSearchIntegTestCase { + + public static final List replicationSettings = Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT).build() }, + new Object[] { Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT).build() } + ); + public ParameterizedStaticSettingsOpenSearchIntegTestCase(Settings nodeSettings) { super(nodeSettings); } From 5d327a2f48f4c0fcd8a443fa6675dad8b18b5cb5 Mon Sep 17 00:00:00 2001 From: Thomas Seidl Date: Sat, 17 Feb 2024 05:26:46 +0100 Subject: [PATCH 12/41] Fix the "highlight.max_analyzer_offset" request parameter with "plain" highlighter (#10919) --- CHANGELOG.md | 1 + .../search.highlight/30_max_analyzed_offset.yml | 12 ++++++++++++ .../subphase/highlight/PlainHighlighter.java | 16 +++++++++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60e68edbfffbf..6055b55f83738 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -120,6 +120,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add support of special WrappingSearchAsyncActionPhase so the onPhaseStart() will always be followed by onPhaseEnd() within AbstractSearchAsyncAction ([#12293](https://github.com/opensearch-project/OpenSearch/pull/12293)) - Add a system property to configure YamlParser codepoint limits ([#12298](https://github.com/opensearch-project/OpenSearch/pull/12298)) - Prevent read beyond slice boundary in ByteArrayIndexInput ([#10481](https://github.com/opensearch-project/OpenSearch/issues/10481)) +- Fix the "highlight.max_analyzer_offset" request parameter with "plain" highlighter ([#10919](https://github.com/opensearch-project/OpenSearch/pull/10919)) ### Security diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml index a18ac45e62175..4ee905972d106 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml @@ -78,3 +78,15 @@ setup: index: test1 body: {"query" : {"match" : {"field2" : "fox"}}, "highlight" : {"type" : "plain", "fields" : {"field2" : {}}}} - match: { error.root_cause.0.type: "illegal_argument_exception" } + +--- +"Plain highlighter on a field WITHOUT OFFSETS using max_analyzer_offset should SUCCEED": + - skip: + version: " - 2.1.99" + reason: only starting supporting the parameter max_analyzer_offset on version 2.2 + - do: + search: + rest_total_hits_as_int: true + index: test1 + body: {"query" : {"match" : {"field1" : "quick"}}, "highlight" : {"type" : "plain", "fields" : {"field1" : {"max_analyzer_offset": 10}}}} + - match: {hits.hits.0.highlight.field1.0: "The quick "} diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/PlainHighlighter.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/PlainHighlighter.java index eb5f4f3c14eb2..c06a733203434 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/PlainHighlighter.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/PlainHighlighter.java @@ -123,13 +123,27 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc List textsToHighlight; Analyzer analyzer = context.mapperService().documentMapper().mappers().indexAnalyzer(); final int maxAnalyzedOffset = context.getIndexSettings().getHighlightMaxAnalyzedOffset(); + final Integer fieldMaxAnalyzedOffset = field.fieldOptions().maxAnalyzerOffset(); + if (fieldMaxAnalyzedOffset != null && fieldMaxAnalyzedOffset > maxAnalyzedOffset) { + throw new IllegalArgumentException( + "max_analyzer_offset has exceeded [" + + maxAnalyzedOffset + + "] - maximum allowed to be analyzed for highlighting. " + + "This maximum can be set by changing the [" + + IndexSettings.MAX_ANALYZED_OFFSET_SETTING.getKey() + + "] index level setting. " + + "For large texts, indexing with offsets or term vectors is recommended!" + ); + } textsToHighlight = HighlightUtils.loadFieldValues(fieldType, context.getQueryShardContext(), hitContext, fieldContext.forceSource); for (Object textToHighlight : textsToHighlight) { String text = convertFieldValue(fieldType, textToHighlight); int textLength = text.length(); - if (textLength > maxAnalyzedOffset) { + if (fieldMaxAnalyzedOffset != null && textLength > fieldMaxAnalyzedOffset) { + text = text.substring(0, fieldMaxAnalyzedOffset); + } else if (textLength > maxAnalyzedOffset) { throw new IllegalArgumentException( "The length of [" + fieldContext.fieldName From f86bfa855867e24ff047494089b40eb46318aace Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Sun, 18 Feb 2024 16:45:41 -0800 Subject: [PATCH 13/41] Avoid string out of bounds error in snapshot delete (#12337) * Avoid string out of bounds error in snapshot delete Test failure #8771 shows cases where certain random seeds trigger this case. The bug is clear: the substring() call should happen after the startsWith() check in case the blob name is shorter than the prefix length being used as the start index of the substring call. I don't yet know if/how this manifests in real deployments. Signed-off-by: Andrew Ross * Extract common UUID parsing method Signed-off-by: Andrew Ross --------- Signed-off-by: Andrew Ross --- .../blobstore/BlobStoreRepository.java | 92 +++++++++---------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 8a2260e1f6d90..18f4ab70024f4 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -1116,10 +1116,7 @@ private void executeStaleShardDelete( String indexId = fileToDeletePath[1]; String shardId = fileToDeletePath[2]; String shallowSnapBlob = fileToDeletePath[3]; - String snapshotUUID = shallowSnapBlob.substring( - SHALLOW_SNAPSHOT_PREFIX.length(), - shallowSnapBlob.length() - ".dat".length() - ); + String snapshotUUID = extractShallowSnapshotUUID(shallowSnapBlob).orElseThrow(); BlobContainer shardContainer = blobStore().blobContainer(indicesPath().add(indexId).add(shardId)); RemoteStoreShardShallowCopySnapshot remoteStoreShardShallowCopySnapshot = REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.read( @@ -1586,44 +1583,43 @@ private void executeOneStaleIndexDelete( try { logger.debug("[{}] Found stale index [{}]. Cleaning it up", metadata.name(), indexSnId); if (remoteStoreLockManagerFactory != null) { - Map shardBlobs = indexEntry.getValue().children(); - if (!shardBlobs.isEmpty()) { - for (Map.Entry shardBlob : shardBlobs.entrySet()) { - Map shardLevelBlobs = shardBlob.getValue().listBlobs(); - for (Map.Entry shardLevelBlob : shardLevelBlobs.entrySet()) { - String blob = shardLevelBlob.getKey(); - String snapshotUUID = blob.substring(SHALLOW_SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()); - if (blob.startsWith(SHALLOW_SNAPSHOT_PREFIX) && blob.endsWith(".dat")) { - RemoteStoreShardShallowCopySnapshot remoteStoreShardShallowCopySnapshot = - REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.read( - shardBlob.getValue(), - snapshotUUID, - namedXContentRegistry - ); - String indexUUID = remoteStoreShardShallowCopySnapshot.getIndexUUID(); - String remoteStoreRepoForIndex = remoteStoreShardShallowCopySnapshot.getRemoteStoreRepository(); - // Releasing lock files before deleting the shallow-snap-UUID file because in case of any failure - // while releasing the lock file, we would still have the corresponding shallow-snap-UUID file - // and that would be used during next delete operation for releasing this stale lock file - RemoteStoreLockManager remoteStoreMetadataLockManager = remoteStoreLockManagerFactory - .newLockManager(remoteStoreRepoForIndex, indexUUID, shardBlob.getKey()); - remoteStoreMetadataLockManager.release( - FileLockInfo.getLockInfoBuilder().withAcquirerId(snapshotUUID).build() + final Map shardBlobs = indexEntry.getValue().children(); + for (Map.Entry shardBlob : shardBlobs.entrySet()) { + for (String blob : shardBlob.getValue().listBlobs().keySet()) { + final Optional snapshotUUID = extractShallowSnapshotUUID(blob); + if (snapshotUUID.isPresent()) { + RemoteStoreShardShallowCopySnapshot remoteStoreShardShallowCopySnapshot = + REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.read( + shardBlob.getValue(), + snapshotUUID.get(), + namedXContentRegistry ); - if (!isIndexPresent(clusterService, indexUUID)) { - // this is a temporary solution where snapshot deletion triggers remote store side - // cleanup if index is already deleted. We will add a poller in future to take - // care of remote store side cleanup. - // see https://github.com/opensearch-project/OpenSearch/issues/8469 - new RemoteSegmentStoreDirectoryFactory( - remoteStoreLockManagerFactory.getRepositoriesService(), - threadPool - ).newDirectory( - remoteStoreRepoForIndex, - indexUUID, - new ShardId(Index.UNKNOWN_INDEX_NAME, indexUUID, Integer.valueOf(shardBlob.getKey())) - ).close(); - } + String indexUUID = remoteStoreShardShallowCopySnapshot.getIndexUUID(); + String remoteStoreRepoForIndex = remoteStoreShardShallowCopySnapshot.getRemoteStoreRepository(); + // Releasing lock files before deleting the shallow-snap-UUID file because in case of any failure + // while releasing the lock file, we would still have the corresponding shallow-snap-UUID file + // and that would be used during next delete operation for releasing this stale lock file + RemoteStoreLockManager remoteStoreMetadataLockManager = remoteStoreLockManagerFactory.newLockManager( + remoteStoreRepoForIndex, + indexUUID, + shardBlob.getKey() + ); + remoteStoreMetadataLockManager.release( + FileLockInfo.getLockInfoBuilder().withAcquirerId(snapshotUUID.get()).build() + ); + if (!isIndexPresent(clusterService, indexUUID)) { + // this is a temporary solution where snapshot deletion triggers remote store side + // cleanup if index is already deleted. We will add a poller in future to take + // care of remote store side cleanup. + // see https://github.com/opensearch-project/OpenSearch/issues/8469 + new RemoteSegmentStoreDirectoryFactory( + remoteStoreLockManagerFactory.getRepositoriesService(), + threadPool + ).newDirectory( + remoteStoreRepoForIndex, + indexUUID, + new ShardId(Index.UNKNOWN_INDEX_NAME, indexUUID, Integer.parseInt(shardBlob.getKey())) + ).close(); } } } @@ -3362,12 +3358,7 @@ private static List unusedBlobs( blob.substring(SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()) ) == false) || (remoteStoreLockManagerFactory != null - ? (blob.startsWith(SHALLOW_SNAPSHOT_PREFIX) - && blob.endsWith(".dat") - && survivingSnapshotUUIDs.contains( - blob.substring(SHALLOW_SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()) - ) == false) - : false) + && extractShallowSnapshotUUID(blob).map(survivingSnapshotUUIDs::contains).orElse(false)) || (blob.startsWith(UPLOADED_DATA_BLOB_PREFIX) && updatedSnapshots.findNameFile(canonicalName(blob)) == null) || FsBlobContainer.isTempBlobName(blob) ) @@ -3509,6 +3500,13 @@ private static void failStoreIfCorrupted(Store store, Exception e) { } } + private static Optional extractShallowSnapshotUUID(String blobName) { + if (blobName.startsWith(SHALLOW_SNAPSHOT_PREFIX)) { + return Optional.of(blobName.substring(SHALLOW_SNAPSHOT_PREFIX.length(), blobName.length() - ".dat".length())); + } + return Optional.empty(); + } + /** * The result of removing a snapshot from a shard folder in the repository. */ From 75d64e2bf3da15636c2b22753f6f99560aa77b82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 10:49:07 -0500 Subject: [PATCH 14/41] Bump gradle/wrapper-validation-action from 1 to 2 (#12367) * Bump gradle/wrapper-validation-action from 1 to 2 Bumps [gradle/wrapper-validation-action](https://github.com/gradle/wrapper-validation-action) from 1 to 2. - [Release notes](https://github.com/gradle/wrapper-validation-action/releases) - [Commits](https://github.com/gradle/wrapper-validation-action/compare/v1...v2) --- updated-dependencies: - dependency-name: gradle/wrapper-validation-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/wrapper.yml | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wrapper.yml b/.github/workflows/wrapper.yml index 6dd48ca15eaa9..dcf2a09717d28 100644 --- a/.github/workflows/wrapper.yml +++ b/.github/workflows/wrapper.yml @@ -8,4 +8,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: gradle/wrapper-validation-action@v1 + - uses: gradle/wrapper-validation-action@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6055b55f83738..f158eafdf7c54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,6 +106,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `peter-evans/find-comment` from 2 to 3 ([#12288](https://github.com/opensearch-project/OpenSearch/pull/12288)) - Bump `com.google.api.grpc:proto-google-common-protos` from 2.25.1 to 2.33.0 ([#12289](https://github.com/opensearch-project/OpenSearch/pull/12289)) - Bump `com.squareup.okio:okio` from 3.7.0 to 3.8.0 ([#12290](https://github.com/opensearch-project/OpenSearch/pull/12290)) +- Bump `gradle/wrapper-validation-action` from 1 to 2 ([#12367](https://github.com/opensearch-project/OpenSearch/pull/12367)) ### Changed - Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499)) From f5e1c4a207d07e3618f4954a438a3d10b2b09cda Mon Sep 17 00:00:00 2001 From: rishavz_sagar Date: Tue, 20 Feb 2024 02:26:59 +0530 Subject: [PATCH 15/41] Fixing FSHealthy tests (#12366) Signed-off-by: RS146BIJAY --- .../java/org/opensearch/monitor/fs/FsHealthServiceTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java b/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java index a42c302b516d2..48b2941fe3b7e 100644 --- a/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java +++ b/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java @@ -185,7 +185,7 @@ public void testLoggingOnHungIO() throws Exception { } public void testFailsHealthOnHungIOBeyondHealthyTimeout() throws Exception { - long healthyTimeoutThreshold = randomLongBetween(500, 1000); + long healthyTimeoutThreshold = randomLongBetween(1500, 2000); long refreshInterval = randomLongBetween(500, 1000); long slowLogThreshold = randomLongBetween(100, 200); long delayBetweenChecks = 100; From 43a2ce95675fef788683c22624eb53fa7b7da4a4 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 19 Feb 2024 16:25:08 -0500 Subject: [PATCH 16/41] Bump netty from 4.1.106.Final to 4.1.107.Final (#12372) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- .../licenses/netty-buffer-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-codec-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-codec-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-common-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-common-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.107.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.106.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-codec-dns-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-codec-dns-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-codec-socks-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-codec-socks-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-handler-proxy-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-handler-proxy-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-resolver-dns-4.1.107.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.106.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.107.Final.jar.sha1 | 1 + .../repository-hdfs/licenses/netty-all-4.1.106.Final.jar.sha1 | 1 - .../repository-hdfs/licenses/netty-all-4.1.107.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-buffer-4.1.106.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-buffer-4.1.107.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-codec-4.1.106.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-codec-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.107.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-common-4.1.106.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-common-4.1.107.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-handler-4.1.106.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-handler-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.107.Final.jar.sha1 | 1 + .../netty-transport-classes-epoll-4.1.106.Final.jar.sha1 | 1 - .../netty-transport-classes-epoll-4.1.107.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.106.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.107.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-buffer-4.1.106.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-buffer-4.1.107.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-codec-4.1.106.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-codec-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.107.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-common-4.1.106.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-common-4.1.107.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-handler-4.1.106.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-handler-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-buffer-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-codec-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-codec-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-codec-dns-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-codec-dns-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-common-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-common-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-resolver-dns-4.1.107.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.106.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.107.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.106.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.107.Final.jar.sha1 | 1 + 90 files changed, 46 insertions(+), 45 deletions(-) delete mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-common-4.1.107.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-buffer-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-buffer-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-common-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-common-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-handler-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-handler-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-resolver-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-resolver-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.107.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-common-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-common-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-common-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index f158eafdf7c54..0ad26225bb9dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.google.api.grpc:proto-google-common-protos` from 2.25.1 to 2.33.0 ([#12289](https://github.com/opensearch-project/OpenSearch/pull/12289)) - Bump `com.squareup.okio:okio` from 3.7.0 to 3.8.0 ([#12290](https://github.com/opensearch-project/OpenSearch/pull/12290)) - Bump `gradle/wrapper-validation-action` from 1 to 2 ([#12367](https://github.com/opensearch-project/OpenSearch/pull/12367)) +- Bump `netty` from 4.1.106.Final to 4.1.107.Final ([#12372](https://github.com/opensearch-project/OpenSearch/pull/12372)) ### Changed - Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index c7af58ab6b0b9..5f5b07a8955fc 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -26,7 +26,7 @@ jakarta_annotation = 1.3.5 # when updating the JNA version, also update the version in buildSrc/build.gradle jna = 5.13.0 -netty = 4.1.106.Final +netty = 4.1.107.Final joda = 2.12.2 # project reactor diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 deleted file mode 100644 index 1f170375e9347..0000000000000 --- a/modules/transport-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c5495ba59a627641b3a7c23f6bcb801874c7f7b0 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..beb44fc0f4cf9 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +8509a72b8a5a2d33d611e99254aed39765c3ad82 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 deleted file mode 100644 index a75ea81b7ee03..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -025171b63aa1e7a5fd8a7e4e660d6d3110241ea7 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..4c74bb06fd83b --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +0a1d32debf2ed07c5852ab5b2904c43adb76c39e \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 deleted file mode 100644 index 7f5d1adbff740..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -21a07cdf0fc46b313fe2248f1275cdbdac0ba87b \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..38eb2e5bad80a --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +04d8e9e51b7254bd26a42fe17bdcae32e4c6ebb3 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 deleted file mode 100644 index c4a0c1fae8e24..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -41515e8c51eeaaddceabdb4f86fbc5dbfc25b70e \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..5b3d3311edc9f --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +3885ffe7dd05c9773df70c61009f34a5a8a383ec \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 deleted file mode 100644 index bda8b9376e992..0000000000000 --- a/modules/transport-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -132defb4689f41b51b483b7202b22b6e89fe35fd \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..bbe91c6ccfb1d --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +4f17a547530d64becd7179507b25f4154bcfba57 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 deleted file mode 100644 index 749cc807bcce2..0000000000000 --- a/modules/transport-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -874c970c4ff958b1140dde52bc17e6a9e7cde662 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..ba27b38632622 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +d4c6b05f4d9aca117981297fb7f02953102ebb5e \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 deleted file mode 100644 index bd160c07ad0ff..0000000000000 --- a/modules/transport-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e185ae573db04939215f94d6ba869758dcecbde9 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..3bc0f7b3fed09 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +dfee84308341a42131dd0f8ac0e1e02d627c19f3 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 deleted file mode 100644 index 2dab4bff2cc0e..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8fcca44ae16b98e15965093e7696832019fd6f27 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..19419999300dd --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +d6a105c621b47d1410e0e09419d7209d2d46e914 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 deleted file mode 100644 index b4b977fdad7de..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2da179bd95903f0fa73218b8f0d02690c0cfbc94 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..407ecaffdad30 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +4d61d4959741109b3eccd7337f11fc89fa90a74a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 deleted file mode 100644 index 005cc2388bd89..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9d4b3315bb625ec2d73fa569fb6bce4589243d5e \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.107.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..d823de7ffadd4 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-dns-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +945e8ad5ab7ec4f11fb0257d2594af0cfae1d4b7 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 deleted file mode 100644 index c4a0c1fae8e24..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -41515e8c51eeaaddceabdb4f86fbc5dbfc25b70e \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..5b3d3311edc9f --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +3885ffe7dd05c9773df70c61009f34a5a8a383ec \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.106.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.106.Final.jar.sha1 deleted file mode 100644 index 4fc3f5c43edb0..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-socks-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -129f9bd6aa0cf28bfd7d45a8b6a598eed9c67702 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.107.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..114d77a1bb95f --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-socks-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +3b7070e9acfe262bb0bd936c4051116631796b3b \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.106.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.106.Final.jar.sha1 deleted file mode 100644 index 3dcfa7f26db06..0000000000000 --- a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bffc88b7c56d7b553453e4244247a1b0ba1fdc8a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.107.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..5a4bde479eb38 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +ebc495e9b2bc2c9ab60a264b40f62dc0671d9f6e \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 deleted file mode 100644 index 0617c6999c586..0000000000000 --- a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3970474ed55aa1e3e8de5a5602c342c6b8155371 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..a62cb0fefcc40 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +d655d09e972dee46f580dbcf41c0d1356aea9e1b \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 deleted file mode 100644 index b4b977fdad7de..0000000000000 --- a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2da179bd95903f0fa73218b8f0d02690c0cfbc94 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..407ecaffdad30 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +4d61d4959741109b3eccd7337f11fc89fa90a74a \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.106.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.106.Final.jar.sha1 deleted file mode 100644 index f9affd7887093..0000000000000 --- a/plugins/repository-hdfs/licenses/netty-all-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6ec5bd2be65d5529e58e9e482b747c1135b3736b \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.107.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..0e3595fecb0d2 --- /dev/null +++ b/plugins/repository-hdfs/licenses/netty-all-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +3019703b67413ef3d6150da1f49753f4010507ce \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.106.Final.jar.sha1 deleted file mode 100644 index 1f170375e9347..0000000000000 --- a/plugins/repository-s3/licenses/netty-buffer-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c5495ba59a627641b3a7c23f6bcb801874c7f7b0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..beb44fc0f4cf9 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-buffer-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +8509a72b8a5a2d33d611e99254aed39765c3ad82 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.106.Final.jar.sha1 deleted file mode 100644 index a75ea81b7ee03..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -025171b63aa1e7a5fd8a7e4e660d6d3110241ea7 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..4c74bb06fd83b --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +0a1d32debf2ed07c5852ab5b2904c43adb76c39e \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.106.Final.jar.sha1 deleted file mode 100644 index 7f5d1adbff740..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -21a07cdf0fc46b313fe2248f1275cdbdac0ba87b \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..38eb2e5bad80a --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +04d8e9e51b7254bd26a42fe17bdcae32e4c6ebb3 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 deleted file mode 100644 index c4a0c1fae8e24..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -41515e8c51eeaaddceabdb4f86fbc5dbfc25b70e \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..5b3d3311edc9f --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +3885ffe7dd05c9773df70c61009f34a5a8a383ec \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.106.Final.jar.sha1 deleted file mode 100644 index bda8b9376e992..0000000000000 --- a/plugins/repository-s3/licenses/netty-common-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -132defb4689f41b51b483b7202b22b6e89fe35fd \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..bbe91c6ccfb1d --- /dev/null +++ b/plugins/repository-s3/licenses/netty-common-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +4f17a547530d64becd7179507b25f4154bcfba57 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.106.Final.jar.sha1 deleted file mode 100644 index 749cc807bcce2..0000000000000 --- a/plugins/repository-s3/licenses/netty-handler-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -874c970c4ff958b1140dde52bc17e6a9e7cde662 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..ba27b38632622 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-handler-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +d4c6b05f4d9aca117981297fb7f02953102ebb5e \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.106.Final.jar.sha1 deleted file mode 100644 index bd160c07ad0ff..0000000000000 --- a/plugins/repository-s3/licenses/netty-resolver-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e185ae573db04939215f94d6ba869758dcecbde9 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..3bc0f7b3fed09 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-resolver-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +dfee84308341a42131dd0f8ac0e1e02d627c19f3 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.106.Final.jar.sha1 deleted file mode 100644 index 2dab4bff2cc0e..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8fcca44ae16b98e15965093e7696832019fd6f27 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..19419999300dd --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +d6a105c621b47d1410e0e09419d7209d2d46e914 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.106.Final.jar.sha1 deleted file mode 100644 index 58b668b7e80a9..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c058d5c712e00e8560e519970b3d27747778b8f2 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..6b9a35acb2c20 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +9234407d6a46745599735765c4d3755c7fc84162 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 deleted file mode 100644 index b4b977fdad7de..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2da179bd95903f0fa73218b8f0d02690c0cfbc94 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..407ecaffdad30 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +4d61d4959741109b3eccd7337f11fc89fa90a74a \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.106.Final.jar.sha1 deleted file mode 100644 index 1f170375e9347..0000000000000 --- a/plugins/transport-nio/licenses/netty-buffer-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c5495ba59a627641b3a7c23f6bcb801874c7f7b0 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..beb44fc0f4cf9 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-buffer-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +8509a72b8a5a2d33d611e99254aed39765c3ad82 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.106.Final.jar.sha1 deleted file mode 100644 index a75ea81b7ee03..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -025171b63aa1e7a5fd8a7e4e660d6d3110241ea7 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..4c74bb06fd83b --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +0a1d32debf2ed07c5852ab5b2904c43adb76c39e \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.106.Final.jar.sha1 deleted file mode 100644 index 7f5d1adbff740..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-http-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -21a07cdf0fc46b313fe2248f1275cdbdac0ba87b \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..38eb2e5bad80a --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +04d8e9e51b7254bd26a42fe17bdcae32e4c6ebb3 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.106.Final.jar.sha1 deleted file mode 100644 index bda8b9376e992..0000000000000 --- a/plugins/transport-nio/licenses/netty-common-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -132defb4689f41b51b483b7202b22b6e89fe35fd \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..bbe91c6ccfb1d --- /dev/null +++ b/plugins/transport-nio/licenses/netty-common-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +4f17a547530d64becd7179507b25f4154bcfba57 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.106.Final.jar.sha1 deleted file mode 100644 index 749cc807bcce2..0000000000000 --- a/plugins/transport-nio/licenses/netty-handler-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -874c970c4ff958b1140dde52bc17e6a9e7cde662 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..ba27b38632622 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-handler-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +d4c6b05f4d9aca117981297fb7f02953102ebb5e \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.106.Final.jar.sha1 deleted file mode 100644 index bd160c07ad0ff..0000000000000 --- a/plugins/transport-nio/licenses/netty-resolver-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e185ae573db04939215f94d6ba869758dcecbde9 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..3bc0f7b3fed09 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-resolver-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +dfee84308341a42131dd0f8ac0e1e02d627c19f3 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.106.Final.jar.sha1 deleted file mode 100644 index 2dab4bff2cc0e..0000000000000 --- a/plugins/transport-nio/licenses/netty-transport-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8fcca44ae16b98e15965093e7696832019fd6f27 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..19419999300dd --- /dev/null +++ b/plugins/transport-nio/licenses/netty-transport-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +d6a105c621b47d1410e0e09419d7209d2d46e914 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 deleted file mode 100644 index 1f170375e9347..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c5495ba59a627641b3a7c23f6bcb801874c7f7b0 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..beb44fc0f4cf9 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +8509a72b8a5a2d33d611e99254aed39765c3ad82 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 deleted file mode 100644 index a75ea81b7ee03..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -025171b63aa1e7a5fd8a7e4e660d6d3110241ea7 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..4c74bb06fd83b --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +0a1d32debf2ed07c5852ab5b2904c43adb76c39e \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 deleted file mode 100644 index 005cc2388bd89..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9d4b3315bb625ec2d73fa569fb6bce4589243d5e \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..d823de7ffadd4 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +945e8ad5ab7ec4f11fb0257d2594af0cfae1d4b7 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 deleted file mode 100644 index 7f5d1adbff740..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -21a07cdf0fc46b313fe2248f1275cdbdac0ba87b \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..38eb2e5bad80a --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +04d8e9e51b7254bd26a42fe17bdcae32e4c6ebb3 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 deleted file mode 100644 index c4a0c1fae8e24..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -41515e8c51eeaaddceabdb4f86fbc5dbfc25b70e \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..5b3d3311edc9f --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +3885ffe7dd05c9773df70c61009f34a5a8a383ec \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 deleted file mode 100644 index bda8b9376e992..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -132defb4689f41b51b483b7202b22b6e89fe35fd \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..bbe91c6ccfb1d --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +4f17a547530d64becd7179507b25f4154bcfba57 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 deleted file mode 100644 index 749cc807bcce2..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -874c970c4ff958b1140dde52bc17e6a9e7cde662 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..ba27b38632622 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +d4c6b05f4d9aca117981297fb7f02953102ebb5e \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 deleted file mode 100644 index bd160c07ad0ff..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e185ae573db04939215f94d6ba869758dcecbde9 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..3bc0f7b3fed09 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +dfee84308341a42131dd0f8ac0e1e02d627c19f3 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 deleted file mode 100644 index 0617c6999c586..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3970474ed55aa1e3e8de5a5602c342c6b8155371 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..a62cb0fefcc40 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +d655d09e972dee46f580dbcf41c0d1356aea9e1b \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 deleted file mode 100644 index 2dab4bff2cc0e..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8fcca44ae16b98e15965093e7696832019fd6f27 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..19419999300dd --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +d6a105c621b47d1410e0e09419d7209d2d46e914 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 deleted file mode 100644 index b4b977fdad7de..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2da179bd95903f0fa73218b8f0d02690c0cfbc94 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 new file mode 100644 index 0000000000000..407ecaffdad30 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 @@ -0,0 +1 @@ +4d61d4959741109b3eccd7337f11fc89fa90a74a \ No newline at end of file From 94f79a2deebfde28e4cc12d6bd5a83ca207633c2 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Mon, 19 Feb 2024 17:47:27 -0800 Subject: [PATCH 17/41] Mute DeleteSnapshotIT.testDeleteMultipleShallowCopySnapshotsCase1 (#12373) Signed-off-by: Andrew Ross --- .../java/org/opensearch/snapshots/DeleteSnapshotIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java index e79bf1c16b586..73feeb84308ab 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java @@ -92,6 +92,7 @@ public void testDeleteShallowCopySnapshot() throws Exception { } // Deleting multiple shallow copy snapshots as part of single delete call with repo having only shallow copy snapshots. + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9208") public void testDeleteMultipleShallowCopySnapshotsCase1() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); final Path remoteStoreRepoPath = randomRepoPath(); From 69c08c8b87eba3d5fd25882ca5571f46e628d061 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 20 Feb 2024 10:04:18 -0600 Subject: [PATCH 18/41] Bump `peternied/check-pull-request-description-checklist` from v1 to v1.1 (#12387) Signed-off-by: Peter Nied Signed-off-by: Peter Nied --- .github/workflows/pull-request-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 11998e36c2dbb..7efcf529588ed 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -17,7 +17,7 @@ jobs: name: Verify Description Checklist runs-on: ubuntu-latest steps: - - uses: peternied/check-pull-request-description-checklist@v1 + - uses: peternied/check-pull-request-description-checklist@v1.1 with: checklist-items: | New functionality includes testing. From 10a6bc713a8161b1ac5fe18103f906d8812d9f27 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 20 Feb 2024 12:47:47 -0500 Subject: [PATCH 19/41] Bump opentelemetry from 1.34.1 to 1.35.0 (#12388) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- .../telemetry-otel/licenses/opentelemetry-api-1.34.1.jar.sha1 | 1 - .../telemetry-otel/licenses/opentelemetry-api-1.35.0.jar.sha1 | 1 + .../licenses/opentelemetry-context-1.34.1.jar.sha1 | 1 - .../licenses/opentelemetry-context-1.35.0.jar.sha1 | 1 + .../licenses/opentelemetry-exporter-common-1.34.1.jar.sha1 | 1 - .../licenses/opentelemetry-exporter-common-1.35.0.jar.sha1 | 1 + .../licenses/opentelemetry-exporter-logging-1.34.1.jar.sha1 | 1 - .../licenses/opentelemetry-exporter-logging-1.35.0.jar.sha1 | 1 + .../licenses/opentelemetry-exporter-otlp-1.34.1.jar.sha1 | 1 - .../licenses/opentelemetry-exporter-otlp-1.35.0.jar.sha1 | 1 + .../licenses/opentelemetry-exporter-otlp-common-1.34.1.jar.sha1 | 1 - .../licenses/opentelemetry-exporter-otlp-common-1.35.0.jar.sha1 | 1 + .../opentelemetry-exporter-sender-okhttp-1.34.1.jar.sha1 | 1 - .../opentelemetry-exporter-sender-okhttp-1.35.0.jar.sha1 | 1 + .../opentelemetry-extension-incubator-1.34.1-alpha.jar.sha1 | 1 - .../opentelemetry-extension-incubator-1.35.0-alpha.jar.sha1 | 1 + .../telemetry-otel/licenses/opentelemetry-sdk-1.34.1.jar.sha1 | 1 - .../telemetry-otel/licenses/opentelemetry-sdk-1.35.0.jar.sha1 | 1 + .../licenses/opentelemetry-sdk-common-1.34.1.jar.sha1 | 1 - .../licenses/opentelemetry-sdk-common-1.35.0.jar.sha1 | 1 + .../licenses/opentelemetry-sdk-logs-1.34.1.jar.sha1 | 1 - .../licenses/opentelemetry-sdk-logs-1.35.0.jar.sha1 | 1 + .../licenses/opentelemetry-sdk-metrics-1.34.1.jar.sha1 | 1 - .../licenses/opentelemetry-sdk-metrics-1.35.0.jar.sha1 | 1 + .../licenses/opentelemetry-sdk-trace-1.34.1.jar.sha1 | 1 - .../licenses/opentelemetry-sdk-trace-1.35.0.jar.sha1 | 1 + 28 files changed, 15 insertions(+), 14 deletions(-) delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-1.34.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-1.35.0.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-1.34.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-1.35.0.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.34.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.35.0.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.34.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.35.0.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.34.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.35.0.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.34.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.35.0.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.34.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.35.0.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.34.1-alpha.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.35.0-alpha.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-1.34.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-1.35.0.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.34.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.35.0.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.34.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.35.0.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.34.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.35.0.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.34.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.35.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ad26225bb9dd..92d7d87aeddf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,6 +108,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.squareup.okio:okio` from 3.7.0 to 3.8.0 ([#12290](https://github.com/opensearch-project/OpenSearch/pull/12290)) - Bump `gradle/wrapper-validation-action` from 1 to 2 ([#12367](https://github.com/opensearch-project/OpenSearch/pull/12367)) - Bump `netty` from 4.1.106.Final to 4.1.107.Final ([#12372](https://github.com/opensearch-project/OpenSearch/pull/12372)) +- Bump `opentelemetry` from 1.34.1 to 1.35.0 ([#12388](https://github.com/opensearch-project/OpenSearch/pull/12388)) ### Changed - Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 5f5b07a8955fc..7da7bedf9b09c 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -70,5 +70,5 @@ jzlib = 1.1.3 resteasy = 6.2.4.Final # opentelemetry dependencies -opentelemetry = 1.34.1 +opentelemetry = 1.35.0 opentelemetrysemconv = 1.23.1-alpha diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-1.34.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-1.34.1.jar.sha1 deleted file mode 100644 index 19f734ca17b79..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-api-1.34.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b4aea155f6d6b1032eba85378564431cfd86f562 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-1.35.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-1.35.0.jar.sha1 new file mode 100644 index 0000000000000..e81b44b9e057f --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-api-1.35.0.jar.sha1 @@ -0,0 +1 @@ +388c49986bc20f3b4bea58470eb16decd230c2db \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-1.34.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-context-1.34.1.jar.sha1 deleted file mode 100644 index 4c06d28cba199..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-context-1.34.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3fcc87f3d810ce49d865ee54b40831559c5e129b \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-1.35.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-context-1.35.0.jar.sha1 new file mode 100644 index 0000000000000..0054417ef7b30 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-context-1.35.0.jar.sha1 @@ -0,0 +1 @@ +6e2aa0e28c5069121cf11b2c93225942358f1423 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.34.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.34.1.jar.sha1 deleted file mode 100644 index 91a5c0f715d2b..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.34.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19c9a3f52851a1333b648ed83c82d16eb4c64afd \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.35.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.35.0.jar.sha1 new file mode 100644 index 0000000000000..27f96d15d6a70 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.35.0.jar.sha1 @@ -0,0 +1 @@ +d0784bf59b74a2dc369551cc6d200e243ce8cca0 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.34.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.34.1.jar.sha1 deleted file mode 100644 index 6c05600ae3b08..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.34.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b3e74d5b8cf5e60d9965042fa284085bbe081ce3 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.35.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.35.0.jar.sha1 new file mode 100644 index 0000000000000..dc21252f19d11 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.35.0.jar.sha1 @@ -0,0 +1 @@ +5f038bc2b9a1f415c8f74c4a35e0d92fae64c430 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.34.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.34.1.jar.sha1 deleted file mode 100644 index f54e6f6893050..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.34.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -af68f90f0410b7b3a1900d3e0a15ad51b10ffd5b \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.35.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.35.0.jar.sha1 new file mode 100644 index 0000000000000..088f2475af00f --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.35.0.jar.sha1 @@ -0,0 +1 @@ +1a7baebfbc6c569163bc74a5add9819cc411d582 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.34.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.34.1.jar.sha1 deleted file mode 100644 index 49d40b36ba85b..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.34.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4acab18052267e280d1f9de22c591a5c88bed3a6 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.35.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.35.0.jar.sha1 new file mode 100644 index 0000000000000..438e431f1a7d4 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.35.0.jar.sha1 @@ -0,0 +1 @@ +5506cb34a43fb733564a2aee47763d34cada9a7a \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.34.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.34.1.jar.sha1 deleted file mode 100644 index a01de2aa84c43..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.34.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9f07e1764389e076a36fb7d9e5769e29f3dab950 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.35.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.35.0.jar.sha1 new file mode 100644 index 0000000000000..7fb0f09d60c0f --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.35.0.jar.sha1 @@ -0,0 +1 @@ +d5abbce20bf88dff97b9ec7104bf13d163042f30 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.34.1-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.34.1-alpha.jar.sha1 deleted file mode 100644 index a5fc8c2059104..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.34.1-alpha.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9201e6a43a0a89515626f7516c7d1b2c349f76df \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.35.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.35.0-alpha.jar.sha1 new file mode 100644 index 0000000000000..8ec097d471e16 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.35.0-alpha.jar.sha1 @@ -0,0 +1 @@ +72720d7880110d02aad6d69066cc0311c568f17d \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.34.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.34.1.jar.sha1 deleted file mode 100644 index cd746f0756e46..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.34.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ab49eb621d6d01f0ad2f016989d0352ef18ea9a2 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.35.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.35.0.jar.sha1 new file mode 100644 index 0000000000000..7c9200f50e438 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.35.0.jar.sha1 @@ -0,0 +1 @@ +82fe6a227fb3148aae2e61978cf77f7005a66bca \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.34.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.34.1.jar.sha1 deleted file mode 100644 index 740737dc13efc..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.34.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -01fcd8bad38d7b8987f6fc93bd7e933240eb727e \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.35.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.35.0.jar.sha1 new file mode 100644 index 0000000000000..9fd80da7597c2 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.35.0.jar.sha1 @@ -0,0 +1 @@ +5d2bc29d8f2ef2cf5a2239ac6990a2c89118456d \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.34.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.34.1.jar.sha1 deleted file mode 100644 index e6ff3dbafda22..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.34.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -abad9abc80dfe6118a60413afa161696bbf8dd43 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.35.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.35.0.jar.sha1 new file mode 100644 index 0000000000000..d4dc7528c83d6 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.35.0.jar.sha1 @@ -0,0 +1 @@ +da2122051bd95d3a36bf34f72f1b0dd9b105fd1f \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.34.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.34.1.jar.sha1 deleted file mode 100644 index 36ec960c4f7be..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.34.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d88407ae475e5f4e859a81e4f61e362e939f7bc2 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.35.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.35.0.jar.sha1 new file mode 100644 index 0000000000000..7e1b206d42ba4 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.35.0.jar.sha1 @@ -0,0 +1 @@ +97942849d51081e766a29646175b752bb79d7ce0 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.34.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.34.1.jar.sha1 deleted file mode 100644 index 293b82f206c99..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.34.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -121a75c2ba9ed8b80f5ff131c2411a5d460f38d0 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.35.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.35.0.jar.sha1 new file mode 100644 index 0000000000000..47125e70aa884 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.35.0.jar.sha1 @@ -0,0 +1 @@ +861b6a3c43a15ca3782f1fa17b024b9afa4b3ea6 \ No newline at end of file From 9b7541f756ac56036916219f38c1987ee57d7aa7 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 20 Feb 2024 14:48:53 -0500 Subject: [PATCH 20/41] Bump Apache Lucene from 9.9.2 to 9.10.0 (#12392) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- libs/core/licenses/lucene-core-9.10.0.jar.sha1 | 1 + libs/core/licenses/lucene-core-9.9.2.jar.sha1 | 1 - libs/core/src/main/java/org/opensearch/Version.java | 2 +- .../licenses/lucene-expressions-9.10.0.jar.sha1 | 1 + .../lang-expression/licenses/lucene-expressions-9.9.2.jar.sha1 | 1 - .../analysis-icu/licenses/lucene-analysis-icu-9.10.0.jar.sha1 | 1 + .../analysis-icu/licenses/lucene-analysis-icu-9.9.2.jar.sha1 | 1 - .../licenses/lucene-analysis-kuromoji-9.10.0.jar.sha1 | 1 + .../licenses/lucene-analysis-kuromoji-9.9.2.jar.sha1 | 1 - .../licenses/lucene-analysis-nori-9.10.0.jar.sha1 | 1 + .../analysis-nori/licenses/lucene-analysis-nori-9.9.2.jar.sha1 | 1 - .../licenses/lucene-analysis-phonetic-9.10.0.jar.sha1 | 1 + .../licenses/lucene-analysis-phonetic-9.9.2.jar.sha1 | 1 - .../licenses/lucene-analysis-smartcn-9.10.0.jar.sha1 | 1 + .../licenses/lucene-analysis-smartcn-9.9.2.jar.sha1 | 1 - .../licenses/lucene-analysis-stempel-9.10.0.jar.sha1 | 1 + .../licenses/lucene-analysis-stempel-9.9.2.jar.sha1 | 1 - .../licenses/lucene-analysis-morfologik-9.10.0.jar.sha1 | 1 + .../licenses/lucene-analysis-morfologik-9.9.2.jar.sha1 | 1 - server/licenses/lucene-analysis-common-9.10.0.jar.sha1 | 1 + server/licenses/lucene-analysis-common-9.9.2.jar.sha1 | 1 - server/licenses/lucene-backward-codecs-9.10.0.jar.sha1 | 1 + server/licenses/lucene-backward-codecs-9.9.2.jar.sha1 | 1 - server/licenses/lucene-core-9.10.0.jar.sha1 | 1 + server/licenses/lucene-core-9.9.2.jar.sha1 | 1 - server/licenses/lucene-grouping-9.10.0.jar.sha1 | 1 + server/licenses/lucene-grouping-9.9.2.jar.sha1 | 1 - server/licenses/lucene-highlighter-9.10.0.jar.sha1 | 1 + server/licenses/lucene-highlighter-9.9.2.jar.sha1 | 1 - server/licenses/lucene-join-9.10.0.jar.sha1 | 1 + server/licenses/lucene-join-9.9.2.jar.sha1 | 1 - server/licenses/lucene-memory-9.10.0.jar.sha1 | 1 + server/licenses/lucene-memory-9.9.2.jar.sha1 | 1 - server/licenses/lucene-misc-9.10.0.jar.sha1 | 1 + server/licenses/lucene-misc-9.9.2.jar.sha1 | 1 - server/licenses/lucene-queries-9.10.0.jar.sha1 | 1 + server/licenses/lucene-queries-9.9.2.jar.sha1 | 1 - server/licenses/lucene-queryparser-9.10.0.jar.sha1 | 1 + server/licenses/lucene-queryparser-9.9.2.jar.sha1 | 1 - server/licenses/lucene-sandbox-9.10.0.jar.sha1 | 1 + server/licenses/lucene-sandbox-9.9.2.jar.sha1 | 1 - server/licenses/lucene-spatial-extras-9.10.0.jar.sha1 | 1 + server/licenses/lucene-spatial-extras-9.9.2.jar.sha1 | 1 - server/licenses/lucene-spatial3d-9.10.0.jar.sha1 | 1 + server/licenses/lucene-spatial3d-9.9.2.jar.sha1 | 1 - server/licenses/lucene-suggest-9.10.0.jar.sha1 | 1 + server/licenses/lucene-suggest-9.9.2.jar.sha1 | 1 - .../java/org/opensearch/index/engine/TranslogLeafReader.java | 3 +++ .../main/java/org/opensearch/index/get/ShardGetService.java | 1 + .../org/opensearch/search/lookup/LeafFieldsLookupTests.java | 1 + 52 files changed, 31 insertions(+), 25 deletions(-) create mode 100644 libs/core/licenses/lucene-core-9.10.0.jar.sha1 delete mode 100644 libs/core/licenses/lucene-core-9.9.2.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.10.0.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.9.2.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.10.0.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.2.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.10.0.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.2.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.10.0.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.2.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.10.0.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.2.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.10.0.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.2.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.10.0.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.2.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.10.0.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-core-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-join-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.9.2.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.10.0.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.9.2.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 92d7d87aeddf3..3dc9fac006716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `gradle/wrapper-validation-action` from 1 to 2 ([#12367](https://github.com/opensearch-project/OpenSearch/pull/12367)) - Bump `netty` from 4.1.106.Final to 4.1.107.Final ([#12372](https://github.com/opensearch-project/OpenSearch/pull/12372)) - Bump `opentelemetry` from 1.34.1 to 1.35.0 ([#12388](https://github.com/opensearch-project/OpenSearch/pull/12388)) +- Bump Apache Lucene from 9.9.2 to 9.10.0 ([#12392](https://github.com/opensearch-project/OpenSearch/pull/12392)) ### Changed - Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 7da7bedf9b09c..7fc74aff552be 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.9.2 +lucene = 9.10.0 bundled_jdk_vendor = adoptium bundled_jdk = 21.0.2+13 diff --git a/libs/core/licenses/lucene-core-9.10.0.jar.sha1 b/libs/core/licenses/lucene-core-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..31b6bcd29f418 --- /dev/null +++ b/libs/core/licenses/lucene-core-9.10.0.jar.sha1 @@ -0,0 +1 @@ +64e5624754d59386be5d9159c68f81ff96298704 \ No newline at end of file diff --git a/libs/core/licenses/lucene-core-9.9.2.jar.sha1 b/libs/core/licenses/lucene-core-9.9.2.jar.sha1 deleted file mode 100644 index 2d03d69369b9f..0000000000000 --- a/libs/core/licenses/lucene-core-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7699f80220fc80b08413902560904623b88beb9f \ No newline at end of file diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 5038bb90cc18d..2971c4683192f 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -100,7 +100,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_11_2 = new Version(2110299, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_9_2); public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_9_2); - public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_9_2); + public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_10_0); public static final Version CURRENT = V_3_0_0; public static Version fromId(int id) { diff --git a/modules/lang-expression/licenses/lucene-expressions-9.10.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..4c85d77f41cd6 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.10.0.jar.sha1 @@ -0,0 +1 @@ +e2daf2e568fe120e84567505f7c85643bb0b9dd2 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.9.2.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.9.2.jar.sha1 deleted file mode 100644 index e073455415e24..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -045df3828b6460c032a6551040e31ea432b0aad6 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.10.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..a6f523b96685c --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.10.0.jar.sha1 @@ -0,0 +1 @@ +ed54097d13dbaae6e758c31f55a473ba6566ae8d \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.2.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.2.jar.sha1 deleted file mode 100644 index b318a2d89db7d..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f7e549fdac07140f4cd379a0f517c38434165e95 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.10.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..0572d78c75516 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.10.0.jar.sha1 @@ -0,0 +1 @@ +a158d8a7d386c782a10ec15827ff3f0aa6fd8291 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.2.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.2.jar.sha1 deleted file mode 100644 index 87510efa881bc..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -eb19738fd3ca85071fef96978a056a7c94d8e793 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.10.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..b81b33d7b28f7 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.10.0.jar.sha1 @@ -0,0 +1 @@ +79eb4c5e1c92b613341d05173dfcdce182fa5f80 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.2.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.2.jar.sha1 deleted file mode 100644 index fb3746eb27840..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7e83a83741155ececf9193a4f967e570e170236d \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.10.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..66245cc1d36d9 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.10.0.jar.sha1 @@ -0,0 +1 @@ +b6ba8f0ecee48b94911c08ee12c20f059ec38ec5 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.2.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.2.jar.sha1 deleted file mode 100644 index e88299f106bb2..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dabdea96c7a6d00363b0093a580e7d783efa69a4 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.10.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..5f17e786f91d5 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.10.0.jar.sha1 @@ -0,0 +1 @@ +ccf4512816ba9cd5c2cca9182db78a2e48e7e16f \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.2.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.2.jar.sha1 deleted file mode 100644 index 05dd3328b032d..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fb551d9ae6a1bf12bc90a4d26dd8fa3eefac8cb9 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.10.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..f7b19bc3b38cd --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.10.0.jar.sha1 @@ -0,0 +1 @@ +b596817e70f00bc54d66e82c118f47ca91b9e401 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.2.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.2.jar.sha1 deleted file mode 100644 index b47a1e4052407..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0d495b478861b2d29e0a58d273ca0e6e755292e9 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.10.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..7fb543858c1a7 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.10.0.jar.sha1 @@ -0,0 +1 @@ +7acc74d35684269ffa74909395044e0e02581285 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.2.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.2.jar.sha1 deleted file mode 100644 index b7d54cb230445..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9958d813d4cfdd890b4611c679ed36775480fa0d \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.10.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..d33a965b3506c --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.10.0.jar.sha1 @@ -0,0 +1 @@ +92e559808a23f61c818ef90a9ccab3669a25caa0 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.9.2.jar.sha1 b/server/licenses/lucene-analysis-common-9.9.2.jar.sha1 deleted file mode 100644 index 6eefe1b002fde..0000000000000 --- a/server/licenses/lucene-analysis-common-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2a4712ee9d87e0f9942a81195dbd5223dbeaa541 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.10.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..01b6cbb54f3e9 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.10.0.jar.sha1 @@ -0,0 +1 @@ +6570ebf974d07025ad4cd9ffaa9927546b534704 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.9.2.jar.sha1 b/server/licenses/lucene-backward-codecs-9.9.2.jar.sha1 deleted file mode 100644 index 377acc22a1d6b..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b53707366cf2891bb301f85a44c5108fc2d5d1b5 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.10.0.jar.sha1 b/server/licenses/lucene-core-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..31b6bcd29f418 --- /dev/null +++ b/server/licenses/lucene-core-9.10.0.jar.sha1 @@ -0,0 +1 @@ +64e5624754d59386be5d9159c68f81ff96298704 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.9.2.jar.sha1 b/server/licenses/lucene-core-9.9.2.jar.sha1 deleted file mode 100644 index 2d03d69369b9f..0000000000000 --- a/server/licenses/lucene-core-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7699f80220fc80b08413902560904623b88beb9f \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.10.0.jar.sha1 b/server/licenses/lucene-grouping-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..21f2062273355 --- /dev/null +++ b/server/licenses/lucene-grouping-9.10.0.jar.sha1 @@ -0,0 +1 @@ +b82838299e4fa423fc574c1c07405410823f74ce \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.9.2.jar.sha1 b/server/licenses/lucene-grouping-9.9.2.jar.sha1 deleted file mode 100644 index 1e321d751bff4..0000000000000 --- a/server/licenses/lucene-grouping-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -72b6c47fd80933d259859d38325e3e020c8e017b \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.10.0.jar.sha1 b/server/licenses/lucene-highlighter-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..2148a8ef957f0 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.10.0.jar.sha1 @@ -0,0 +1 @@ +2f21ade4b4896f1ece2b3a823e1640c762c9d0cf \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.9.2.jar.sha1 b/server/licenses/lucene-highlighter-9.9.2.jar.sha1 deleted file mode 100644 index d5c2e2a5504f2..0000000000000 --- a/server/licenses/lucene-highlighter-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0968d74d6794af65798819304f10ceb947080332 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.10.0.jar.sha1 b/server/licenses/lucene-join-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..93b8f50522800 --- /dev/null +++ b/server/licenses/lucene-join-9.10.0.jar.sha1 @@ -0,0 +1 @@ +541b722bb842817e2ec899cd4d0de2a136953f53 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.9.2.jar.sha1 b/server/licenses/lucene-join-9.9.2.jar.sha1 deleted file mode 100644 index 60da109fb5c83..0000000000000 --- a/server/licenses/lucene-join-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -902e4b65686e29c1489a7cdf43ed2ce3b7fe8ace \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.10.0.jar.sha1 b/server/licenses/lucene-memory-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..82ca74ccdfc42 --- /dev/null +++ b/server/licenses/lucene-memory-9.10.0.jar.sha1 @@ -0,0 +1 @@ +06620029b6c7d84480fb7ad1ba0d72963111b765 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.9.2.jar.sha1 b/server/licenses/lucene-memory-9.9.2.jar.sha1 deleted file mode 100644 index bebb36d7ed684..0000000000000 --- a/server/licenses/lucene-memory-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -08175beb4135c23e6918c822381cd51cd2e118a8 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.10.0.jar.sha1 b/server/licenses/lucene-misc-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..2d69a4cfa1148 --- /dev/null +++ b/server/licenses/lucene-misc-9.10.0.jar.sha1 @@ -0,0 +1 @@ +3621206c534d253a0205f8a0293c146ff34d8b03 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.9.2.jar.sha1 b/server/licenses/lucene-misc-9.9.2.jar.sha1 deleted file mode 100644 index 15f572ec715bf..0000000000000 --- a/server/licenses/lucene-misc-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -39b015aa603db42b55f0833193bdfc75d38842ba \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.10.0.jar.sha1 b/server/licenses/lucene-queries-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..d7c38977e4fc8 --- /dev/null +++ b/server/licenses/lucene-queries-9.10.0.jar.sha1 @@ -0,0 +1 @@ +bbe2360c60a43bf41fbf05d5f0b4f471547217ba \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.9.2.jar.sha1 b/server/licenses/lucene-queries-9.9.2.jar.sha1 deleted file mode 100644 index 9a94e40b69e85..0000000000000 --- a/server/licenses/lucene-queries-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -671252594e15280818f56d4b369bc48c0ab00dac \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.10.0.jar.sha1 b/server/licenses/lucene-queryparser-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..b02a3cd926375 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.10.0.jar.sha1 @@ -0,0 +1 @@ +c50f82d244ea5adac2d2d9295de85ddccc2d45cb \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.9.2.jar.sha1 b/server/licenses/lucene-queryparser-9.9.2.jar.sha1 deleted file mode 100644 index f23292999ae6a..0000000000000 --- a/server/licenses/lucene-queryparser-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7cf8dea63ab511ea9a322916198c3abd0402c8b2 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.10.0.jar.sha1 b/server/licenses/lucene-sandbox-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..74b9e925398b1 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.10.0.jar.sha1 @@ -0,0 +1 @@ +80bc56b23b34cddab97557e5ada7cd0558dbf5e5 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.9.2.jar.sha1 b/server/licenses/lucene-sandbox-9.9.2.jar.sha1 deleted file mode 100644 index a30f00d72a689..0000000000000 --- a/server/licenses/lucene-sandbox-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4e2bdc6a5b65dd15b78477cf2c49cf7b707ba757 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.10.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..76e9fea920bb8 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.10.0.jar.sha1 @@ -0,0 +1 @@ +055e6aa874b6ec9ddafc2e115055de1445c2df49 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.9.2.jar.sha1 b/server/licenses/lucene-spatial-extras-9.9.2.jar.sha1 deleted file mode 100644 index dafeee401659a..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2693d4d4d3b9a0bf5b632ccbb01132eade1c4e14 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.10.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..3268b8bd557d0 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.10.0.jar.sha1 @@ -0,0 +1 @@ +06945a8b2a703062ce4b286ecceae7182b4f5f44 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.9.2.jar.sha1 b/server/licenses/lucene-spatial3d-9.9.2.jar.sha1 deleted file mode 100644 index 47a1c05187098..0000000000000 --- a/server/licenses/lucene-spatial3d-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -61866a44eafce499ebbc20bcb75c8b8a60a5449b \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.10.0.jar.sha1 b/server/licenses/lucene-suggest-9.10.0.jar.sha1 new file mode 100644 index 0000000000000..aaa10c449adb3 --- /dev/null +++ b/server/licenses/lucene-suggest-9.10.0.jar.sha1 @@ -0,0 +1 @@ +d2068ad997165a414927e8c0072d0b88741af069 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.9.2.jar.sha1 b/server/licenses/lucene-suggest-9.9.2.jar.sha1 deleted file mode 100644 index 115983c932983..0000000000000 --- a/server/licenses/lucene-suggest-9.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -352ad5296f48a482e2b65bb7d5d6ba2977bb035e \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java index c1f69d1ef3638..dea389bb6a0ff 100644 --- a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java +++ b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java @@ -90,6 +90,7 @@ public final class TranslogLeafReader extends LeafReader { 0, VectorEncoding.FLOAT32, VectorSimilarityFunction.EUCLIDEAN, + false, false ); private static final FieldInfo FAKE_ROUTING_FIELD = new FieldInfo( @@ -108,6 +109,7 @@ public final class TranslogLeafReader extends LeafReader { 0, VectorEncoding.FLOAT32, VectorSimilarityFunction.EUCLIDEAN, + false, false ); private static final FieldInfo FAKE_ID_FIELD = new FieldInfo( @@ -126,6 +128,7 @@ public final class TranslogLeafReader extends LeafReader { 0, VectorEncoding.FLOAT32, VectorSimilarityFunction.EUCLIDEAN, + false, false ); public static Set ALL_FIELD_NAMES = Sets.newHashSet(FAKE_SOURCE_FIELD.name, FAKE_ROUTING_FIELD.name, FAKE_ID_FIELD.name); diff --git a/server/src/main/java/org/opensearch/index/get/ShardGetService.java b/server/src/main/java/org/opensearch/index/get/ShardGetService.java index 831bb8aff3be3..d4eeb8aae8e24 100644 --- a/server/src/main/java/org/opensearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/opensearch/index/get/ShardGetService.java @@ -331,6 +331,7 @@ private GetResult innerGetLoadFromStoredFields( 0, VectorEncoding.FLOAT32, VectorSimilarityFunction.EUCLIDEAN, + false, false ); StoredFieldVisitor.Status status = fieldVisitor.needsField(fieldInfo); diff --git a/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java b/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java index 85aacfbd63ee2..8c4b8ad6d1776 100644 --- a/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java +++ b/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java @@ -85,6 +85,7 @@ public void setUp() throws Exception { 0, VectorEncoding.FLOAT32, VectorSimilarityFunction.EUCLIDEAN, + false, false ); From 6e702dcc9246503cb21ca78a3cb4bacad9e0be26 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Tue, 20 Feb 2024 12:56:11 -0800 Subject: [PATCH 21/41] Mute RemoteIndexPrimaryRelocationIT (#12389) When this test fails, it takes [15 minutes to timeout][1]. There are many examples like this. While it generally succeeds after a retry or two, I think it is better to mute this test rather than observe regular failures that extend build times. [1]: https://build.ci.opensearch.org/job/gradle-check/33937/testReport/ Signed-off-by: Andrew Ross --- build.gradle | 1 - .../opensearch/remotestore/RemoteIndexPrimaryRelocationIT.java | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6f9aa0ea9e439..2aac4a1e893e9 100644 --- a/build.gradle +++ b/build.gradle @@ -516,7 +516,6 @@ subprojects { includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexClusterDefaultDocRep") includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexIT") includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexTranslogDisabledIT") - includeClasses.add("org.opensearch.remotestore.RemoteIndexPrimaryRelocationIT") includeClasses.add("org.opensearch.remotestore.RemoteStoreBackpressureIT") includeClasses.add("org.opensearch.remotestore.RemoteStoreIT") includeClasses.add("org.opensearch.remotestore.RemoteStoreRefreshListenerIT") diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexPrimaryRelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexPrimaryRelocationIT.java index 67316ed0e6e6b..869032a84c2c2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexPrimaryRelocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexPrimaryRelocationIT.java @@ -44,6 +44,7 @@ public Settings indexSettings() { .build(); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9191") public void testPrimaryRelocationWhileIndexing() throws Exception { internalCluster().startClusterManagerOnlyNode(); super.testPrimaryRelocationWhileIndexing(); From 74589ef62bbacb37f27fcd510ee37adebd997e38 Mon Sep 17 00:00:00 2001 From: Sorabh Date: Tue, 20 Feb 2024 14:40:47 -0800 Subject: [PATCH 22/41] Disable concurrent search path for composite aggregations. For more details see: https://github.com/opensearch-project/OpenSearch/issues/12331 (#12375) Signed-off-by: Sorabh Hamirwasia --- .../opensearch.release-notes-2.12.0.md | 1 + .../aggregations/bucket/CompositeAggIT.java | 99 +++++++++++++++++++ .../CompositeAggregationFactory.java | 3 +- 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/CompositeAggIT.java diff --git a/release-notes/opensearch.release-notes-2.12.0.md b/release-notes/opensearch.release-notes-2.12.0.md index b5bd7767b8294..b188efe2c6bfe 100644 --- a/release-notes/opensearch.release-notes-2.12.0.md +++ b/release-notes/opensearch.release-notes-2.12.0.md @@ -142,6 +142,7 @@ - [Query Insights] Implement Top N Queries feature to collect and gather information about high latency queries in a window ([#11904](https://github.com/opensearch-project/OpenSearch/pull/11904)) - Add override support for sampling based on action ([#9621](https://github.com/opensearch-project/OpenSearch/issues/9621)) - Added custom sampler support based on transport action in request ([#9621](https://github.com/opensearch-project/OpenSearch/issues/9621)) +- Disable concurrent search for composite aggregation([#12375](https://github.com/opensearch-project/OpenSearch/pull/12375)) ### Removed - Remove deprecated classes for Rounding ([#10956](https://github.com/opensearch-project/OpenSearch/issues/10956)) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/CompositeAggIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/CompositeAggIT.java new file mode 100644 index 0000000000000..5a38ba670f1dc --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/CompositeAggIT.java @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.bucket; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; +import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; + +@OpenSearchIntegTestCase.SuiteScopeTestCase +public class CompositeAggIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { + + public CompositeAggIT(Settings staticSettings) { + super(staticSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + public void setupSuiteScopeCluster() throws Exception { + assertAcked( + prepareCreate( + "idx", + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ).setMapping("type", "type=keyword", "num", "type=integer", "score", "type=integer") + ); + waitForRelocation(ClusterHealthStatus.GREEN); + + client().prepareIndex("idx").setId("1").setSource("type", "type1", "num", "1", "score", "5").get(); + client().prepareIndex("idx").setId("1").setSource("type", "type2", "num", "11", "score", "50").get(); + refresh("idx"); + client().prepareIndex("idx").setId("1").setSource("type", "type1", "num", "1", "score", "2").get(); + client().prepareIndex("idx").setId("1").setSource("type", "type2", "num", "12", "score", "20").get(); + refresh("idx"); + client().prepareIndex("idx").setId("1").setSource("type", "type1", "num", "3", "score", "10").get(); + client().prepareIndex("idx").setId("1").setSource("type", "type2", "num", "13", "score", "15").get(); + refresh("idx"); + client().prepareIndex("idx").setId("1").setSource("type", "type1", "num", "3", "score", "1").get(); + client().prepareIndex("idx").setId("1").setSource("type", "type2", "num", "13", "score", "100").get(); + refresh("idx"); + + waitForRelocation(ClusterHealthStatus.GREEN); + refresh(); + } + + public void testCompositeAggWithNoSubAgg() { + SearchResponse rsp = client().prepareSearch("idx") + .addAggregation(new CompositeAggregationBuilder("my_composite", getTestValueSources())) + .get(); + assertSearchResponse(rsp); + } + + public void testCompositeAggWithSubAgg() { + SearchResponse rsp = client().prepareSearch("idx") + .addAggregation( + new CompositeAggregationBuilder("my_composite", getTestValueSources()).subAggregation( + new MaxAggregationBuilder("max").field("score") + ) + ) + .get(); + assertSearchResponse(rsp); + } + + private List> getTestValueSources() { + final List> sources = new ArrayList<>(); + sources.add(new TermsValuesSourceBuilder("keyword_vs").field("type")); + sources.add(new TermsValuesSourceBuilder("num_vs").field("num")); + return sources; + } +} diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationFactory.java index 2ff79fb623def..4af14ab014db5 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationFactory.java @@ -80,6 +80,7 @@ protected Aggregator createInternal( @Override protected boolean supportsConcurrentSegmentSearch() { - return true; + // See https://github.com/opensearch-project/OpenSearch/issues/12331 for details + return false; } } From b0687ebec3744c9aaf6d3361fd1260789e02da99 Mon Sep 17 00:00:00 2001 From: Daniel Widdis Date: Tue, 20 Feb 2024 15:08:22 -0800 Subject: [PATCH 23/41] Update TRIAGING instructions for closed issues (#12382) Signed-off-by: Daniel Widdis --- TRIAGING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TRIAGING.md b/TRIAGING.md index 47cb44a4f5ba2..3917f1e1442b9 100644 --- a/TRIAGING.md +++ b/TRIAGING.md @@ -68,7 +68,7 @@ Yes, there are several labels that are used to identify the 'state' of issues fi | Outcome | Label | Description | Canned Response | |--------------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Accepted | `-untriaged` | The issue has the details needed to be directed towards area owners. | "Thanks for filing this issue, please feel free to submit a pull request." | -| Rejected | N/A | The issue will be closed with a reason for why it was rejected. Reasons might include lack of details, or being outside the scope of the project. | "Thanks for creating this issue; however, it isn't being accepted due to {REASON}. Please feel free to re-open after addressing the reason." | +| Rejected | N/A | The issue will be closed with a reason for why it was rejected. Reasons might include lack of details, or being outside the scope of the project. | "Thanks for creating this issue; however, it isn't being accepted due to {REASON}. Please feel free to open a new issue after addressing the reason." | | Area Triage | `+{AREALABEL}` | OpenSearch has many different areas. If it's unclear whether an issue should be accepted, it will be labeled with the area and an owner will be @mentioned for follow-up. | "Thanks for creating this issue; the triage meeting was unsure if this issue should be accepted, @{PERSON} or someone from the area please review and then accept or reject this issue?" | | Transfer | N/A | If the issue applies to another repository within the OpenSearch Project, it will be transferred accordingly. | "@opensearch-project/triage, can you please transfer this issue to project {REPOSITORY}." Or, if someone at the meeting has permissions, they can start the transfer. | From 041373b3e55bd56d8cd082fd92f72540ba01482e Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 20 Feb 2024 17:40:24 -0600 Subject: [PATCH 24/41] Views (#11957) Views, simplify data access and manipulation by providing a virtual layer over one or more indices Signed-off-by: Peter Nied Signed-off-by: Peter Nied --- CHANGELOG.md | 1 + .../action/admin/indices/view/ViewIT.java | 154 ++++++++++ .../admin/indices/view/ViewTestBase.java | 79 +++++ .../opensearch/OpenSearchServerException.java | 36 ++- .../org/opensearch/action/ActionModule.java | 23 ++ .../admin/indices/view/CreateViewAction.java | 279 ++++++++++++++++++ .../admin/indices/view/DeleteViewAction.java | 156 ++++++++++ .../admin/indices/view/GetViewAction.java | 214 ++++++++++++++ .../indices/view/ListViewNamesAction.java | 133 +++++++++ .../admin/indices/view/SearchViewAction.java | 128 ++++++++ .../admin/indices/view/UpdateViewAction.java | 121 ++++++++ .../view/ViewAlreadyExistsException.java | 28 ++ .../indices/view/ViewNotFoundException.java | 28 ++ .../admin/indices/view/ViewService.java | 178 +++++++++++ .../admin/indices/view/package-info.java | 10 + .../java/org/opensearch/client/Client.java | 14 + .../opensearch/client/IndicesAdminClient.java | 27 ++ .../client/support/AbstractClient.java | 66 +++++ .../org/opensearch/cluster/ClusterModule.java | 3 + .../opensearch/cluster/metadata/Metadata.java | 34 +++ .../org/opensearch/cluster/metadata/View.java | 205 +++++++++++++ .../cluster/metadata/ViewMetadata.java | 187 ++++++++++++ .../main/java/org/opensearch/node/Node.java | 4 + .../action/admin/indices/RestViewAction.java | 240 +++++++++++++++ .../transport/TransportService.java | 3 +- .../ExceptionSerializationTests.java | 4 + .../indices/view/CreateViewRequestTests.java | 98 ++++++ .../indices/view/DeleteViewRequestTests.java | 44 +++ .../indices/view/GetViewResponseTests.java | 36 +++ .../view/ListViewNamesRequestTests.java | 35 +++ .../view/ListViewNamesResponseTests.java | 25 ++ .../indices/view/SearchViewRequestTests.java | 52 ++++ .../admin/indices/view/ViewServiceTest.java | 194 ++++++++++++ .../cluster/metadata/ViewTests.java | 80 +++++ 34 files changed, 2913 insertions(+), 6 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/ViewAlreadyExistsException.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/ViewNotFoundException.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/package-info.java create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/View.java create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java create mode 100644 server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewRequestTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/DeleteViewRequestTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/GetViewResponseTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesRequestTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesResponseTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java create mode 100644 server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dc9fac006716..170f1386aa136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625)) - [Admission Control] Integrate CPU AC with ResourceUsageCollector and add CPU AC stats to nodes/stats ([#10887](https://github.com/opensearch-project/OpenSearch/pull/10887)) - [S3 Repository] Add setting to control connection count for sync client ([#12028](https://github.com/opensearch-project/OpenSearch/pull/12028)) +- Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957)) - Add Remote Store Migration Experimental flag and allow mixed mode clusters under same ([#11986](https://github.com/opensearch-project/OpenSearch/pull/11986)) ### Dependencies diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java new file mode 100644 index 0000000000000..85c70e098652c --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java @@ -0,0 +1,154 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.cluster.metadata.View; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; +import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.hamcrest.MatcherAssert; + +import java.util.List; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +@ClusterScope(scope = Scope.TEST, numDataNodes = 2) +public class ViewIT extends ViewTestBase { + + public void testCreateView() throws Exception { + final String viewName = randomAlphaOfLength(8); + final String indexPattern = randomAlphaOfLength(8); + + logger.info("Testing createView with valid parameters"); + final View view = createView(viewName, indexPattern).getView(); + MatcherAssert.assertThat(view.getName(), is(viewName)); + MatcherAssert.assertThat(view.getTargets().size(), is(1)); + MatcherAssert.assertThat(view.getTargets().first().getIndexPattern(), is(indexPattern)); + + logger.info("Testing createView with existing view name"); + final Exception ex = assertThrows(ViewAlreadyExistsException.class, () -> createView(viewName, randomAlphaOfLength(8))); + MatcherAssert.assertThat(ex.getMessage(), is("View [" + viewName + "] already exists")); + } + + public void testCreateViewTargetsSet() throws Exception { + final String viewName = randomAlphaOfLength(8); + final String indexPattern = "a" + randomAlphaOfLength(8); + final String indexPattern2 = "b" + randomAlphaOfLength(8); + final List targetPatterns = List.of(indexPattern2, indexPattern, indexPattern); + + logger.info("Testing createView with targets that will be reordered and deduplicated"); + final View view = createView(viewName, targetPatterns).getView(); + MatcherAssert.assertThat(view.getName(), is(viewName)); + MatcherAssert.assertThat(view.getTargets().size(), is(2)); + MatcherAssert.assertThat(view.getTargets().first().getIndexPattern(), is(indexPattern)); + MatcherAssert.assertThat(view.getTargets().last().getIndexPattern(), is(indexPattern2)); + } + + public void testGetView() throws Exception { + final String viewName = randomAlphaOfLength(8); + createView(viewName, randomAlphaOfLength(8)); + + final View view = getView(viewName).getView(); + MatcherAssert.assertThat(view.getName(), is(viewName)); + + logger.info("Testing getView with non-existent view"); + final String nonExistentView = "non-existent-" + randomAlphaOfLength(8); + final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> getView(nonExistentView)); + MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist")); + } + + public void testDeleteView() throws Exception { + final String viewName = randomAlphaOfLength(8); + createView(viewName, randomAlphaOfLength(8)); + + logger.info("Testing deleteView with existing view"); + deleteView(viewName); + final Exception whenDeletedEx = assertThrows(ViewNotFoundException.class, () -> getView(viewName)); + MatcherAssert.assertThat(whenDeletedEx.getMessage(), is("View [" + viewName + "] does not exist")); + + logger.info("Testing deleteView with non-existent view"); + final String nonExistentView = "non-existent-" + randomAlphaOfLength(8); + final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> deleteView(nonExistentView)); + MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist")); + } + + public void testUpdateView() throws Exception { + final String viewName = randomAlphaOfLength(8); + final String originalIndexPattern = randomAlphaOfLength(8); + final View originalView = createView(viewName, originalIndexPattern).getView(); + + logger.info("Testing updateView with existing view"); + final String newDescription = randomAlphaOfLength(20); + final String newIndexPattern = "newPattern-" + originalIndexPattern; + final View updatedView = updateView(viewName, newDescription, newIndexPattern).getView(); + + MatcherAssert.assertThat(updatedView, not(is(originalView))); + MatcherAssert.assertThat(updatedView.getDescription(), is(newDescription)); + MatcherAssert.assertThat(updatedView.getTargets(), hasSize(1)); + MatcherAssert.assertThat(updatedView.getTargets().first().getIndexPattern(), is(newIndexPattern)); + + logger.info("Testing updateView with non-existent view"); + final String nonExistentView = "non-existent-" + randomAlphaOfLength(8); + final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> updateView(nonExistentView, null, "index-*")); + MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist")); + } + + public void testListViewNames() throws Exception { + logger.info("Testing listViewNames when no views have been created"); + MatcherAssert.assertThat(listViewNames(), is(List.of())); + + final String view1 = "view1"; + final String view2 = "view2"; + createView(view1, "index-1-*"); + createView(view2, "index-2-*"); + + logger.info("Testing listViewNames"); + final List views = listViewNames(); + MatcherAssert.assertThat(views, containsInAnyOrder(view1, view2)); + + logger.info("Testing listViewNames after deleting a view"); + deleteView(view1); + final List viewsAfterDeletion = listViewNames(); + MatcherAssert.assertThat(viewsAfterDeletion, not(contains(view1))); + MatcherAssert.assertThat(viewsAfterDeletion, contains(view2)); + } + + public void testSearchOperations() throws Exception { + final String indexInView1 = "index-1"; + final String indexInView2 = "index-2"; + final String indexNotInView = "another-index-1"; + + final int indexInView1DocCount = createIndexWithDocs(indexInView1); + final int indexInView2DocCount = createIndexWithDocs(indexInView2); + createIndexWithDocs(indexNotInView); + + logger.info("Testing view with no matches"); + createView("no-matches", "this-pattern-will-match-nothing"); + final Exception ex = assertThrows(IndexNotFoundException.class, () -> searchView("no-matches")); + MatcherAssert.assertThat(ex.getMessage(), is("no such index [this-pattern-will-match-nothing]")); + + logger.info("Testing view with exact index match"); + createView("only-index-1", "index-1"); + assertHitCount(searchView("only-index-1"), indexInView1DocCount); + + logger.info("Testing view with wildcard matches"); + createView("both-indices", "index-*"); + assertHitCount(searchView("both-indices"), indexInView1DocCount + indexInView2DocCount); + + logger.info("Testing searchView with non-existent view"); + final String nonExistentView = "non-existent-" + randomAlphaOfLength(8); + final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> searchView(nonExistentView)); + MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist")); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java new file mode 100644 index 0000000000000..a44ba0cf7c717 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.test.BackgroundIndexer; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; + +public abstract class ViewTestBase extends OpenSearchIntegTestCase { + + protected int createIndexWithDocs(final String indexName) throws Exception { + createIndex(indexName); + ensureGreen(indexName); + + final int numOfDocs = scaledRandomIntBetween(0, 200); + try (final BackgroundIndexer indexer = new BackgroundIndexer(indexName, "_doc", client(), numOfDocs)) { + waitForDocs(numOfDocs, indexer); + } + + refresh(indexName); + assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numOfDocs); + return numOfDocs; + } + + protected GetViewAction.Response createView(final String name, final String indexPattern) throws Exception { + return createView(name, List.of(indexPattern)); + } + + protected GetViewAction.Response createView(final String name, final List targets) throws Exception { + final CreateViewAction.Request request = new CreateViewAction.Request( + name, + null, + targets.stream().map(CreateViewAction.Request.Target::new).collect(Collectors.toList()) + ); + return client().admin().indices().createView(request).actionGet(); + } + + protected GetViewAction.Response getView(final String name) { + return client().admin().indices().getView(new GetViewAction.Request(name)).actionGet(); + + } + + protected void deleteView(final String name) { + client().admin().indices().deleteView(new DeleteViewAction.Request(name)).actionGet(); + performRemoteStoreTestAction(); + } + + protected List listViewNames() { + return client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); + } + + protected SearchResponse searchView(final String viewName) throws Exception { + final SearchViewAction.Request request = new SearchViewAction.Request(viewName, new SearchRequest()); + final SearchResponse response = client().searchView(request).actionGet(); + return response; + } + + protected GetViewAction.Response updateView(final String name, final String description, final String indexPattern) { + final CreateViewAction.Request request = new CreateViewAction.Request( + name, + description, + List.of(new CreateViewAction.Request.Target(indexPattern)) + ); + final GetViewAction.Response response = client().admin().indices().updateView(request).actionGet(); + return response; + } +} diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index 39c22e60f038a..c5a5ce12b238c 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -8,13 +8,11 @@ package org.opensearch; -import org.opensearch.core.index.snapshots.IndexShardSnapshotException; -import org.opensearch.crypto.CryptoRegistryException; - import static org.opensearch.OpenSearchException.OpenSearchExceptionHandle; import static org.opensearch.OpenSearchException.OpenSearchExceptionHandleRegistry.registerExceptionHandle; import static org.opensearch.OpenSearchException.UNKNOWN_VERSION_ADDED; import static org.opensearch.Version.V_2_10_0; +import static org.opensearch.Version.V_2_13_0; import static org.opensearch.Version.V_2_1_0; import static org.opensearch.Version.V_2_4_0; import static org.opensearch.Version.V_2_5_0; @@ -678,7 +676,12 @@ public static void registerExceptions() { ) ); registerExceptionHandle( - new OpenSearchExceptionHandle(IndexShardSnapshotException.class, IndexShardSnapshotException::new, 98, UNKNOWN_VERSION_ADDED) + new OpenSearchExceptionHandle( + org.opensearch.core.index.snapshots.IndexShardSnapshotException.class, + org.opensearch.core.index.snapshots.IndexShardSnapshotException::new, + 98, + UNKNOWN_VERSION_ADDED + ) ); registerExceptionHandle( new OpenSearchExceptionHandle( @@ -1174,7 +1177,30 @@ public static void registerExceptions() { V_2_7_0 ) ); - registerExceptionHandle(new OpenSearchExceptionHandle(CryptoRegistryException.class, CryptoRegistryException::new, 171, V_2_10_0)); + registerExceptionHandle( + new OpenSearchExceptionHandle( + org.opensearch.crypto.CryptoRegistryException.class, + org.opensearch.crypto.CryptoRegistryException::new, + 171, + V_2_10_0 + ) + ); + registerExceptionHandle( + new OpenSearchExceptionHandle( + org.opensearch.action.admin.indices.view.ViewNotFoundException.class, + org.opensearch.action.admin.indices.view.ViewNotFoundException::new, + 172, + V_2_13_0 + ) + ); + registerExceptionHandle( + new OpenSearchExceptionHandle( + org.opensearch.action.admin.indices.view.ViewAlreadyExistsException.class, + org.opensearch.action.admin.indices.view.ViewAlreadyExistsException::new, + 173, + V_2_13_0 + ) + ); registerExceptionHandle( new OpenSearchExceptionHandle( org.opensearch.cluster.block.IndexCreateBlockException.class, diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 46775466aa615..b19bf9590f43b 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -224,6 +224,12 @@ import org.opensearch.action.admin.indices.upgrade.post.UpgradeSettingsAction; import org.opensearch.action.admin.indices.validate.query.TransportValidateQueryAction; import org.opensearch.action.admin.indices.validate.query.ValidateQueryAction; +import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.action.admin.indices.view.DeleteViewAction; +import org.opensearch.action.admin.indices.view.GetViewAction; +import org.opensearch.action.admin.indices.view.ListViewNamesAction; +import org.opensearch.action.admin.indices.view.SearchViewAction; +import org.opensearch.action.admin.indices.view.UpdateViewAction; import org.opensearch.action.bulk.BulkAction; import org.opensearch.action.bulk.TransportBulkAction; import org.opensearch.action.bulk.TransportShardBulkAction; @@ -409,6 +415,7 @@ import org.opensearch.rest.action.admin.indices.RestUpgradeAction; import org.opensearch.rest.action.admin.indices.RestUpgradeStatusAction; import org.opensearch.rest.action.admin.indices.RestValidateQueryAction; +import org.opensearch.rest.action.admin.indices.RestViewAction; import org.opensearch.rest.action.cat.AbstractCatAction; import org.opensearch.rest.action.cat.RestAliasAction; import org.opensearch.rest.action.cat.RestAllocationAction; @@ -721,6 +728,14 @@ public void reg actions.register(ResolveIndexAction.INSTANCE, ResolveIndexAction.TransportAction.class); actions.register(DataStreamsStatsAction.INSTANCE, DataStreamsStatsAction.TransportAction.class); + // Views: + actions.register(CreateViewAction.INSTANCE, CreateViewAction.TransportAction.class); + actions.register(DeleteViewAction.INSTANCE, DeleteViewAction.TransportAction.class); + actions.register(GetViewAction.INSTANCE, GetViewAction.TransportAction.class); + actions.register(UpdateViewAction.INSTANCE, UpdateViewAction.TransportAction.class); + actions.register(ListViewNamesAction.INSTANCE, ListViewNamesAction.TransportAction.class); + actions.register(SearchViewAction.INSTANCE, SearchViewAction.TransportAction.class); + // Persistent tasks: actions.register(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class); actions.register(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class); @@ -915,6 +930,14 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestResolveIndexAction()); registerHandler.accept(new RestDataStreamsStatsAction()); + // View API + registerHandler.accept(new RestViewAction.CreateViewHandler()); + registerHandler.accept(new RestViewAction.DeleteViewHandler()); + registerHandler.accept(new RestViewAction.GetViewHandler()); + registerHandler.accept(new RestViewAction.UpdateViewHandler()); + registerHandler.accept(new RestViewAction.SearchViewHandler()); + registerHandler.accept(new RestViewAction.ListViewNamesHandler()); + // CAT API registerHandler.accept(new RestAllocationAction()); registerHandler.accept(new RestCatSegmentReplicationAction()); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java new file mode 100644 index 0000000000000..9faf25ce10732 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java @@ -0,0 +1,279 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ActionType; +import org.opensearch.action.ValidateActions; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; +import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.View; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.ValidationException; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.util.CollectionUtils; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** Action to create a view */ +@ExperimentalApi +public class CreateViewAction extends ActionType { + + private static final int MAX_NAME_LENGTH = 64; + private static final int MAX_DESCRIPTION_LENGTH = 256; + private static final int MAX_TARGET_COUNT = 25; + private static final int MAX_TARGET_INDEX_PATTERN_LENGTH = 64; + + public static final CreateViewAction INSTANCE = new CreateViewAction(); + public static final String NAME = "cluster:admin/views/create"; + + private CreateViewAction() { + super(NAME, GetViewAction.Response::new); + } + + /** + * Request for Creating View + */ + @ExperimentalApi + public static class Request extends ClusterManagerNodeRequest { + private final String name; + private final String description; + private final List targets; + + public Request(final String name, final String description, final List targets) { + this.name = name; + this.description = Objects.requireNonNullElse(description, ""); + this.targets = targets; + } + + public Request(final StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + this.description = in.readString(); + this.targets = in.readList(Target::new); + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public List getTargets() { + return new ArrayList<>(targets); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request that = (Request) o; + return name.equals(that.name) && description.equals(that.description) && targets.equals(that.targets); + } + + @Override + public int hashCode() { + return Objects.hash(name, description, targets); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.isNullOrEmpty(name)) { + validationException = ValidateActions.addValidationError("name cannot be empty or null", validationException); + } + if (name != null && name.length() > MAX_NAME_LENGTH) { + validationException = ValidateActions.addValidationError( + "name must be less than " + MAX_NAME_LENGTH + " characters in length", + validationException + ); + } + if (description != null && description.length() > MAX_DESCRIPTION_LENGTH) { + validationException = ValidateActions.addValidationError( + "description must be less than " + MAX_DESCRIPTION_LENGTH + " characters in length", + validationException + ); + } + if (CollectionUtils.isEmpty(targets)) { + validationException = ValidateActions.addValidationError("targets cannot be empty", validationException); + } else { + if (targets.size() > MAX_TARGET_COUNT) { + validationException = ValidateActions.addValidationError( + "view cannot have more than " + MAX_TARGET_COUNT + " targets", + validationException + ); + } + for (final Target target : targets) { + final var validationMessages = Optional.ofNullable(target.validate()) + .map(ValidationException::validationErrors) + .orElse(List.of()); + for (final String validationMessage : validationMessages) { + validationException = ValidateActions.addValidationError(validationMessage, validationException); + } + } + } + + return validationException; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + out.writeString(description); + out.writeList(targets); + } + + /** View target representation for create requests */ + @ExperimentalApi + public static class Target implements Writeable { + public final String indexPattern; + + public Target(final String indexPattern) { + this.indexPattern = indexPattern; + } + + public Target(final StreamInput in) throws IOException { + this.indexPattern = in.readString(); + } + + public String getIndexPattern() { + return indexPattern; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Target that = (Target) o; + return indexPattern.equals(that.indexPattern); + } + + @Override + public int hashCode() { + return Objects.hash(indexPattern); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(indexPattern); + } + + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (Strings.isNullOrEmpty(indexPattern)) { + validationException = ValidateActions.addValidationError("index pattern cannot be empty or null", validationException); + } + if (indexPattern != null && indexPattern.length() > MAX_TARGET_INDEX_PATTERN_LENGTH) { + validationException = ValidateActions.addValidationError( + "target index pattern must be less than " + MAX_TARGET_INDEX_PATTERN_LENGTH + " characters in length", + validationException + ); + } + + return validationException; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "target", + args -> new Target((String) args[0]) + ); + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), View.Target.INDEX_PATTERN_FIELD); + } + + public static Target fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + } + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "create_view_request", + args -> new Request((String) args[0], (String) args[1], (List) args[2]) + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), View.NAME_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), View.DESCRIPTION_FIELD); + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Target.fromXContent(p), View.TARGETS_FIELD); + } + + public static Request fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + } + + /** + * Transport Action for creating a View + */ + public static class TransportAction extends TransportClusterManagerNodeAction { + + private final ViewService viewService; + + @Inject + public TransportAction( + final TransportService transportService, + final ClusterService clusterService, + final ThreadPool threadPool, + final ActionFilters actionFilters, + final IndexNameExpressionResolver indexNameExpressionResolver, + final ViewService viewService + ) { + super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver); + this.viewService = viewService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected GetViewAction.Response read(final StreamInput in) throws IOException { + return new GetViewAction.Response(in); + } + + @Override + protected void clusterManagerOperation( + final Request request, + final ClusterState state, + final ActionListener listener + ) throws Exception { + viewService.createView(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(final Request request, final ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java new file mode 100644 index 0000000000000..abb3c3f4db5f6 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java @@ -0,0 +1,156 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ActionType; +import org.opensearch.action.ValidateActions; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; +import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.View; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.Objects; + +/** Action to delete a view */ +@SuppressWarnings("deprecation") +@ExperimentalApi +public class DeleteViewAction extends ActionType { + + public static final DeleteViewAction INSTANCE = new DeleteViewAction(); + public static final String NAME = "cluster:admin/views/delete"; + + public DeleteViewAction() { + super(NAME, AcknowledgedResponse::new); + } + + /** Request for delete view */ + @ExperimentalApi + public static class Request extends ClusterManagerNodeRequest { + private final String name; + + public Request(final String name) { + this.name = name; + } + + public Request(final StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + } + + public String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final Request that = (Request) o; + return name.equals(that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.isNullOrEmpty(name)) { + validationException = ValidateActions.addValidationError("name cannot be empty or null", validationException); + } + + return validationException; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + } + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "delete_view_request", + args -> new Request((String) args[0]) + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), View.NAME_FIELD); + } + + public static Request fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + } + + /** + * Transport Action for deleting a View + */ + public static class TransportAction extends TransportClusterManagerNodeAction { + + private final ViewService viewService; + + @Inject + public TransportAction( + final TransportService transportService, + final ClusterService clusterService, + final ThreadPool threadPool, + final ActionFilters actionFilters, + final IndexNameExpressionResolver indexNameExpressionResolver, + final ViewService viewService + ) { + super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver); + this.viewService = viewService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected AcknowledgedResponse read(final StreamInput in) throws IOException { + return new AcknowledgedResponse(in); + } + + @Override + protected void clusterManagerOperation( + final Request request, + final ClusterState state, + final ActionListener listener + ) throws Exception { + viewService.deleteView(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(final Request request, final ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java new file mode 100644 index 0000000000000..762eea965c8c1 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java @@ -0,0 +1,214 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ActionType; +import org.opensearch.action.ValidateActions; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; +import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.View; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.ParseField; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.ToXContent.Params; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.Objects; + +/** Action to get a view */ +@ExperimentalApi +public class GetViewAction extends ActionType { + + public static final GetViewAction INSTANCE = new GetViewAction(); + public static final String NAME = "views:data/read/get"; + + public GetViewAction() { + super(NAME, GetViewAction.Response::new); + } + + /** Request for get view */ + @ExperimentalApi + public static class Request extends ClusterManagerNodeRequest { + private final String name; + + public Request(final String name) { + this.name = name; + } + + public Request(final StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + } + + public String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final Request that = (Request) o; + return name.equals(that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.isNullOrEmpty(name)) { + validationException = ValidateActions.addValidationError("name cannot be empty or null", validationException); + } + + return validationException; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + } + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "get_view_request", + args -> new Request((String) args[0]) + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), View.NAME_FIELD); + } + + public static Request fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + } + + /** Response with a view */ + @ExperimentalApi + public static class Response extends ActionResponse implements ToXContentObject { + + private final View view; + + public Response(final View view) { + this.view = view; + } + + public Response(final StreamInput in) throws IOException { + super(in); + this.view = new View(in); + } + + public View getView() { + return view; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final Response that = (Response) o; + return getView().equals(that.getView()); + } + + @Override + public int hashCode() { + return Objects.hash(getView()); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + this.view.writeTo(out); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field("view", view); + builder.endObject(); + return builder; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "view_response", + args -> new Response((View) args[0]) + ); + static { + PARSER.declareObject(ConstructingObjectParser.constructorArg(), View.PARSER, new ParseField("view")); + } + + public static Response fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + } + + /** + * Transport Action for getting a View + */ + public static class TransportAction extends TransportClusterManagerNodeAction { + + private final ViewService viewService; + + @Inject + public TransportAction( + final TransportService transportService, + final ClusterService clusterService, + final ThreadPool threadPool, + final ActionFilters actionFilters, + final IndexNameExpressionResolver indexNameExpressionResolver, + final ViewService viewService + ) { + super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver); + this.viewService = viewService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected Response read(final StreamInput in) throws IOException { + return new Response(in); + } + + @Override + protected void clusterManagerOperation(final Request request, final ClusterState state, final ActionListener listener) + throws Exception { + viewService.getView(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(final Request request, final ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java new file mode 100644 index 0000000000000..eac0b1d5558ca --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java @@ -0,0 +1,133 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ActionType; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** Action to list a view names */ +@ExperimentalApi +public class ListViewNamesAction extends ActionType { + + public static final ListViewNamesAction INSTANCE = new ListViewNamesAction(); + public static final String NAME = "views:data/read/list"; + + public ListViewNamesAction() { + super(NAME, ListViewNamesAction.Response::new); + } + + /** Request for list view names */ + @ExperimentalApi + public static class Request extends ActionRequest { + public Request() {} + + public Request(final StreamInput in) {} + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final Request that = (Request) o; + return true; + } + + @Override + public int hashCode() { + return 1; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + } + + /** Response for list view names */ + @ExperimentalApi + public static class Response extends ActionResponse implements ToXContentObject { + + private final List views; + + public Response(final List views) { + this.views = views; + } + + public Response(final StreamInput in) throws IOException { + views = in.readStringList(); + } + + public List getViewNames() { + return views; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final Response that = (Response) o; + return views.equals(that.views); + } + + @Override + public int hashCode() { + return Objects.hash(views); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeStringCollection(views); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field("views", views); + builder.endObject(); + return builder; + } + } + + /** + * Transport Action for getting a View + */ + public static class TransportAction extends HandledTransportAction { + + private final ViewService viewService; + + @Inject + public TransportAction(final TransportService transportService, final ActionFilters actionFilters, final ViewService viewService) { + super(NAME, transportService, actionFilters, Request::new); + this.viewService = viewService; + } + + @Override + protected void doExecute(Task task, Request request, ActionListener listener) { + viewService.listViewNames(listener); + } + + } + +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java new file mode 100644 index 0000000000000..1e20221242f06 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java @@ -0,0 +1,128 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ActionType; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.Objects; +import java.util.function.Function; + +import static org.opensearch.action.ValidateActions.addValidationError; + +/** Action to create a view */ +@ExperimentalApi +public class SearchViewAction extends ActionType { + + public static final SearchViewAction INSTANCE = new SearchViewAction(); + public static final String NAME = "views:data/read/search"; + + private SearchViewAction() { + super(NAME, SearchResponse::new); + } + + /** + * Wraps the functionality of search requests and tailors for what is available + * when searching through views + */ + @ExperimentalApi + public static class Request extends SearchRequest { + + private final String view; + + public Request(final String view, final SearchRequest searchRequest) { + super(searchRequest); + this.view = view; + } + + public Request(final StreamInput in) throws IOException { + super(in); + view = in.readString(); + } + + public String getView() { + return view; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request that = (Request) o; + return view.equals(that.view) && super.equals(that); + } + + @Override + public int hashCode() { + return Objects.hash(view, super.hashCode()); + } + + @Override + public ActionRequestValidationException validate() { + final Function unsupported = (String x) -> x + " is not supported when searching views"; + ActionRequestValidationException validationException = super.validate(); + + if (scroll() != null) { + validationException = addValidationError(unsupported.apply("Scroll"), validationException); + } + + // TODO: Filter out any additional search features that are not supported. + // Required before removing @ExperimentalApi annotations. + + if (Strings.isNullOrEmpty(view)) { + validationException = addValidationError("View is required", validationException); + } + + return validationException; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(view); + } + + @Override + public String toString() { + return super.toString().replace("SearchRequest{", "SearchViewAction.Request{view=" + view + ","); + } + } + + /** + * Transport Action for searching a View + */ + public static class TransportAction extends HandledTransportAction { + + private final ViewService viewService; + + @Inject + public TransportAction(final TransportService transportService, final ActionFilters actionFilters, final ViewService viewService) { + super(NAME, transportService, actionFilters, Request::new); + this.viewService = viewService; + } + + @Override + protected void doExecute(final Task task, final Request request, final ActionListener listener) { + viewService.searchView(request, listener); + } + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java new file mode 100644 index 0000000000000..9182684c73a0b --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java @@ -0,0 +1,121 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.ActionType; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.View; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.List; + +/** Action to update a view */ +@ExperimentalApi +public class UpdateViewAction extends ActionType { + + public static final UpdateViewAction INSTANCE = new UpdateViewAction(); + public static final String NAME = "cluster:admin/views/update"; + + public UpdateViewAction() { + super(NAME, GetViewAction.Response::new); + } + + /** Request for update view */ + @ExperimentalApi + public static class Request { + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "create_view_request", + false, + (args, viewName) -> new CreateViewAction.Request(viewName, (String) args[0], (List) args[1]) + ); + + static { + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), View.DESCRIPTION_FIELD); + PARSER.declareObjectArray( + ConstructingObjectParser.constructorArg(), + (p, c) -> CreateViewAction.Request.Target.fromXContent(p), + View.TARGETS_FIELD + ); + } + + public static CreateViewAction.Request fromXContent(final XContentParser parser, final String viewName) throws IOException { + return PARSER.parse(parser, viewName); + } + } + + /** + * Transport Action for updating a View + */ + @ExperimentalApi + public static class TransportAction extends TransportClusterManagerNodeAction { + + private final ViewService viewService; + + @Inject + public TransportAction( + final TransportService transportService, + final ClusterService clusterService, + final ThreadPool threadPool, + final ActionFilters actionFilters, + final IndexNameExpressionResolver indexNameExpressionResolver, + final ViewService viewService + ) { + super( + NAME, + transportService, + clusterService, + threadPool, + actionFilters, + CreateViewAction.Request::new, + indexNameExpressionResolver + ); + this.viewService = viewService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected GetViewAction.Response read(final StreamInput in) throws IOException { + return new GetViewAction.Response(in); + } + + @Override + protected void clusterManagerOperation( + final CreateViewAction.Request request, + final ClusterState state, + final ActionListener listener + ) throws Exception { + viewService.updateView(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(final CreateViewAction.Request request, final ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } + +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewAlreadyExistsException.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewAlreadyExistsException.java new file mode 100644 index 0000000000000..90a69158286b4 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewAlreadyExistsException.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.common.io.stream.StreamInput; + +import java.io.IOException; + +/** Exception thrown when a view already exists */ +@ExperimentalApi +public class ViewAlreadyExistsException extends ResourceAlreadyExistsException { + + public ViewAlreadyExistsException(final String viewName) { + super("View [{}] already exists", viewName); + } + + public ViewAlreadyExistsException(final StreamInput in) throws IOException { + super(in); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewNotFoundException.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewNotFoundException.java new file mode 100644 index 0000000000000..3a90e6b0bc791 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewNotFoundException.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.ResourceNotFoundException; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.common.io.stream.StreamInput; + +import java.io.IOException; + +/** Exception thrown when a view is not found */ +@ExperimentalApi +public class ViewNotFoundException extends ResourceNotFoundException { + + public ViewNotFoundException(final String viewName) { + super("View [{}] does not exist", viewName); + } + + public ViewNotFoundException(final StreamInput in) throws IOException { + super(in); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java new file mode 100644 index 0000000000000..294f88decba1f --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -0,0 +1,178 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchAction; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.ClusterStateUpdateTask; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.View; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.action.ActionListener; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeSet; +import java.util.function.LongSupplier; +import java.util.stream.Collectors; + +/** Service to interact with views, create, retrieve, update, and delete */ +@ExperimentalApi +public class ViewService { + + private final static Logger LOG = LogManager.getLogger(ViewService.class); + private final ClusterService clusterService; + private final NodeClient client; + private final LongSupplier timeProvider; + + public ViewService(final ClusterService clusterService, final NodeClient client, final LongSupplier timeProvider) { + this.clusterService = clusterService; + this.client = client; + this.timeProvider = Optional.ofNullable(timeProvider).orElse(System::currentTimeMillis); + } + + public void createView(final CreateViewAction.Request request, final ActionListener listener) { + final long currentTime = timeProvider.getAsLong(); + + final List targets = request.getTargets() + .stream() + .map(target -> new View.Target(target.getIndexPattern())) + .collect(Collectors.toList()); + final View view = new View(request.getName(), request.getDescription(), currentTime, currentTime, new TreeSet<>(targets)); + + createOrUpdateView(Operation.CreateView, view, listener); + } + + public void updateView(final CreateViewAction.Request request, final ActionListener listener) { + final View originalView = getViewOrThrowException(request.getName()); + + final long currentTime = timeProvider.getAsLong(); + final List targets = request.getTargets() + .stream() + .map(target -> new View.Target(target.getIndexPattern())) + .collect(Collectors.toList()); + final View updatedView = new View( + request.getName(), + request.getDescription(), + originalView.getCreatedAt(), + currentTime, + new TreeSet<>(targets) + ); + + createOrUpdateView(Operation.UpdateView, updatedView, listener); + } + + public void deleteView(final DeleteViewAction.Request request, final ActionListener listener) { + getViewOrThrowException(request.getName()); + + clusterService.submitStateUpdateTask("delete_view_task", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(final ClusterState currentState) throws Exception { + return new ClusterState.Builder(clusterService.state()).metadata( + Metadata.builder(currentState.metadata()).removeView(request.getName()) + ).build(); + } + + @Override + public void onFailure(final String source, final Exception e) { + LOG.error("Unable to delete view, from " + source, e); + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { + listener.onResponse(new AcknowledgedResponse(true)); + } + }); + } + + public void getView(final GetViewAction.Request request, final ActionListener listener) { + final View view = getViewOrThrowException(request.getName()); + + listener.onResponse(new GetViewAction.Response(view)); + } + + public void listViewNames(final ActionListener listener) { + final List viewNames = new ArrayList<>( + Optional.ofNullable(clusterService) + .map(ClusterService::state) + .map(ClusterState::metadata) + .map(Metadata::views) + .map(Map::keySet) + .orElseThrow() + ); + + listener.onResponse(new ListViewNamesAction.Response(viewNames)); + } + + public void searchView(final SearchViewAction.Request request, final ActionListener listener) { + final View view = getViewOrThrowException(request.getView()); + + final String[] indices = view.getTargets().stream().map(View.Target::getIndexPattern).toArray(String[]::new); + request.indices(indices); + + client.executeLocally(SearchAction.INSTANCE, request, listener); + } + + View getViewOrThrowException(final String viewName) { + return Optional.ofNullable(clusterService) + .map(ClusterService::state) + .map(ClusterState::metadata) + .map(Metadata::views) + .map(views -> views.get(viewName)) + .orElseThrow(() -> new ViewNotFoundException(viewName)); + } + + private enum Operation { + CreateView("create", false), + UpdateView("update", true); + + private final String name; + private final boolean allowOverriding; + + Operation(final String name, final boolean allowOverriding) { + this.name = name; + this.allowOverriding = allowOverriding; + } + } + + private void createOrUpdateView(final Operation operation, final View view, final ActionListener listener) { + clusterService.submitStateUpdateTask(operation.name + "_view_task", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(final ClusterState currentState) throws Exception { + if (!operation.allowOverriding && currentState.metadata().views().containsKey(view.getName())) { + throw new ViewAlreadyExistsException(view.getName()); + } + return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).put(view)) + .build(); + } + + @Override + public void onFailure(final String source, final Exception e) { + LOG.error("Unable to " + operation.name + " view, from " + source, e); + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { + final View createdView = newState.getMetadata().views().get(view.getName()); + final GetViewAction.Response response = new GetViewAction.Response(createdView); + listener.onResponse(response); + } + }); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/package-info.java b/server/src/main/java/org/opensearch/action/admin/indices/view/package-info.java new file mode 100644 index 0000000000000..db0556b1bf334 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** View transport handlers. */ +package org.opensearch.action.admin.indices.view; diff --git a/server/src/main/java/org/opensearch/client/Client.java b/server/src/main/java/org/opensearch/client/Client.java index f4ae383249f61..322b435bdf35c 100644 --- a/server/src/main/java/org/opensearch/client/Client.java +++ b/server/src/main/java/org/opensearch/client/Client.java @@ -34,6 +34,8 @@ import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse; import org.opensearch.action.admin.indices.segments.PitSegmentsRequest; +import org.opensearch.action.admin.indices.view.ListViewNamesAction; +import org.opensearch.action.admin.indices.view.SearchViewAction; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkRequestBuilder; import org.opensearch.action.bulk.BulkResponse; @@ -466,6 +468,18 @@ public interface Client extends OpenSearchClient, Releasable { */ void fieldCaps(FieldCapabilitiesRequest request, ActionListener listener); + /** Search a view */ + void searchView(final SearchViewAction.Request request, final ActionListener listener); + + /** Search a view */ + ActionFuture searchView(final SearchViewAction.Request request); + + /** List all view names */ + void listViewNames(final ListViewNamesAction.Request request, ActionListener listener); + + /** List all view names */ + ActionFuture listViewNames(final ListViewNamesAction.Request request); + /** * Returns this clients settings */ diff --git a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java index 20dab1caa36c4..588584cd8a280 100644 --- a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java @@ -125,6 +125,9 @@ import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.opensearch.action.admin.indices.validate.query.ValidateQueryResponse; +import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.action.admin.indices.view.DeleteViewAction; +import org.opensearch.action.admin.indices.view.GetViewAction; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.metadata.IndexMetadata.APIBlock; import org.opensearch.common.Nullable; @@ -838,4 +841,28 @@ public interface IndicesAdminClient extends OpenSearchClient { * Resolves names and wildcard expressions to indices, aliases, and data streams */ ActionFuture resolveIndex(ResolveIndexAction.Request request); + + /** Create a view */ + void createView(CreateViewAction.Request request, ActionListener listener); + + /** Create a view */ + ActionFuture createView(CreateViewAction.Request request); + + /** Get the details of a view */ + void getView(GetViewAction.Request request, ActionListener listener); + + /** Get the details of a view */ + ActionFuture getView(GetViewAction.Request request); + + /** Delete a view */ + void deleteView(DeleteViewAction.Request request, ActionListener listener); + + /** Delete a view */ + ActionFuture deleteView(DeleteViewAction.Request request); + + /** Update a view */ + void updateView(CreateViewAction.Request request, ActionListener listener); + + /** Update a view */ + ActionFuture updateView(CreateViewAction.Request request); } diff --git a/server/src/main/java/org/opensearch/client/support/AbstractClient.java b/server/src/main/java/org/opensearch/client/support/AbstractClient.java index 786bfa38bb19c..6c6049f04231b 100644 --- a/server/src/main/java/org/opensearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/opensearch/client/support/AbstractClient.java @@ -312,6 +312,12 @@ import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.opensearch.action.admin.indices.validate.query.ValidateQueryResponse; +import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.action.admin.indices.view.DeleteViewAction; +import org.opensearch.action.admin.indices.view.GetViewAction; +import org.opensearch.action.admin.indices.view.ListViewNamesAction; +import org.opensearch.action.admin.indices.view.SearchViewAction; +import org.opensearch.action.admin.indices.view.UpdateViewAction; import org.opensearch.action.bulk.BulkAction; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkRequestBuilder; @@ -742,6 +748,26 @@ public FieldCapabilitiesRequestBuilder prepareFieldCaps(String... indices) { return new FieldCapabilitiesRequestBuilder(this, FieldCapabilitiesAction.INSTANCE, indices); } + @Override + public void searchView(final SearchViewAction.Request request, final ActionListener listener) { + execute(SearchViewAction.INSTANCE, request); + } + + @Override + public ActionFuture searchView(final SearchViewAction.Request request) { + return execute(SearchViewAction.INSTANCE, request); + } + + @Override + public void listViewNames(final ListViewNamesAction.Request request, ActionListener listener) { + execute(ListViewNamesAction.INSTANCE, request, listener); + } + + @Override + public ActionFuture listViewNames(final ListViewNamesAction.Request request) { + return execute(ListViewNamesAction.INSTANCE, request); + } + static class Admin implements AdminClient { private final ClusterAdmin clusterAdmin; @@ -2070,6 +2096,46 @@ public void resolveIndex(ResolveIndexAction.Request request, ActionListener resolveIndex(ResolveIndexAction.Request request) { return execute(ResolveIndexAction.INSTANCE, request); } + + @Override + public void createView(CreateViewAction.Request request, ActionListener listener) { + execute(CreateViewAction.INSTANCE, request); + } + + @Override + public ActionFuture createView(CreateViewAction.Request request) { + return execute(CreateViewAction.INSTANCE, request); + } + + /** Gets a view */ + public void getView(GetViewAction.Request request, ActionListener listener) { + execute(GetViewAction.INSTANCE, request, listener); + } + + /** Gets a view */ + public ActionFuture getView(GetViewAction.Request request) { + return execute(GetViewAction.INSTANCE, request); + } + + /** Create a view */ + public void deleteView(DeleteViewAction.Request request, ActionListener listener) { + execute(DeleteViewAction.INSTANCE, request, listener); + } + + /** Create a view */ + public ActionFuture deleteView(DeleteViewAction.Request request) { + return execute(DeleteViewAction.INSTANCE, request); + } + + /** Create a view */ + public void updateView(CreateViewAction.Request request, ActionListener listener) { + execute(UpdateViewAction.INSTANCE, request, listener); + } + + /** Create a view */ + public ActionFuture updateView(CreateViewAction.Request request) { + return execute(UpdateViewAction.INSTANCE, request); + } } @Override diff --git a/server/src/main/java/org/opensearch/cluster/ClusterModule.java b/server/src/main/java/org/opensearch/cluster/ClusterModule.java index bad881f8bda76..d2f4888ae8971 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterModule.java @@ -49,6 +49,7 @@ import org.opensearch.cluster.metadata.MetadataMappingService; import org.opensearch.cluster.metadata.MetadataUpdateSettingsService; import org.opensearch.cluster.metadata.RepositoriesMetadata; +import org.opensearch.cluster.metadata.ViewMetadata; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; import org.opensearch.cluster.routing.DelayedAllocationService; import org.opensearch.cluster.routing.allocation.AllocationService; @@ -195,6 +196,7 @@ public static List getNamedWriteables() { ComposableIndexTemplateMetadata::readDiffFrom ); registerMetadataCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom); + registerMetadataCustom(entries, ViewMetadata.TYPE, ViewMetadata::new, ViewMetadata::readDiffFrom); registerMetadataCustom(entries, WeightedRoutingMetadata.TYPE, WeightedRoutingMetadata::new, WeightedRoutingMetadata::readDiffFrom); registerMetadataCustom( entries, @@ -292,6 +294,7 @@ public static List getNamedXWriteables() { DataStreamMetadata::fromXContent ) ); + entries.add(new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(ViewMetadata.TYPE), ViewMetadata::fromXContent)); entries.add( new NamedXContentRegistry.Entry( Metadata.Custom.class, diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index 1871ed24973c2..59dc86ea28ed6 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -831,6 +831,10 @@ public Map dataStreams() { .orElse(Collections.emptyMap()); } + public Map views() { + return Optional.ofNullable((ViewMetadata) this.custom(ViewMetadata.TYPE)).map(ViewMetadata::views).orElse(Collections.emptyMap()); + } + public DecommissionAttributeMetadata decommissionAttributeMetadata() { return custom(DecommissionAttributeMetadata.TYPE); } @@ -1325,6 +1329,36 @@ public Builder removeDataStream(String name) { return this; } + private Map getViews() { + return Optional.ofNullable(customs.get(ViewMetadata.TYPE)) + .map(o -> (ViewMetadata) o) + .map(vmd -> vmd.views()) + .orElse(new HashMap<>()); + } + + public View view(final String viewName) { + return getViews().get(viewName); + } + + public Builder views(final Map views) { + this.customs.put(ViewMetadata.TYPE, new ViewMetadata(views)); + return this; + } + + public Builder put(final View view) { + Objects.requireNonNull(view, "view cannot be null"); + final var replacementViews = new HashMap<>(getViews()); + replacementViews.put(view.getName(), view); + return views(replacementViews); + } + + public Builder removeView(final String viewName) { + Objects.requireNonNull(viewName, "viewName cannot be null"); + final var replacementViews = new HashMap<>(getViews()); + replacementViews.remove(viewName); + return views(replacementViews); + } + public Custom getCustom(String type) { return customs.get(type); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/View.java b/server/src/main/java/org/opensearch/cluster/metadata/View.java new file mode 100644 index 0000000000000..1b1639bbca945 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/metadata/View.java @@ -0,0 +1,205 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.cluster.AbstractDiffable; +import org.opensearch.cluster.Diff; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.stream.Collectors; + +/** View of data in OpenSearch indices */ +@ExperimentalApi +public class View extends AbstractDiffable implements ToXContentObject { + + private final String name; + private final String description; + private final long createdAt; + private final long modifiedAt; + private final SortedSet targets; + + public View(final String name, final String description, final Long createdAt, final Long modifiedAt, final Set targets) { + this.name = Objects.requireNonNull(name, "Name must be provided"); + this.description = description; + this.createdAt = createdAt != null ? createdAt : -1; + this.modifiedAt = modifiedAt != null ? modifiedAt : -1; + this.targets = new TreeSet<>(Objects.requireNonNull(targets, "Targets are required on a view")); + } + + public View(final StreamInput in) throws IOException { + this(in.readString(), in.readOptionalString(), in.readZLong(), in.readZLong(), new TreeSet<>(in.readList(Target::new))); + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public long getCreatedAt() { + return createdAt; + } + + public long getModifiedAt() { + return modifiedAt; + } + + public SortedSet getTargets() { + return new TreeSet<>(targets); + } + + public static Diff readDiffFrom(final StreamInput in) throws IOException { + return readDiffFrom(View::new, in); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + View that = (View) o; + return name.equals(that.name) + && description.equals(that.description) + && createdAt == that.createdAt + && modifiedAt == that.modifiedAt + && targets.equals(that.targets); + } + + @Override + public int hashCode() { + return Objects.hash(name, description, createdAt, modifiedAt, targets); + } + + /** The source of data used to project the view */ + @ExperimentalApi + public static class Target implements Writeable, ToXContentObject, Comparable { + + private final String indexPattern; + + public Target(final String indexPattern) { + this.indexPattern = Objects.requireNonNull(indexPattern, "IndexPattern is required"); + } + + public Target(final StreamInput in) throws IOException { + this(in.readString()); + } + + public String getIndexPattern() { + return indexPattern; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Target that = (Target) o; + return indexPattern.equals(that.indexPattern); + } + + @Override + public int hashCode() { + return Objects.hash(indexPattern); + } + + public static final ParseField INDEX_PATTERN_FIELD = new ParseField("indexPattern"); + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field(INDEX_PATTERN_FIELD.getPreferredName(), indexPattern); + builder.endObject(); + return builder; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "target", + args -> new Target((String) args[0]) + ); + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_PATTERN_FIELD); + } + + public static Target fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(indexPattern); + } + + @Override + public int compareTo(final Target o) { + if (this == o) return 0; + + final Target other = (Target) o; + return this.indexPattern.compareTo(other.indexPattern); + } + } + + public static final ParseField NAME_FIELD = new ParseField("name"); + public static final ParseField DESCRIPTION_FIELD = new ParseField("description"); + public static final ParseField CREATED_AT_FIELD = new ParseField("createdAt"); + public static final ParseField MODIFIED_AT_FIELD = new ParseField("modifiedAt"); + public static final ParseField TARGETS_FIELD = new ParseField("targets"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "view", + args -> new View((String) args[0], (String) args[1], (Long) args[2], (Long) args[3], new TreeSet<>((List) args[4])) + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), DESCRIPTION_FIELD); + PARSER.declareLongOrNull(ConstructingObjectParser.optionalConstructorArg(), -1L, CREATED_AT_FIELD); + PARSER.declareLongOrNull(ConstructingObjectParser.optionalConstructorArg(), -1L, MODIFIED_AT_FIELD); + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Target.fromXContent(p), TARGETS_FIELD); + } + + public static View fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field(NAME_FIELD.getPreferredName(), name); + builder.field(DESCRIPTION_FIELD.getPreferredName(), description); + builder.field(CREATED_AT_FIELD.getPreferredName(), createdAt); + builder.field(MODIFIED_AT_FIELD.getPreferredName(), modifiedAt); + builder.field(TARGETS_FIELD.getPreferredName(), targets); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(name); + out.writeOptionalString(description); + out.writeZLong(createdAt); + out.writeZLong(modifiedAt); + out.writeList(targets.stream().collect(Collectors.toList())); + } +} diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java new file mode 100644 index 0000000000000..a89068078be58 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java @@ -0,0 +1,187 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.Version; +import org.opensearch.cluster.Diff; +import org.opensearch.cluster.DiffableUtils; +import org.opensearch.cluster.NamedDiff; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.ParseField; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static org.opensearch.cluster.metadata.ComposableIndexTemplateMetadata.MINIMMAL_SUPPORTED_VERSION; + +/** View metadata */ +@ExperimentalApi +public class ViewMetadata implements Metadata.Custom { + + public static final String TYPE = "view"; + private static final ParseField VIEW_FIELD = new ParseField("view"); + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE, + false, + a -> new ViewMetadata((Map) a[0]) + ); + + static { + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { + Map views = new HashMap<>(); + while (p.nextToken() != XContentParser.Token.END_OBJECT) { + views.put(p.currentName(), View.fromXContent(p)); + } + return views; + }, VIEW_FIELD); + } + + private final Map views; + + public ViewMetadata(final Map views) { + this.views = views; + } + + public ViewMetadata(final StreamInput in) throws IOException { + this.views = in.readMap(StreamInput::readString, View::new); + } + + public Map views() { + return this.views; + } + + @Override + public Diff diff(final Metadata.Custom before) { + return new ViewMetadata.ViewMetadataDiff((ViewMetadata) before, this); + } + + public static NamedDiff readDiffFrom(final StreamInput in) throws IOException { + return new ViewMetadata.ViewMetadataDiff(in); + } + + @Override + public EnumSet context() { + return Metadata.ALL_CONTEXTS; + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public Version getMinimalSupportedVersion() { + return MINIMMAL_SUPPORTED_VERSION; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeMap(this.views, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + } + + public static ViewMetadata fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(VIEW_FIELD.getPreferredName()); + for (Map.Entry entry : views.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); + return builder; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public int hashCode() { + return Objects.hash(this.views); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + ViewMetadata other = (ViewMetadata) obj; + return Objects.equals(this.views, other.views); + } + + @Override + public String toString() { + return Strings.toString(MediaTypeRegistry.JSON, this); + } + + /** + * Builder of view metadata. + */ + @ExperimentalApi + public static class Builder { + + private final Map views = new HashMap<>(); + + public Builder putDataStream(final View view) { + views.put(view.getName(), view); + return this; + } + + public ViewMetadata build() { + return new ViewMetadata(views); + } + } + + /** + * A diff between view metadata. + */ + static class ViewMetadataDiff implements NamedDiff { + + final Diff> dataStreamDiff; + + ViewMetadataDiff(ViewMetadata before, ViewMetadata after) { + this.dataStreamDiff = DiffableUtils.diff(before.views, after.views, DiffableUtils.getStringKeySerializer()); + } + + ViewMetadataDiff(StreamInput in) throws IOException { + this.dataStreamDiff = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), View::new, View::readDiffFrom); + } + + @Override + public Metadata.Custom apply(Metadata.Custom part) { + return new ViewMetadata(dataStreamDiff.apply(((ViewMetadata) part).views)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + dataStreamDiff.writeTo(out); + } + + @Override + public String getWriteableName() { + return TYPE; + } + } +} diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 547f610f4a752..40374113fe295 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -44,6 +44,7 @@ import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.action.ActionType; import org.opensearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus; +import org.opensearch.action.admin.indices.view.ViewService; import org.opensearch.action.search.SearchExecutionStatsCollector; import org.opensearch.action.search.SearchPhaseController; import org.opensearch.action.search.SearchRequestOperationsCompositeListenerFactory; @@ -864,6 +865,8 @@ protected Node( metadataCreateIndexService ); + final ViewService viewService = new ViewService(clusterService, client, null); + Collection pluginComponents = pluginsService.filterPlugins(Plugin.class) .stream() .flatMap( @@ -1232,6 +1235,7 @@ protected Node( b.bind(MetadataCreateIndexService.class).toInstance(metadataCreateIndexService); b.bind(AwarenessReplicaBalance.class).toInstance(awarenessReplicaBalance); b.bind(MetadataCreateDataStreamService.class).toInstance(metadataCreateDataStreamService); + b.bind(ViewService.class).toInstance(viewService); b.bind(SearchService.class).toInstance(searchService); b.bind(SearchTransportService.class).toInstance(searchTransportService); b.bind(SearchPhaseController.class) diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java new file mode 100644 index 0000000000000..47be439a97fc4 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -0,0 +1,240 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.admin.indices; + +import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.action.admin.indices.view.DeleteViewAction; +import org.opensearch.action.admin.indices.view.GetViewAction; +import org.opensearch.action.admin.indices.view.ListViewNamesAction; +import org.opensearch.action.admin.indices.view.SearchViewAction; +import org.opensearch.action.admin.indices.view.UpdateViewAction; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.ValidationException; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.NamedRoute; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestCancellableNodeClient; +import org.opensearch.rest.action.RestStatusToXContentListener; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.rest.action.search.RestSearchAction; + +import java.io.IOException; +import java.util.List; +import java.util.function.IntConsumer; + +import static org.opensearch.rest.RestRequest.Method.DELETE; +import static org.opensearch.rest.RestRequest.Method.GET; +import static org.opensearch.rest.RestRequest.Method.POST; +import static org.opensearch.rest.RestRequest.Method.PUT; + +/** All rest handlers for view actions */ +@ExperimentalApi +public class RestViewAction { + + public static final String VIEW_NAME = "view_name"; + public static final String VIEW_NAME_PARAMETER = "{" + VIEW_NAME + "}"; + + /** Handler for create view */ + @ExperimentalApi + public static class CreateViewHandler extends BaseRestHandler { + + @Override + public List routes() { + return List.of(new NamedRoute.Builder().path("/views").method(POST).uniqueName(CreateViewAction.NAME).build()); + } + + @Override + public String getName() { + return CreateViewAction.NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + try (final XContentParser parser = request.contentParser()) { + final CreateViewAction.Request createViewAction = CreateViewAction.Request.fromXContent(parser); + + final ValidationException validationResult = createViewAction.validate(); + if (validationResult != null) { + throw validationResult; + } + + return channel -> client.admin().indices().createView(createViewAction, new RestToXContentListener<>(channel)); + } + } + } + + /** Handler for delete view */ + @ExperimentalApi + public static class DeleteViewHandler extends BaseRestHandler { + + @Override + public List routes() { + return List.of( + new NamedRoute.Builder().path("/views/" + VIEW_NAME_PARAMETER).method(DELETE).uniqueName(DeleteViewAction.NAME).build() + ); + } + + @Override + public String getName() { + return DeleteViewAction.NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final String viewId = request.param(VIEW_NAME); + + final DeleteViewAction.Request deleteRequest = new DeleteViewAction.Request(viewId); + + final ValidationException validationResult = deleteRequest.validate(); + if (validationResult != null) { + throw validationResult; + } + + return channel -> client.admin().indices().deleteView(deleteRequest, new RestToXContentListener<>(channel)); + } + } + + /** Handler for update view */ + @ExperimentalApi + public static class UpdateViewHandler extends BaseRestHandler { + + @Override + public List routes() { + return List.of( + new NamedRoute.Builder().path("/views/" + VIEW_NAME_PARAMETER).method(PUT).uniqueName(UpdateViewAction.NAME).build() + ); + } + + @Override + public String getName() { + return UpdateViewAction.NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final String viewId = request.param(VIEW_NAME); + + try (final XContentParser parser = request.contentParser()) { + final CreateViewAction.Request updateRequest = UpdateViewAction.Request.fromXContent(parser, viewId); + + final ValidationException validationResult = updateRequest.validate(); + if (validationResult != null) { + throw validationResult; + } + + return channel -> client.admin().indices().updateView(updateRequest, new RestToXContentListener<>(channel)); + } + } + } + + /** Handler for get view */ + @ExperimentalApi + public static class GetViewHandler extends BaseRestHandler { + + @Override + public List routes() { + return List.of( + new NamedRoute.Builder().path("/views/" + VIEW_NAME_PARAMETER).method(GET).uniqueName(GetViewAction.NAME).build() + ); + } + + @Override + public String getName() { + return GetViewAction.NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final String viewId = request.param(VIEW_NAME); + + final GetViewAction.Request getRequest = new GetViewAction.Request(viewId); + + final ValidationException validationResult = getRequest.validate(); + if (validationResult != null) { + throw validationResult; + } + + return channel -> client.admin().indices().getView(getRequest, new RestToXContentListener<>(channel)); + } + } + + /** Handler for get view */ + @ExperimentalApi + public static class ListViewNamesHandler extends BaseRestHandler { + + @Override + public List routes() { + return List.of(new NamedRoute.Builder().path("/views/").method(GET).uniqueName(ListViewNamesAction.NAME).build()); + } + + @Override + public String getName() { + return ListViewNamesAction.NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + return channel -> client.listViewNames(new ListViewNamesAction.Request(), new RestToXContentListener<>(channel)); + } + } + + /** Handler for search view */ + @ExperimentalApi + public static class SearchViewHandler extends BaseRestHandler { + @Override + public List routes() { + return List.of( + new NamedRoute.Builder().path("/views/" + VIEW_NAME_PARAMETER + "/_search") + .method(GET) + .uniqueName(SearchViewAction.NAME) + .build(), + new NamedRoute.Builder().path("/views/" + VIEW_NAME_PARAMETER + "/_search") + .method(POST) + .uniqueName(SearchViewAction.NAME) + .build() + ); + } + + @Override + public String getName() { + return SearchViewAction.NAME; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final String viewId = request.param(VIEW_NAME); + + final SearchViewAction.Request viewSearchRequest = new SearchViewAction.Request(viewId, new SearchRequest()); + final IntConsumer setSize = size -> viewSearchRequest.source().size(size); + + request.withContentOrSourceParamParserOrNull( + parser -> RestSearchAction.parseSearchRequest( + viewSearchRequest, + request, + parser, + client.getNamedWriteableRegistry(), + setSize + ) + ); + + final ValidationException validationResult = viewSearchRequest.validate(); + if (validationResult != null) { + throw validationResult; + } + + return channel -> { + final RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); + cancelClient.execute(SearchViewAction.INSTANCE, viewSearchRequest, new RestStatusToXContentListener<>(channel)); + }; + } + } +} diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index d50266d8c9e4a..652d57f4c5348 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -1105,7 +1105,8 @@ public TransportAddress[] addressesFromString(String address) throws UnknownHost "cluster:admin", "cluster:monitor", "cluster:internal", - "internal:" + "internal:", + "views:" ) ) ); diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java index 2f47bfb4df70a..d7026159d9ec0 100644 --- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java @@ -40,6 +40,8 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.action.RoutingMissingException; import org.opensearch.action.TimestampParsingException; +import org.opensearch.action.admin.indices.view.ViewAlreadyExistsException; +import org.opensearch.action.admin.indices.view.ViewNotFoundException; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.action.support.replication.ReplicationOperation; @@ -892,6 +894,8 @@ public void testIds() { ids.put(169, NodeWeighedAwayException.class); ids.put(170, SearchPipelineProcessingException.class); ids.put(171, CryptoRegistryException.class); + ids.put(172, ViewNotFoundException.class); + ids.put(173, ViewAlreadyExistsException.class); ids.put(10001, IndexCreateBlockException.class); Map, Integer> reverse = new HashMap<>(); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewRequestTests.java new file mode 100644 index 0000000000000..e2211bb120366 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewRequestTests.java @@ -0,0 +1,98 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.AbstractWireSerializingTestCase; +import org.hamcrest.MatcherAssert; + +import java.util.List; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.nullValue; + +public class CreateViewRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return CreateViewAction.Request::new; + } + + @Override + protected CreateViewAction.Request createTestInstance() { + return new CreateViewAction.Request( + randomAlphaOfLength(8), + randomAlphaOfLength(8), + randomList(5, () -> new CreateViewAction.Request.Target(randomAlphaOfLength(8))) + ); + } + + public void testValidateRequest() { + final CreateViewAction.Request request = new CreateViewAction.Request( + "my-view", + "this is a description", + List.of(new CreateViewAction.Request.Target("my-indices-*")) + ); + + MatcherAssert.assertThat(request.validate(), nullValue()); + } + + public void testValidateRequestWithoutName() { + final CreateViewAction.Request request = new CreateViewAction.Request("", null, null); + final ActionRequestValidationException e = request.validate(); + + MatcherAssert.assertThat(e.validationErrors(), contains("name cannot be empty or null", "targets cannot be empty")); + } + + public void testSizeThresholds() { + final String validName = randomAlphaOfLength(8); + final String validDescription = randomAlphaOfLength(20); + final int validTargetLength = randomIntBetween(1, 5); + final String validIndexPattern = randomAlphaOfLength(8); + + final CreateViewAction.Request requestNameTooBig = new CreateViewAction.Request( + randomAlphaOfLength(65), + validDescription, + randomList(1, validTargetLength, () -> new CreateViewAction.Request.Target(validIndexPattern)) + ); + MatcherAssert.assertThat( + requestNameTooBig.validate().validationErrors(), + contains("name must be less than 64 characters in length") + ); + + final CreateViewAction.Request requestDescriptionTooBig = new CreateViewAction.Request( + validName, + randomAlphaOfLength(257), + randomList(1, validTargetLength, () -> new CreateViewAction.Request.Target(validIndexPattern)) + ); + MatcherAssert.assertThat( + requestDescriptionTooBig.validate().validationErrors(), + contains("description must be less than 256 characters in length") + ); + + final CreateViewAction.Request requestTargetsSize = new CreateViewAction.Request( + validName, + validDescription, + randomList(26, 26, () -> new CreateViewAction.Request.Target(validIndexPattern)) + ); + MatcherAssert.assertThat(requestTargetsSize.validate().validationErrors(), contains("view cannot have more than 25 targets")); + + final CreateViewAction.Request requestTargetsIndexPatternSize = new CreateViewAction.Request( + validName, + validDescription, + randomList(1, 1, () -> new CreateViewAction.Request.Target(randomAlphaOfLength(65))) + ); + MatcherAssert.assertThat( + requestTargetsIndexPatternSize.validate().validationErrors(), + contains("target index pattern must be less than 64 characters in length") + ); + } + +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/DeleteViewRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/DeleteViewRequestTests.java new file mode 100644 index 0000000000000..29305e3dfb92f --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/DeleteViewRequestTests.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.AbstractWireSerializingTestCase; +import org.hamcrest.MatcherAssert; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.nullValue; + +public class DeleteViewRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return DeleteViewAction.Request::new; + } + + @Override + protected DeleteViewAction.Request createTestInstance() { + return new DeleteViewAction.Request(randomAlphaOfLength(8)); + } + + public void testValidateRequest() { + final DeleteViewAction.Request request = new DeleteViewAction.Request("my-view"); + + MatcherAssert.assertThat(request.validate(), nullValue()); + } + + public void testValidateRequestWithoutName() { + final DeleteViewAction.Request request = new DeleteViewAction.Request(""); + final ActionRequestValidationException e = request.validate(); + + MatcherAssert.assertThat(e.validationErrors(), contains("name cannot be empty or null")); + } + +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/GetViewResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/GetViewResponseTests.java new file mode 100644 index 0000000000000..44dfbe5f1d781 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/GetViewResponseTests.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.cluster.metadata.View; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.AbstractWireSerializingTestCase; + +import java.util.TreeSet; + +public class GetViewResponseTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return GetViewAction.Response::new; + } + + @Override + protected GetViewAction.Response createTestInstance() { + return new GetViewAction.Response( + new View( + randomAlphaOfLength(8), + randomAlphaOfLength(8), + randomLong(), + randomLong(), + new TreeSet<>(randomList(5, () -> new View.Target(randomAlphaOfLength(8)))) + ) + ); + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesRequestTests.java new file mode 100644 index 0000000000000..80a2827d158bb --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesRequestTests.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.AbstractWireSerializingTestCase; +import org.hamcrest.MatcherAssert; + +import static org.hamcrest.Matchers.nullValue; + +public class ListViewNamesRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return ListViewNamesAction.Request::new; + } + + @Override + protected ListViewNamesAction.Request createTestInstance() { + return new ListViewNamesAction.Request(); + } + + public void testValidateRequest() { + final ListViewNamesAction.Request request = new ListViewNamesAction.Request(); + + MatcherAssert.assertThat(request.validate(), nullValue()); + } + +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesResponseTests.java new file mode 100644 index 0000000000000..ee8409fe3c805 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesResponseTests.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.AbstractWireSerializingTestCase; + +public class ListViewNamesResponseTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return ListViewNamesAction.Response::new; + } + + @Override + protected ListViewNamesAction.Response createTestInstance() { + return new ListViewNamesAction.Response(randomList(5, () -> randomAlphaOfLength(8))); + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java new file mode 100644 index 0000000000000..d49c0c1a8f2bd --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.AbstractWireSerializingTestCase; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +public class SearchViewRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return SearchViewAction.Request::new; + } + + @Override + protected SearchViewAction.Request createTestInstance() { + try { + return new SearchViewAction.Request(randomAlphaOfLength(8), new SearchRequest()); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + public void testValidateRequest() throws IOException { + final SearchViewAction.Request request = new SearchViewAction.Request("my-view", new SearchRequest()); + MatcherAssert.assertThat(request.validate(), nullValue()); + } + + public void testValidateRequestWithoutName() { + final SearchViewAction.Request request = new SearchViewAction.Request((String) null, new SearchRequest()); + final ActionRequestValidationException e = request.validate(); + + MatcherAssert.assertThat(e.validationErrors().size(), equalTo(1)); + MatcherAssert.assertThat(e.validationErrors().get(0), containsString("View is required")); + } + +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java new file mode 100644 index 0000000000000..91813e1336cf2 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java @@ -0,0 +1,194 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.search.SearchAction; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.View; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.action.ActionListener; +import org.hamcrest.MatcherAssert; +import org.junit.After; +import org.junit.Before; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; + +import static org.opensearch.test.OpenSearchTestCase.randomAlphaOfLength; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +@SuppressWarnings("unchecked") +public class ViewServiceTest { + + private final View.Target typicalTarget = new View.Target(randomAlphaOfLength(8)); + private final View typicalView = new View( + "view-" + randomAlphaOfLength(8), + "description " + randomAlphaOfLength(20), + -1L, + -1L, + Set.of(typicalTarget) + ); + + private ClusterService clusterService; + private NodeClient nodeClient; + private final AtomicLong currentTime = new AtomicLong(0); + private LongSupplier timeProvider = currentTime::longValue; + private ViewService viewService; + + @Before + public void before() { + clusterService = mock(ClusterService.class); + nodeClient = mock(NodeClient.class); + timeProvider = mock(LongSupplier.class); + doAnswer(invocation -> currentTime.get()).when(timeProvider).getAsLong(); + viewService = spy(new ViewService(clusterService, nodeClient, timeProvider)); + } + + @After + public void after() { + verifyNoMoreInteractions(timeProvider, clusterService, nodeClient); + } + + private CreateViewAction.Request createTypicalViewRequest() { + return new CreateViewAction.Request( + randomAlphaOfLength(8), + randomAlphaOfLength(20), + List.of(new CreateViewAction.Request.Target(randomAlphaOfLength(8))) + ); + } + + public void createView() { + final var request = createTypicalViewRequest(); + final var listener = mock(ActionListener.class); + setGetViewOrThrowExceptionToReturnTypicalView(); + + viewService.createView(request, listener); + + verify(clusterService).submitStateUpdateTask(eq("create_view_task"), any()); + verify(timeProvider).getAsLong(); + } + + public void updateView() { + final var request = createTypicalViewRequest(); + final var listener = mock(ActionListener.class); + setGetViewOrThrowExceptionToReturnTypicalView(); + + viewService.updateView(request, listener); + + verify(clusterService).submitStateUpdateTask(eq("update_view_task"), any()); + verify(timeProvider).getAsLong(); + } + + public void updateView_doesNotExist() { + final var request = createTypicalViewRequest(); + final var listener = mock(ActionListener.class); + doThrow(new ResourceNotFoundException("abc")).when(viewService).getViewOrThrowException(anyString()); + + final Exception ex = assertThrows(ResourceNotFoundException.class, () -> viewService.updateView(request, listener)); + MatcherAssert.assertThat(ex.getMessage(), equalTo("abc")); + } + + public void deleteView() { + final var request = new DeleteViewAction.Request(randomAlphaOfLength(8)); + final var listener = mock(ActionListener.class); + setGetViewOrThrowExceptionToReturnTypicalView(); + + viewService.deleteView(request, listener); + + verify(clusterService).submitStateUpdateTask(eq("delete_view_task"), any()); + } + + public void deleteView_doesNotExist() { + final var request = new DeleteViewAction.Request(randomAlphaOfLength(8)); + final var listener = mock(ActionListener.class); + doThrow(new ResourceNotFoundException("abc")).when(viewService).getViewOrThrowException(anyString()); + + final ResourceNotFoundException ex = assertThrows(ResourceNotFoundException.class, () -> viewService.deleteView(request, listener)); + + MatcherAssert.assertThat(ex.getMessage(), equalTo("abc")); + } + + public void getView() { + final var request = new GetViewAction.Request(randomAlphaOfLength(8)); + final var listener = mock(ActionListener.class); + setGetViewOrThrowExceptionToReturnTypicalView(); + + viewService.getView(request, listener); + + verify(listener).onResponse(any()); + } + + public void getView_doesNotExist() { + final var request = new GetViewAction.Request(randomAlphaOfLength(8)); + final var listener = mock(ActionListener.class); + doThrow(new ResourceNotFoundException("abc")).when(viewService).getViewOrThrowException(anyString()); + + final ResourceNotFoundException ex = assertThrows(ResourceNotFoundException.class, () -> viewService.getView(request, listener)); + + MatcherAssert.assertThat(ex.getMessage(), equalTo("abc")); + } + + public void listViewNames() { + final var clusterState = new ClusterState.Builder(new ClusterName(randomAlphaOfLength(8))).metadata( + new Metadata.Builder().views(Map.of(typicalView.getName(), typicalView)).build() + ).build(); + final var listener = mock(ActionListener.class); + when(clusterService.state()).thenReturn(clusterState); + + viewService.listViewNames(listener); + + verify(clusterService).state(); + verify(listener).onResponse(any()); + } + + public void listViewNames_noViews() { + final var clusterState = new ClusterState.Builder(new ClusterName(randomAlphaOfLength(8))).build(); + final var listener = mock(ActionListener.class); + when(clusterService.state()).thenReturn(clusterState); + + viewService.listViewNames(listener); + + verify(clusterService).state(); + verify(listener).onResponse(any()); + } + + public void searchView() { + final var request = spy(new SearchViewAction.Request(randomAlphaOfLength(8), new SearchRequest())); + final var listener = mock(ActionListener.class); + setGetViewOrThrowExceptionToReturnTypicalView(); + + viewService.searchView(request, listener); + + verify(nodeClient).executeLocally(eq(SearchAction.INSTANCE), any(), any(ActionListener.class)); + verify(request).indices(typicalTarget.getIndexPattern()); + } + + private void setGetViewOrThrowExceptionToReturnTypicalView() { + doAnswer(invocation -> typicalView).when(viewService).getViewOrThrowException(anyString()); + } +} diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java new file mode 100644 index 0000000000000..ad39e2b103087 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java @@ -0,0 +1,80 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.cluster.metadata.View.Target; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.AbstractSerializingTestCase; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.util.Set; +import java.util.TreeSet; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; + +public class ViewTests extends AbstractSerializingTestCase { + + private static Set randomTargets() { + int numTargets = randomIntBetween(1, 25); + return new TreeSet<>(randomList(1, numTargets, () -> new View.Target(randomAlphaOfLength(8)))); + } + + private static View randomInstance() { + final Set targets = randomTargets(); + final String viewName = randomAlphaOfLength(10); + final String description = randomAlphaOfLength(100); + return new View(viewName, description, Math.abs(randomLong()), Math.abs(randomLong()), targets); + } + + @Override + protected View doParseInstance(XContentParser parser) throws IOException { + return View.fromXContent(parser); + } + + @Override + protected Writeable.Reader instanceReader() { + return View::new; + } + + @Override + protected View createTestInstance() { + return randomInstance(); + } + + public void testNullName() { + final NullPointerException npe = assertThrows(NullPointerException.class, () -> new View(null, null, null, null, null)); + + MatcherAssert.assertThat(npe.getMessage(), equalTo("Name must be provided")); + } + + public void testNullTargets() { + final NullPointerException npe = assertThrows(NullPointerException.class, () -> new View("name", null, null, null, null)); + + MatcherAssert.assertThat(npe.getMessage(), equalTo("Targets are required on a view")); + } + + public void testNullTargetIndexPattern() { + final NullPointerException npe = assertThrows(NullPointerException.class, () -> new View.Target((String) null)); + + MatcherAssert.assertThat(npe.getMessage(), equalTo("IndexPattern is required")); + } + + public void testDefaultValues() { + final View view = new View("myName", null, null, null, Set.of()); + + MatcherAssert.assertThat(view.getName(), equalTo("myName")); + MatcherAssert.assertThat(view.getDescription(), equalTo(null)); + MatcherAssert.assertThat(view.getCreatedAt(), equalTo(-1L)); + MatcherAssert.assertThat(view.getModifiedAt(), equalTo(-1L)); + MatcherAssert.assertThat(view.getTargets(), empty()); + } +} From e97bee8126fa65bbc7f07a67c821ecaee27edc96 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 20 Feb 2024 20:26:29 -0500 Subject: [PATCH 25/41] Update Apache Lucene version to 9.10.0 for OpenSearch 2.13.0 (#12394) Signed-off-by: Andriy Redko --- libs/core/src/main/java/org/opensearch/Version.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 2971c4683192f..db20aace35cca 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -99,7 +99,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_11_1 = new Version(2110199, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_11_2 = new Version(2110299, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_9_2); - public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_9_2); + public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_10_0); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_10_0); public static final Version CURRENT = V_3_0_0; From f34ca960149c4f37f18bddfa4a0b0d7d2dcee9e4 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 21 Feb 2024 08:51:19 -0600 Subject: [PATCH 26/41] Warn about `index.mapper.dynamic` usage instead of erroring (#11193) When migrating from older versions of OpenSearch index.mapper.dynamic could be present, use the deprecation system to warn about this state and ignore this legacy setting so the index is usable. Signed-off-by: Peter Nied --- CHANGELOG.md | 1 + .../index/mapper/MapperService.java | 8 ++++++- .../index/mapper/MapperServiceTests.java | 24 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 170f1386aa136..183a07f599d6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,6 +126,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add a system property to configure YamlParser codepoint limits ([#12298](https://github.com/opensearch-project/OpenSearch/pull/12298)) - Prevent read beyond slice boundary in ByteArrayIndexInput ([#10481](https://github.com/opensearch-project/OpenSearch/issues/10481)) - Fix the "highlight.max_analyzer_offset" request parameter with "plain" highlighter ([#10919](https://github.com/opensearch-project/OpenSearch/pull/10919)) +- Warn about deprecated and ignored index.mapper.dynamic index setting ([#11193](https://github.com/opensearch-project/OpenSearch/pull/11193)) ### Security diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java index 9b8fa7eec37b9..fc8654216e187 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java @@ -237,6 +237,7 @@ public MapperService( ScriptService scriptService ) { super(indexSettings); + this.indexVersionCreated = indexSettings.getIndexVersionCreated(); this.indexAnalyzers = indexAnalyzers; this.documentParser = new DocumentMapperParser( @@ -261,7 +262,12 @@ public MapperService( this.idFieldDataEnabled = idFieldDataEnabled; if (INDEX_MAPPER_DYNAMIC_SETTING.exists(indexSettings.getSettings())) { - throw new IllegalArgumentException("Setting " + INDEX_MAPPER_DYNAMIC_SETTING.getKey() + " was removed after version 6.0.0"); + deprecationLogger.deprecate( + index().getName() + INDEX_MAPPER_DYNAMIC_SETTING.getKey(), + "Index [{}] has setting [{}] that is not supported in OpenSearch, its value will be ignored.", + index().getName(), + INDEX_MAPPER_DYNAMIC_SETTING.getKey() + ); } } diff --git a/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java index bb3f2be8ea748..adcfc9d7b17fc 100644 --- a/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java @@ -63,7 +63,9 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.function.Function; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; @@ -541,6 +543,28 @@ public void testReloadSearchAnalyzers() throws IOException { ); } + public void testMapperDynamicAllowedIgnored() { + final List> scenarios = List.of( + (builder) -> builder.putNull(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey()), + (builder) -> builder.put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), true), + (builder) -> builder.put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false) + ); + + for (int i = 0; i < scenarios.size(); i++) { + final Settings.Builder defaultSettingsBuilder = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1); + + final Settings settings = scenarios.get(i).apply(defaultSettingsBuilder).build(); + + createIndex("test" + i, settings).mapperService(); + } + + assertWarnings( + "[index.mapper.dynamic] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version." + ); + } + private boolean assertSameContainedFilters(TokenFilterFactory[] originalTokenFilter, NamedAnalyzer updatedAnalyzer) { ReloadableCustomAnalyzer updatedReloadableAnalyzer = (ReloadableCustomAnalyzer) updatedAnalyzer.analyzer(); TokenFilterFactory[] newTokenFilters = updatedReloadableAnalyzer.getComponents().getTokenFilters(); From bf050d41b2b3c36178bc7560b9c51cfec3e2b628 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 21 Feb 2024 10:47:06 -0500 Subject: [PATCH 27/41] [BUG] org.opensearch.common.xcontent.XContentParserTests.testString is flaky (#12415) Signed-off-by: Andriy Redko --- .../org/opensearch/common/xcontent/XContentParserTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java index 6e540681a8dca..81a2b0e290121 100644 --- a/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java +++ b/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java @@ -85,7 +85,8 @@ public class XContentParserTests extends OpenSearchTestCase { () -> randomAlphaOfLengthBetween(1, SmileXContent.DEFAULT_MAX_STRING_LEN), /* YAML parser limitation */ XContentType.YAML, - () -> randomRealisticUnicodeOfCodepointLengthBetween(1, YamlXContent.DEFAULT_CODEPOINT_LIMIT) + /* use 75% of the limit, difficult to get the exact size of the content right */ + () -> randomRealisticUnicodeOfCodepointLengthBetween(1, (int) (YamlXContent.DEFAULT_CODEPOINT_LIMIT * 0.75)) ); private static final Map> OFF_LIMIT_GENERATORS = Map.of( From 247e2eec46930e2eaec970b1acfb2f7d18e3e7a9 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 21 Feb 2024 13:09:27 -0800 Subject: [PATCH 28/41] Fix flaky SegRep test testScrollCreatedOnReplica (#12077) * Fix flaky test testScrollCreatedOnReplica Signed-off-by: Marc Handalian * Disable scheduled refresh Signed-off-by: Marc Handalian * Clean up segment collection assertions Signed-off-by: Marc Handalian * Fix spotless Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- .../replication/SegmentReplicationBaseIT.java | 15 ++++ .../replication/SegmentReplicationIT.java | 87 +++++++++++-------- .../index/engine/ReplicaFileTracker.java | 8 +- 3 files changed, 68 insertions(+), 42 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java index 796f09cb9528f..be849452c0f5e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java @@ -8,6 +8,7 @@ package org.opensearch.indices.replication; +import org.apache.lucene.index.SegmentInfos; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; @@ -16,6 +17,8 @@ import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.Nullable; +import org.opensearch.common.collect.Tuple; +import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; @@ -27,12 +30,14 @@ import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.IndicesService; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.transport.TransportService; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -230,4 +235,14 @@ protected void assertReplicaCheckpointUpdated(IndexShard primaryShard) throws Ex } }, 30, TimeUnit.SECONDS); } + + /** + * Returns the latest SIS for a shard but does not incref the segments. + */ + protected SegmentInfos getLatestSegmentInfos(IndexShard shard) throws IOException { + final Tuple, ReplicationCheckpoint> tuple = shard.getLatestSegmentInfosAndCheckpoint(); + try (final GatedCloseable closeable = tuple.v1()) { + return closeable.get(); + } + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 4a848e92800cb..87dd48de38d3e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -53,12 +53,11 @@ import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.allocation.command.CancelAllocationCommand; import org.opensearch.common.action.ActionFuture; -import org.opensearch.common.collect.Tuple; -import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.set.Sets; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.XContentBuilder; @@ -73,7 +72,6 @@ import org.opensearch.index.engine.NRTReplicationReaderManager; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.recovery.FileChunkRequest; -import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.node.NodeClosedException; import org.opensearch.search.SearchService; @@ -92,6 +90,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -1053,32 +1053,31 @@ private void assertAllocationIdsInReplicaShardStats(Set expected, Set assertEquals( getIndexShard(primary, INDEX_NAME).getLatestReplicationCheckpoint().getSegmentInfosVersion(), getIndexShard(replica, INDEX_NAME).getLatestReplicationCheckpoint().getSegmentInfosVersion() ) ); - final IndexShard replicaShard = getIndexShard(replica, INDEX_NAME); - final Tuple, ReplicationCheckpoint> tuple = replicaShard.getLatestSegmentInfosAndCheckpoint(); - final Collection snapshottedSegments; - try (final GatedCloseable closeable = tuple.v1()) { - snapshottedSegments = closeable.get().files(false); - } + // opens a scrolled query before a flush is called. // this is for testing scroll segment consistency between refresh and flush SearchResponse searchResponse = client(replica).prepareSearch() @@ -1092,17 +1091,20 @@ public void testScrollCreatedOnReplica() throws Exception { .setScroll(TimeValue.timeValueDays(1)) .get(); - // force call flush - flush(INDEX_NAME); + final IndexShard replicaShard = getIndexShard(replica, INDEX_NAME); + SegmentInfos latestSegmentInfos = getLatestSegmentInfos(replicaShard); + final Set snapshottedSegments = new HashSet<>(latestSegmentInfos.files(false)); + logger.info("Segments {}", snapshottedSegments); - for (int i = 3; i < 5; i++) { - client().prepareDelete(INDEX_NAME, String.valueOf(i)).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + // index more docs and force merge down to 1 segment + for (int i = 1; i < 5; i++) { + client().prepareIndex(INDEX_NAME) + .setId(String.valueOf(i)) + .setSource(jsonBuilder().startObject().field("field", i).endObject()) + .get(); refresh(INDEX_NAME); - if (randomBoolean()) { - client().admin().indices().prepareForceMerge(INDEX_NAME).setMaxNumSegments(1).setFlush(true).get(); - flush(INDEX_NAME); - } } + // create new on-disk segments and copy them out. assertBusy(() -> { assertEquals( getIndexShard(primary, INDEX_NAME).getLatestReplicationCheckpoint().getSegmentInfosVersion(), @@ -1110,13 +1112,19 @@ public void testScrollCreatedOnReplica() throws Exception { ); }); + // force merge and flush. client().admin().indices().prepareForceMerge(INDEX_NAME).setMaxNumSegments(1).setFlush(true).get(); + // wait for replication to complete assertBusy(() -> { assertEquals( getIndexShard(primary, INDEX_NAME).getLatestReplicationCheckpoint().getSegmentInfosVersion(), getIndexShard(replica, INDEX_NAME).getLatestReplicationCheckpoint().getSegmentInfosVersion() ); }); + logger.info("Local segments after force merge and commit {}", getLatestSegmentInfos(replicaShard).files(false)); + List filesBeforeClearScroll = List.of(replicaShard.store().directory().listAll()); + assertTrue("Files should be preserved", filesBeforeClearScroll.containsAll(snapshottedSegments)); + // Test stats logger.info("--> Collect all scroll query hits"); long scrollHits = 0; @@ -1125,20 +1133,23 @@ public void testScrollCreatedOnReplica() throws Exception { searchResponse = client(replica).prepareSearchScroll(searchResponse.getScrollId()).setScroll(TimeValue.timeValueDays(1)).get(); assertAllSuccessful(searchResponse); } while (searchResponse.getHits().getHits().length > 0); - - List currentFiles = List.of(replicaShard.store().directory().listAll()); - assertTrue("Files should be preserved", currentFiles.containsAll(snapshottedSegments)); + assertEquals(1, scrollHits); client(replica).prepareClearScroll().addScrollId(searchResponse.getScrollId()).get(); - - assertBusy( - () -> assertFalse( - "Files should be cleaned up post scroll clear request", - List.of(replicaShard.store().directory().listAll()).containsAll(snapshottedSegments) - ) + final Set filesAfterClearScroll = Arrays.stream(replicaShard.store().directory().listAll()).collect(Collectors.toSet()); + // there should be no active readers, snapshots, or on-disk commits containing the snapshotted files, check that they have been + // deleted. + Set latestCommitSegments = new HashSet<>(replicaShard.store().readLastCommittedSegmentsInfo().files(false)); + assertEquals( + "Snapshotted files are no longer part of the latest commit", + Collections.emptySet(), + Sets.intersection(latestCommitSegments, snapshottedSegments) + ); + assertEquals( + "All snapshotted files should be deleted", + Collections.emptySet(), + Sets.intersection(filesAfterClearScroll, snapshottedSegments) ); - assertEquals(10, scrollHits); - } /** diff --git a/server/src/main/java/org/opensearch/index/engine/ReplicaFileTracker.java b/server/src/main/java/org/opensearch/index/engine/ReplicaFileTracker.java index 19454967f9ee3..a9cc24abe3c01 100644 --- a/server/src/main/java/org/opensearch/index/engine/ReplicaFileTracker.java +++ b/server/src/main/java/org/opensearch/index/engine/ReplicaFileTracker.java @@ -17,7 +17,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.BiConsumer; +import java.util.function.Consumer; /** * This class is heavily influenced by Lucene's ReplicaFileDeleter class used to keep track of @@ -31,10 +31,10 @@ final class ReplicaFileTracker { public static final Logger logger = LogManager.getLogger(ReplicaFileTracker.class); private final Map refCounts = new HashMap<>(); - private final BiConsumer fileDeleter; + private final Consumer fileDeleter; private final Set EXCLUDE_FILES = Set.of("write.lock"); - public ReplicaFileTracker(BiConsumer fileDeleter) { + public ReplicaFileTracker(Consumer fileDeleter) { this.fileDeleter = fileDeleter; } @@ -82,7 +82,7 @@ private synchronized void delete(Collection toDelete) { private synchronized void delete(String fileName) { assert canDelete(fileName); - fileDeleter.accept("delete unreferenced", fileName); + fileDeleter.accept(fileName); } private synchronized boolean canDelete(String fileName) { From b774875b5cfc96257ddbc7334b039e56e5cc5b91 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 21 Feb 2024 14:26:44 -0800 Subject: [PATCH 29/41] [AUTO] [main] Add bwc version 2.12.1. (#12405) * Add bwc version 2.12.1 Signed-off-by: GitHub Signed-off-by: Kunal Kotwani * Fix version identifier for 2.12.1 Signed-off-by: Kunal Kotwani --------- Signed-off-by: GitHub Signed-off-by: Kunal Kotwani Co-authored-by: opensearch-ci-bot Co-authored-by: Kunal Kotwani --- .ci/bwcVersions | 1 + libs/core/src/main/java/org/opensearch/Version.java | 1 + 2 files changed, 2 insertions(+) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 793f9cc7125ee..1e3b913c5cb5a 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -28,4 +28,5 @@ BWC_VERSION: - "2.11.1" - "2.11.2" - "2.12.0" + - "2.12.1" - "2.13.0" diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index db20aace35cca..1bc0583d5e7df 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -99,6 +99,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_11_1 = new Version(2110199, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_11_2 = new Version(2110299, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_9_2); + public static final Version V_2_12_1 = new Version(2120199, org.apache.lucene.util.Version.LUCENE_9_9_2); public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_10_0); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_10_0); public static final Version CURRENT = V_3_0_0; From f8225ca5172d9d293fd8cdd6bf123d1bb5cf5cf3 Mon Sep 17 00:00:00 2001 From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> Date: Wed, 21 Feb 2024 15:43:25 -0800 Subject: [PATCH 30/41] [Segment Replication] Add random replication strategy (#12297) * Add new method to pick a random replication strategy. Signed-off-by: Rishikesh1159 * replace usage of refresh with refreshandWaitForReplication() Signed-off-by: Rishikesh1159 * Add replication strategy logging. Signed-off-by: Rishikesh1159 --------- Signed-off-by: Rishikesh1159 --- .../cluster/SimpleClusterStateIT.java | 7 ++++++- .../indices/IndicesRequestCacheIT.java | 19 ++++++++++++------- .../test/OpenSearchIntegTestCase.java | 18 ++++++++++++++++++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java index f0337e9c0c84c..af5900b1cba6c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java @@ -97,12 +97,17 @@ protected Collection> nodePlugins() { return Collections.singletonList(PrivateCustomPlugin.class); } + @Override + protected boolean useRandomReplicationStrategy() { + return true; + } + @Before public void indexData() throws Exception { index("foo", "bar", "1", XContentFactory.jsonBuilder().startObject().field("foo", "foo").endObject()); index("fuu", "buu", "1", XContentFactory.jsonBuilder().startObject().field("fuu", "fuu").endObject()); index("baz", "baz", "1", XContentFactory.jsonBuilder().startObject().field("baz", "baz").endObject()); - refresh(); + refreshAndWaitForReplication(); } public void testRoutingTable() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java index 95f6a7a03266c..82577eb1501f3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java @@ -81,6 +81,11 @@ public static Collection parameters() { ); } + @Override + protected boolean useRandomReplicationStrategy() { + return true; + } + // One of the primary purposes of the query cache is to cache aggs results public void testCacheAggs() throws Exception { Client client = client(); @@ -180,7 +185,7 @@ public void testQueryRewrite() throws Exception { // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get(); OpenSearchAssertions.assertAllSuccessful(forceMergeResponse); - refresh(); + refreshAndWaitForReplication(); ensureSearchable("index"); assertCacheState(client, "index", 0, 0); @@ -250,7 +255,7 @@ public void testQueryRewriteMissingValues() throws Exception { // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get(); OpenSearchAssertions.assertAllSuccessful(forceMergeResponse); - refresh(); + refreshAndWaitForReplication(); ensureSearchable("index"); assertCacheState(client, "index", 0, 0); @@ -316,7 +321,7 @@ public void testQueryRewriteDates() throws Exception { // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get(); OpenSearchAssertions.assertAllSuccessful(forceMergeResponse); - refresh(); + refreshAndWaitForReplication(); ensureSearchable("index"); assertCacheState(client, "index", 0, 0); @@ -389,7 +394,7 @@ public void testQueryRewriteDatesWithNow() throws Exception { .setFlush(true) .get(); OpenSearchAssertions.assertAllSuccessful(forceMergeResponse); - refresh(); + refreshAndWaitForReplication(); ensureSearchable("index-1", "index-2", "index-3"); assertCacheState(client, "index-1", 0, 0); @@ -460,7 +465,7 @@ public void testCanCache() throws Exception { // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get(); OpenSearchAssertions.assertAllSuccessful(forceMergeResponse); - refresh(); + refreshAndWaitForReplication(); ensureSearchable("index"); assertCacheState(client, "index", 0, 0); @@ -554,7 +559,7 @@ public void testCacheWithFilteredAlias() throws InterruptedException { // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get(); OpenSearchAssertions.assertAllSuccessful(forceMergeResponse); - refresh(); + refreshAndWaitForReplication(); indexRandomForConcurrentSearch("index"); @@ -661,7 +666,7 @@ public void testCacheWithInvalidation() throws Exception { assertCacheState(client, "index", 1, 1); // Explicit refresh would invalidate cache - refresh(); + refreshAndWaitForReplication(); // Hit same query again resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get(); assertSearchResponse(resp); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 33d5669d33297..47dd033834f1c 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -141,6 +141,7 @@ import org.opensearch.indices.IndicesQueryCache; import org.opensearch.indices.IndicesRequestCache; import org.opensearch.indices.IndicesService; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.indices.store.IndicesStore; import org.opensearch.monitor.os.OsInfo; import org.opensearch.node.NodeMocksPlugin; @@ -208,6 +209,7 @@ import static org.opensearch.index.IndexSettings.INDEX_DOC_ID_FUZZY_SET_FALSE_POSITIVE_PROBABILITY_SETTING; import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.test.XContentTestUtils.convertToMap; import static org.opensearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -1887,9 +1889,25 @@ protected Settings nodeSettings(int nodeOrdinal) { builder.put(TelemetrySettings.TRACER_FEATURE_ENABLED_SETTING.getKey(), true); builder.put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true); } + + // Randomly set a replication strategy for the node. Replication Strategy can still be manually overridden by subclass if needed. + if (useRandomReplicationStrategy()) { + ReplicationType replicationType = randomBoolean() ? ReplicationType.DOCUMENT : ReplicationType.SEGMENT; + logger.info("Randomly using Replication Strategy as {}.", replicationType.toString()); + builder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), replicationType); + } return builder.build(); } + /** + * Used for selecting random replication strategy, either DOCUMENT or SEGMENT. + * This method must be overridden by subclass to use random replication strategy. + * Should be used only on test classes where replication strategy is not critical for tests. + */ + protected boolean useRandomReplicationStrategy() { + return false; + } + protected Path nodeConfigPath(int nodeOrdinal) { return null; } From a50d1f675ad04739129389f52f18431865b0c731 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Wed, 21 Feb 2024 18:32:49 -0800 Subject: [PATCH 31/41] Unmute get field mapping test (#12422) This test was muted due to an [intermediate state][1] during type removal. It now passes per my local testing. [1]: https://github.com/opensearch-project/OpenSearch/issues/2440#issuecomment-1067943238 Signed-off-by: Andrew Ross --- .../test/indices.get_field_mapping/20_missing_field.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/20_missing_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/20_missing_field.yml index a36f807e63e0e..a65908b238013 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/20_missing_field.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/20_missing_field.yml @@ -1,9 +1,5 @@ --- "Return empty object if field doesn't exist, but index does": - - skip: - version: "all" - reason: "AwaitsFix https://github.com/opensearch-project/OpenSearch/issues/2440" - - do: indices.create: index: test_index @@ -18,7 +14,5 @@ indices.get_field_mapping: index: test_index fields: not_existent - ignore: 404 # ignore 404 failures for now - # see: https://github.com/opensearch-project/OpenSearch/issues/2440 - match: { 'test_index.mappings': {}} From fb417566bc877c7a6dea6a773b06f7b4d0d0a7df Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Wed, 21 Feb 2024 18:56:17 -0800 Subject: [PATCH 32/41] Remove PR11626 from 2.12 release notes (#12428) This PR was committed on main but not backported to 2.x in time for the 2.12 release. Signed-off-by: Andrew Ross --- CHANGELOG.md | 1 + release-notes/opensearch.release-notes-2.12.0.md | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 183a07f599d6e..bd826f7b5f04b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add community_id ingest processor ([#12121](https://github.com/opensearch-project/OpenSearch/pull/12121)) - Introduce query level setting `index.query.max_nested_depth` limiting nested queries ([#3268](https://github.com/opensearch-project/OpenSearch/issues/3268) - Add toString methods to MultiSearchRequest, MultiGetRequest and CreateIndexRequest ([#12163](https://github.com/opensearch-project/OpenSearch/pull/12163)) +- Support for returning scores in matched queries ([#11626](https://github.com/opensearch-project/OpenSearch/pull/11626)) ### Dependencies - Bump `peter-evans/find-comment` from 2 to 3 ([#12288](https://github.com/opensearch-project/OpenSearch/pull/12288)) diff --git a/release-notes/opensearch.release-notes-2.12.0.md b/release-notes/opensearch.release-notes-2.12.0.md index b188efe2c6bfe..49955c1f969f0 100644 --- a/release-notes/opensearch.release-notes-2.12.0.md +++ b/release-notes/opensearch.release-notes-2.12.0.md @@ -52,7 +52,6 @@ - Remove concurrent segment search feature flag for GA launch ([#12074](https://github.com/opensearch-project/OpenSearch/pull/12074)) - Enable Fuzzy codec for doc id fields using a bloom filter ([#11022](https://github.com/opensearch-project/OpenSearch/pull/11022)) - [Metrics Framework] Adds support for Histogram metric ([#12062](https://github.com/opensearch-project/OpenSearch/pull/12062)) -- Support for returning scores in matched queries ([#11626](https://github.com/opensearch-project/OpenSearch/pull/11626)) ### Dependencies - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822)) From 87ac37460c16a5b3cfa1cd85cad2cb7468b430a3 Mon Sep 17 00:00:00 2001 From: Alexander Date: Thu, 22 Feb 2024 23:02:53 +0200 Subject: [PATCH 33/41] Shard id awareness of SearchLookup (#1063) * Shard id awareness of SearchLookup Signed-off-by: Alexander Koval * Add unit test for deprecated constructor Signed-off-by: Andrew Ross --------- Signed-off-by: Alexander Koval Signed-off-by: Andrew Ross Co-authored-by: Andrew Ross --- CHANGELOG.md | 1 + .../ExpressionFieldScriptTests.java | 2 +- .../ExpressionNumberSortScriptTests.java | 2 +- .../ExpressionTermsSetQueryTests.java | 2 +- .../index/query/QueryShardContext.java | 6 ++-- .../search/lookup/SearchLookup.java | 32 +++++++++++++++++-- .../fielddata/IndexFieldDataServiceTests.java | 4 ++- .../index/query/QueryShardContextTests.java | 9 +++++- .../support/ScriptValuesTests.java | 2 +- .../search/lookup/SearchLookupTests.java | 21 ++++++++++++ .../index/mapper/MapperServiceTestCase.java | 2 +- .../index/mapper/MapperTestCase.java | 2 +- 12 files changed, 73 insertions(+), 12 deletions(-) create mode 100644 server/src/test/java/org/opensearch/search/lookup/SearchLookupTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index bd826f7b5f04b..b5121a067144b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,6 +103,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Introduce query level setting `index.query.max_nested_depth` limiting nested queries ([#3268](https://github.com/opensearch-project/OpenSearch/issues/3268) - Add toString methods to MultiSearchRequest, MultiGetRequest and CreateIndexRequest ([#12163](https://github.com/opensearch-project/OpenSearch/pull/12163)) - Support for returning scores in matched queries ([#11626](https://github.com/opensearch-project/OpenSearch/pull/11626)) +- Add shard id property to SearchLookup for use in field types provided by plugins ([#1063](https://github.com/opensearch-project/OpenSearch/pull/1063)) ### Dependencies - Bump `peter-evans/find-comment` from 2 to 3 ([#12288](https://github.com/opensearch-project/OpenSearch/pull/12288)) diff --git a/modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionFieldScriptTests.java index 143ff4f5c51bd..d7be890014add 100644 --- a/modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionFieldScriptTests.java +++ b/modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionFieldScriptTests.java @@ -77,7 +77,7 @@ public void setUp() throws Exception { when(fieldData.load(any())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData); + lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData, SearchLookup.UNKNOWN_SHARD_ID); } private FieldScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionNumberSortScriptTests.java index 498c0542e9c3e..94a422503d6bd 100644 --- a/modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionNumberSortScriptTests.java +++ b/modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionNumberSortScriptTests.java @@ -77,7 +77,7 @@ public void setUp() throws Exception { when(fieldData.load(any())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData); + lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData, SearchLookup.UNKNOWN_SHARD_ID); } private NumberSortScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionTermsSetQueryTests.java index 499f94afcb6af..a1d6df80715be 100644 --- a/modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionTermsSetQueryTests.java +++ b/modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionTermsSetQueryTests.java @@ -77,7 +77,7 @@ public void setUp() throws Exception { when(fieldData.load(any())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData); + lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData, SearchLookup.UNKNOWN_SHARD_ID); } private TermsSetQueryScript.LeafFactory compile(String expression) { diff --git a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java index 5069f055a89bb..f3b392559d33e 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java @@ -423,7 +423,8 @@ public SearchLookup lookup() { if (this.lookup == null) { this.lookup = new SearchLookup( getMapperService(), - (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup) + (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup), + shardId ); } return this.lookup; @@ -439,7 +440,8 @@ public SearchLookup newFetchLookup() { */ return new SearchLookup( getMapperService(), - (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup) + (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup), + shardId ); } diff --git a/server/src/main/java/org/opensearch/search/lookup/SearchLookup.java b/server/src/main/java/org/opensearch/search/lookup/SearchLookup.java index 8813865a657dc..906616eb9ba5f 100644 --- a/server/src/main/java/org/opensearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/opensearch/search/lookup/SearchLookup.java @@ -60,6 +60,12 @@ public class SearchLookup { */ private static final int MAX_FIELD_CHAIN_DEPTH = 5; + /** + * This constant should be used in cases when shard id is unknown. + * Mostly it should be used in tests. + */ + public static final int UNKNOWN_SHARD_ID = -1; + /** * The chain of fields for which this lookup was created, used for detecting * loops caused by runtime fields referring to other runtime fields. The chain is empty @@ -74,14 +80,27 @@ public class SearchLookup { private final SourceLookup sourceLookup; private final FieldsLookup fieldsLookup; private final BiFunction, IndexFieldData> fieldDataLookup; + private final int shardId; /** - * Create the top level field lookup for a search request. Provides a way to look up fields from doc_values, - * stored fields, or _source. + * Constructor for backwards compatibility. Use the one with explicit shardId argument. */ + @Deprecated public SearchLookup( MapperService mapperService, BiFunction, IndexFieldData> fieldDataLookup + ) { + this(mapperService, fieldDataLookup, UNKNOWN_SHARD_ID); + } + + /** + * Create the top level field lookup for a search request. Provides a way to look up fields from doc_values, + * stored fields, or _source. + */ + public SearchLookup( + MapperService mapperService, + BiFunction, IndexFieldData> fieldDataLookup, + int shardId ) { this.fieldChain = Collections.emptySet(); docMap = new DocLookup( @@ -91,6 +110,7 @@ public SearchLookup( sourceLookup = new SourceLookup(); fieldsLookup = new FieldsLookup(mapperService); this.fieldDataLookup = fieldDataLookup; + this.shardId = shardId; } /** @@ -109,6 +129,7 @@ private SearchLookup(SearchLookup searchLookup, Set fieldChain) { this.sourceLookup = searchLookup.sourceLookup; this.fieldsLookup = searchLookup.fieldsLookup; this.fieldDataLookup = searchLookup.fieldDataLookup; + this.shardId = searchLookup.shardId; } /** @@ -143,4 +164,11 @@ public DocLookup doc() { public SourceLookup source() { return sourceLookup; } + + public int shardId() { + if (shardId == UNKNOWN_SHARD_ID) { + throw new IllegalStateException("Shard id is unknown for this lookup"); + } + return shardId; + } } diff --git a/server/src/test/java/org/opensearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/opensearch/index/fielddata/IndexFieldDataServiceTests.java index bcdca2236d3f3..3fb43b7dbdc4e 100644 --- a/server/src/test/java/org/opensearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/opensearch/index/fielddata/IndexFieldDataServiceTests.java @@ -138,13 +138,15 @@ public void testGetForFieldRuntimeField() { ); final SetOnce> searchLookupSetOnce = new SetOnce<>(); MappedFieldType ft = mock(MappedFieldType.class); + final int shardId = randomInt(); when(ft.fielddataBuilder(Mockito.any(), Mockito.any())).thenAnswer(invocationOnMock -> { @SuppressWarnings("unchecked") Supplier searchLookup = (Supplier) invocationOnMock.getArguments()[1]; searchLookupSetOnce.set(searchLookup); + assertEquals(searchLookup.get().shardId(), shardId); return (IndexFieldData.Builder) (cache, breakerService) -> null; }); - SearchLookup searchLookup = new SearchLookup(null, null); + SearchLookup searchLookup = new SearchLookup(null, null, shardId); ifdService.getForField(ft, "qualified", () -> searchLookup); assertSame(searchLookup, searchLookupSetOnce.get().get()); } diff --git a/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java index c819d35872c6e..1a2ad49a3f334 100644 --- a/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java @@ -91,6 +91,8 @@ public class QueryShardContextTests extends OpenSearchTestCase { + private static final int SHARD_ID = 0; + public void testFailIfFieldMappingNotFound() { QueryShardContext context = createQueryShardContext(IndexMetadata.INDEX_UUID_NA_VALUE, null); context.setAllowUnmappedFields(false); @@ -307,6 +309,11 @@ public void testFielddataLookupOneFieldManyReferences() throws IOException { assertEquals(Arrays.asList(expectedFirstDoc.toString(), expectedSecondDoc.toString()), collect("field", queryShardContext)); } + public void testSearchLookupShardId() { + SearchLookup searchLookup = createQueryShardContext("uuid", null, null).lookup(); + assertEquals(SHARD_ID, searchLookup.shardId()); + } + public static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias) { return createQueryShardContext(indexUuid, clusterAlias, null); } @@ -343,7 +350,7 @@ private static QueryShardContext createQueryShardContext( } final long nowInMillis = randomNonNegativeLong(); return new QueryShardContext( - 0, + SHARD_ID, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, diff --git a/server/src/test/java/org/opensearch/search/aggregations/support/ScriptValuesTests.java b/server/src/test/java/org/opensearch/search/aggregations/support/ScriptValuesTests.java index 9eb90f2358f98..98dde2c7a31b3 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/support/ScriptValuesTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/support/ScriptValuesTests.java @@ -60,7 +60,7 @@ private static class FakeAggregationScript extends AggregationScript { int index; FakeAggregationScript(Object[][] values) { - super(Collections.emptyMap(), new SearchLookup(null, null) { + super(Collections.emptyMap(), new SearchLookup(null, null, SearchLookup.UNKNOWN_SHARD_ID) { @Override public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) { diff --git a/server/src/test/java/org/opensearch/search/lookup/SearchLookupTests.java b/server/src/test/java/org/opensearch/search/lookup/SearchLookupTests.java new file mode 100644 index 0000000000000..e942c3ab17420 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/lookup/SearchLookupTests.java @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.lookup; + +import org.opensearch.index.mapper.MapperService; +import org.opensearch.test.OpenSearchTestCase; + +import static org.mockito.Mockito.mock; + +public class SearchLookupTests extends OpenSearchTestCase { + public void testDeprecatedConstructorShardId() { + final SearchLookup searchLookup = new SearchLookup(mock(MapperService.class), (a, b) -> null); + assertThrows(IllegalStateException.class, searchLookup::shardId); + } +} diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java index ac78a0d1936ea..a65ce3cbdd380 100644 --- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java @@ -253,7 +253,7 @@ protected QueryShardContext createQueryShardContext(MapperService mapperService) when(queryShardContext.allowExpensiveQueries()).thenReturn(true); when(queryShardContext.lookup()).thenReturn(new SearchLookup(mapperService, (ft, s) -> { throw new UnsupportedOperationException("search lookup not available"); - })); + }, SearchLookup.UNKNOWN_SHARD_ID)); when(queryShardContext.getFieldType(any())).thenAnswer(inv -> mapperService.fieldType(inv.getArguments()[0].toString())); when(queryShardContext.documentMapper(anyString())).thenReturn(mapperService.documentMapper()); return queryShardContext; diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java index da043229c642d..dc5954907a4fa 100644 --- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java @@ -293,7 +293,7 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi withLuceneIndex(mapperService, iw -> { iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field(ft.name(), sourceValue))).rootDoc()); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService, fieldDataLookup); + SearchLookup lookup = new SearchLookup(mapperService, fieldDataLookup, SearchLookup.UNKNOWN_SHARD_ID); ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.doc().getForField(ft)); IndexSearcher searcher = newSearcher(iw); LeafReaderContext context = searcher.getIndexReader().leaves().get(0); From f90baa26894c66bb85f74bb3aaf201d6dd2329a6 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 23 Feb 2024 20:42:50 -0500 Subject: [PATCH 34/41] Add support for scored named queries (version updates) (#12446) Signed-off-by: Andriy Redko --- .../rest-api-spec/test/search/350_matched_queries.yml | 4 ++-- server/src/main/java/org/opensearch/search/SearchHit.java | 4 ++-- .../org/opensearch/search/builder/SearchSourceBuilder.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/350_matched_queries.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/350_matched_queries.yml index 450e9c0a0a7bb..08a20df093c01 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/350_matched_queries.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/350_matched_queries.yml @@ -1,7 +1,7 @@ setup: - skip: - version: " - 2.99.99" - reason: "implemented for versions 3.0.0 and above" + version: " - 2.12.99" + reason: "implemented for versions 2.13.0 and above" --- "matched queries": diff --git a/server/src/main/java/org/opensearch/search/SearchHit.java b/server/src/main/java/org/opensearch/search/SearchHit.java index a3db7e0893a3d..6391353cfe5b1 100644 --- a/server/src/main/java/org/opensearch/search/SearchHit.java +++ b/server/src/main/java/org/opensearch/search/SearchHit.java @@ -205,7 +205,7 @@ public SearchHit(StreamInput in) throws IOException { sortValues = new SearchSortValues(in); size = in.readVInt(); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_13_0)) { if (size > 0) { Map tempMap = in.readMap(StreamInput::readString, StreamInput::readFloat); matchedQueries = tempMap.entrySet() @@ -269,7 +269,7 @@ public void writeTo(StreamOutput out) throws IOException { sortValues.writeTo(out); out.writeVInt(matchedQueries.size()); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_13_0)) { if (!matchedQueries.isEmpty()) { out.writeMap(matchedQueries, StreamOutput::writeString, StreamOutput::writeFloat); } diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java index bdd92a5baa115..1a5a9dc6d1f03 100644 --- a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java @@ -279,7 +279,7 @@ public SearchSourceBuilder(StreamInput in) throws IOException { searchPipelineSource = in.readMap(); } } - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_13_0)) { includeNamedQueriesScore = in.readOptionalBoolean(); } } @@ -347,7 +347,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(searchPipelineSource); } } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_13_0)) { out.writeOptionalBoolean(includeNamedQueriesScore); } } From 7a11720099706f992e4f3e6d72f2c30c5a18fb24 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 09:32:33 -0500 Subject: [PATCH 35/41] Bump org.apache.logging.log4j:log4j-core from 2.22.1 to 2.23.0 in /buildSrc/src/testKit/thirdPartyAudit (#12464) * Bump org.apache.logging.log4j:log4j-core Bumps org.apache.logging.log4j:log4j-core from 2.22.1 to 2.23.0. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5121a067144b..0888191b95b83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -113,6 +113,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `netty` from 4.1.106.Final to 4.1.107.Final ([#12372](https://github.com/opensearch-project/OpenSearch/pull/12372)) - Bump `opentelemetry` from 1.34.1 to 1.35.0 ([#12388](https://github.com/opensearch-project/OpenSearch/pull/12388)) - Bump Apache Lucene from 9.9.2 to 9.10.0 ([#12392](https://github.com/opensearch-project/OpenSearch/pull/12392)) +- Bump `org.apache.logging.log4j:log4j-core` from 2.22.1 to 2.23.0 ([#12464](https://github.com/opensearch-project/OpenSearch/pull/12464)) ### Changed - Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499)) diff --git a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle index 351b42e5bc921..4b8f52ec07615 100644 --- a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle +++ b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle @@ -17,7 +17,7 @@ repositories { } dependencies { - implementation "org.apache.logging.log4j:log4j-core:2.22.1" + implementation "org.apache.logging.log4j:log4j-core:2.23.0" } ["0.0.1", "0.0.2"].forEach { v -> From 24d7098fbadfe5a220b44fc029352a955b72d96e Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 26 Feb 2024 11:36:09 -0500 Subject: [PATCH 36/41] Bump antlr4 from 4.11.1 to 4.13.1 (#12445) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- .../licenses/antlr4-runtime-4.11.1.jar.sha1 | 1 - .../licenses/antlr4-runtime-4.13.1.jar.sha1 | 1 + .../licenses/antlr4-runtime-4.11.1.jar.sha1 | 1 - .../licenses/antlr4-runtime-4.13.1.jar.sha1 | 1 + .../painless/antlr/PainlessLexer.java | 4 +- .../painless/antlr/PainlessParser.java | 52 +++++++++---------- 8 files changed, 32 insertions(+), 31 deletions(-) delete mode 100644 modules/lang-expression/licenses/antlr4-runtime-4.11.1.jar.sha1 create mode 100644 modules/lang-expression/licenses/antlr4-runtime-4.13.1.jar.sha1 delete mode 100644 modules/lang-painless/licenses/antlr4-runtime-4.11.1.jar.sha1 create mode 100644 modules/lang-painless/licenses/antlr4-runtime-4.13.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0888191b95b83..fa3c3e5fc64a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,6 +114,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `opentelemetry` from 1.34.1 to 1.35.0 ([#12388](https://github.com/opensearch-project/OpenSearch/pull/12388)) - Bump Apache Lucene from 9.9.2 to 9.10.0 ([#12392](https://github.com/opensearch-project/OpenSearch/pull/12392)) - Bump `org.apache.logging.log4j:log4j-core` from 2.22.1 to 2.23.0 ([#12464](https://github.com/opensearch-project/OpenSearch/pull/12464)) +- Bump `antlr4` from 4.11.1 to 4.13.1 ([#12445](https://github.com/opensearch-project/OpenSearch/pull/12445)) ### Changed - Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 7fc74aff552be..4e315004584ef 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -18,7 +18,7 @@ asm = 9.6 jettison = 1.5.4 woodstox = 6.4.0 kotlin = 1.7.10 -antlr4 = 4.11.1 +antlr4 = 4.13.1 guava = 32.1.1-jre protobuf = 3.22.3 jakarta_annotation = 1.3.5 diff --git a/modules/lang-expression/licenses/antlr4-runtime-4.11.1.jar.sha1 b/modules/lang-expression/licenses/antlr4-runtime-4.11.1.jar.sha1 deleted file mode 100644 index f1b328a6de624..0000000000000 --- a/modules/lang-expression/licenses/antlr4-runtime-4.11.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -069214c1de1960040729702eb58deac8827135e7 \ No newline at end of file diff --git a/modules/lang-expression/licenses/antlr4-runtime-4.13.1.jar.sha1 b/modules/lang-expression/licenses/antlr4-runtime-4.13.1.jar.sha1 new file mode 100644 index 0000000000000..e50b9bb646727 --- /dev/null +++ b/modules/lang-expression/licenses/antlr4-runtime-4.13.1.jar.sha1 @@ -0,0 +1 @@ +17125bae1d965624e265ef49552f6465a2bfa307 \ No newline at end of file diff --git a/modules/lang-painless/licenses/antlr4-runtime-4.11.1.jar.sha1 b/modules/lang-painless/licenses/antlr4-runtime-4.11.1.jar.sha1 deleted file mode 100644 index f1b328a6de624..0000000000000 --- a/modules/lang-painless/licenses/antlr4-runtime-4.11.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -069214c1de1960040729702eb58deac8827135e7 \ No newline at end of file diff --git a/modules/lang-painless/licenses/antlr4-runtime-4.13.1.jar.sha1 b/modules/lang-painless/licenses/antlr4-runtime-4.13.1.jar.sha1 new file mode 100644 index 0000000000000..e50b9bb646727 --- /dev/null +++ b/modules/lang-painless/licenses/antlr4-runtime-4.13.1.jar.sha1 @@ -0,0 +1 @@ +17125bae1d965624e265ef49552f6465a2bfa307 \ No newline at end of file diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/opensearch/painless/antlr/PainlessLexer.java index 260a2fc0c062c..6e3448e5eea77 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/antlr/PainlessLexer.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/antlr/PainlessLexer.java @@ -45,10 +45,10 @@ import org.antlr.v4.runtime.atn.PredictionContextCache; import org.antlr.v4.runtime.dfa.DFA; -@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue" }) +@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue", "this-escape" }) abstract class PainlessLexer extends Lexer { static { - RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION); + RuntimeMetaData.checkVersion("4.13.1", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/opensearch/painless/antlr/PainlessParser.java index 40e76194f50b2..7ad5d113637c8 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/antlr/PainlessParser.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/antlr/PainlessParser.java @@ -56,7 +56,7 @@ @SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue" }) class PainlessParser extends Parser { static { - RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION); + RuntimeMetaData.checkVersion("4.13.1", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; @@ -337,7 +337,7 @@ public Vocabulary getVocabulary() { @Override public String getGrammarFileName() { - return "java-escape"; + return "PainlessParser.g4"; } @Override @@ -425,8 +425,8 @@ public final SourceContext source() throws RecognitionException { setState(87); _errHandler.sync(this); _la = _input.LA(1); - while (((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155080519840L) != 0 - || (((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 4095L) != 0) { + while ((((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155080519840L) != 0) + || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 4095L) != 0)) { { { setState(84); @@ -571,7 +571,7 @@ public final ParametersContext parameters() throws RecognitionException { setState(109); _errHandler.sync(this); _la = _input.LA(1); - if ((((_la - 81)) & ~0x3f) == 0 && ((1L << (_la - 81)) & 7L) != 0) { + if (((((_la - 81)) & ~0x3f) == 0 && ((1L << (_la - 81)) & 7L) != 0)) { { setState(98); decltype(); @@ -1088,8 +1088,8 @@ public final RstatementContext rstatement() throws RecognitionException { setState(140); _errHandler.sync(this); _la = _input.LA(1); - if (((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155034439840L) != 0 - || (((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 4095L) != 0) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155034439840L) != 0) + || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 4095L) != 0)) { { setState(139); initializer(); @@ -1101,8 +1101,8 @@ public final RstatementContext rstatement() throws RecognitionException { setState(144); _errHandler.sync(this); _la = _input.LA(1); - if (((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155034439840L) != 0 - || (((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 2559L) != 0) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155034439840L) != 0) + || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 2559L) != 0)) { { setState(143); expression(); @@ -1114,8 +1114,8 @@ public final RstatementContext rstatement() throws RecognitionException { setState(148); _errHandler.sync(this); _la = _input.LA(1); - if (((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155034439840L) != 0 - || (((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 2559L) != 0) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155034439840L) != 0) + || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 2559L) != 0)) { { setState(147); afterthought(); @@ -1470,8 +1470,8 @@ public final DstatementContext dstatement() throws RecognitionException { setState(193); _errHandler.sync(this); _la = _input.LA(1); - if (((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155034439840L) != 0 - || (((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 2559L) != 0) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155034439840L) != 0) + || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 2559L) != 0)) { { setState(192); expression(); @@ -1661,8 +1661,8 @@ public final BlockContext block() throws RecognitionException { setState(212); _errHandler.sync(this); _la = _input.LA(1); - if (((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155071795360L) != 0 - || (((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 4095L) != 0) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155071795360L) != 0) + || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 4095L) != 0)) { { setState(211); dstatement(); @@ -2491,7 +2491,7 @@ private NoncondexpressionContext noncondexpression(int _p) throws RecognitionExc if (!(precpred(_ctx, 13))) throw new FailedPredicateException(this, "precpred(_ctx, 13)"); setState(269); _la = _input.LA(1); - if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & 7516192768L) != 0)) { + if (!((((_la) & ~0x3f) == 0 && ((1L << _la) & 7516192768L) != 0))) { _errHandler.recoverInline(this); } else { if (_input.LA(1) == Token.EOF) matchedEOF = true; @@ -2545,7 +2545,7 @@ private NoncondexpressionContext noncondexpression(int _p) throws RecognitionExc if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)"); setState(278); _la = _input.LA(1); - if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & 240518168576L) != 0)) { + if (!((((_la) & ~0x3f) == 0 && ((1L << _la) & 240518168576L) != 0))) { _errHandler.recoverInline(this); } else { if (_input.LA(1) == Token.EOF) matchedEOF = true; @@ -2563,7 +2563,7 @@ private NoncondexpressionContext noncondexpression(int _p) throws RecognitionExc if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)"); setState(281); _la = _input.LA(1); - if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & 4123168604160L) != 0)) { + if (!((((_la) & ~0x3f) == 0 && ((1L << _la) & 4123168604160L) != 0))) { _errHandler.recoverInline(this); } else { if (_input.LA(1) == Token.EOF) matchedEOF = true; @@ -2581,7 +2581,7 @@ private NoncondexpressionContext noncondexpression(int _p) throws RecognitionExc if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)"); setState(284); _la = _input.LA(1); - if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & 65970697666560L) != 0)) { + if (!((((_la) & ~0x3f) == 0 && ((1L << _la) & 65970697666560L) != 0))) { _errHandler.recoverInline(this); } else { if (_input.LA(1) == Token.EOF) matchedEOF = true; @@ -2861,7 +2861,7 @@ public final ExpressionContext expression() throws RecognitionException { noncondexpression(0); setState(320); _la = _input.LA(1); - if (!((((_la - 60)) & ~0x3f) == 0 && ((1L << (_la - 60)) & 4095L) != 0)) { + if (!(((((_la - 60)) & ~0x3f) == 0 && ((1L << (_la - 60)) & 4095L) != 0))) { _errHandler.recoverInline(this); } else { if (_input.LA(1) == Token.EOF) matchedEOF = true; @@ -3938,7 +3938,7 @@ public final PrimaryContext primary() throws RecognitionException { enterOuterAlt(_localctx, 2); { setState(400); _la = _input.LA(1); - if (!((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 15L) != 0)) { + if (!(((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 15L) != 0))) { _errHandler.recoverInline(this); } else { if (_input.LA(1) == Token.EOF) matchedEOF = true; @@ -4564,8 +4564,8 @@ public final ArrayinitializerContext arrayinitializer() throws RecognitionExcept setState(469); _errHandler.sync(this); _la = _input.LA(1); - if (((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155034439840L) != 0 - || (((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 2559L) != 0) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155034439840L) != 0) + || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 2559L) != 0)) { { setState(461); expression(); @@ -4923,8 +4923,8 @@ public final ArgumentsContext arguments() throws RecognitionException { setState(524); _errHandler.sync(this); _la = _input.LA(1); - if (((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155101548704L) != 0 - || (((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 4095L) != 0) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 864691155101548704L) != 0) + || ((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & 4095L) != 0)) { { setState(516); argument(); @@ -5104,7 +5104,7 @@ public final LambdaContext lambda() throws RecognitionException { setState(543); _errHandler.sync(this); _la = _input.LA(1); - if ((((_la - 81)) & ~0x3f) == 0 && ((1L << (_la - 81)) & 7L) != 0) { + if (((((_la - 81)) & ~0x3f) == 0 && ((1L << (_la - 81)) & 7L) != 0)) { { setState(535); lamtype(); From db4245d2b4f82985d547efd71ea2de8b50250ca3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 13:53:26 -0500 Subject: [PATCH 37/41] Bump com.netflix.nebula.ospackage-base from 11.8.0 to 11.8.1 in /distribution/packages (#12461) * Bump com.netflix.nebula.ospackage-base in /distribution/packages Bumps com.netflix.nebula.ospackage-base from 11.8.0 to 11.8.1. --- updated-dependencies: - dependency-name: com.netflix.nebula.ospackage-base dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + distribution/packages/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa3c3e5fc64a9..41b113134ab66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -115,6 +115,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump Apache Lucene from 9.9.2 to 9.10.0 ([#12392](https://github.com/opensearch-project/OpenSearch/pull/12392)) - Bump `org.apache.logging.log4j:log4j-core` from 2.22.1 to 2.23.0 ([#12464](https://github.com/opensearch-project/OpenSearch/pull/12464)) - Bump `antlr4` from 4.11.1 to 4.13.1 ([#12445](https://github.com/opensearch-project/OpenSearch/pull/12445)) +- Bump `com.netflix.nebula.ospackage-base` from 11.8.0 to 11.8.1 ([#12461](https://github.com/opensearch-project/OpenSearch/pull/12461)) ### Changed - Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499)) diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 43c38c5ad0c67..4e85d19986e43 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -63,7 +63,7 @@ import java.util.regex.Pattern */ plugins { - id "com.netflix.nebula.ospackage-base" version "11.8.0" + id "com.netflix.nebula.ospackage-base" version "11.8.1" } void addProcessFilesTask(String type, boolean jdk) { From 0bd3ccd80c92f296d18537d811039eedb4d2a929 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 15:04:18 -0500 Subject: [PATCH 38/41] Bump peter-evans/create-or-update-comment from 3 to 4 (#12462) * Bump peter-evans/create-or-update-comment from 3 to 4 Bumps [peter-evans/create-or-update-comment](https://github.com/peter-evans/create-or-update-comment) from 3 to 4. - [Release notes](https://github.com/peter-evans/create-or-update-comment/releases) - [Commits](https://github.com/peter-evans/create-or-update-comment/compare/v3...v4) --- updated-dependencies: - dependency-name: peter-evans/create-or-update-comment dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/check-compatibility.yml | 2 +- .github/workflows/gradle-check.yml | 6 +++--- .github/workflows/poc-checklist.yml | 2 +- CHANGELOG.md | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/check-compatibility.yml b/.github/workflows/check-compatibility.yml index 1ad44e6334059..b2f22a90938cc 100644 --- a/.github/workflows/check-compatibility.yml +++ b/.github/workflows/check-compatibility.yml @@ -61,7 +61,7 @@ jobs: body-includes: 'Compatibility status:' - name: Add comment on the PR - uses: peter-evans/create-or-update-comment@v3 + uses: peter-evans/create-or-update-comment@v4 with: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.number }} diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml index 8c33d41c6b2b4..8ac44cc37d27c 100644 --- a/.github/workflows/gradle-check.yml +++ b/.github/workflows/gradle-check.yml @@ -78,7 +78,7 @@ jobs: - name: Create Comment Success if: ${{ github.event_name == 'pull_request_target' && success() && env.result == 'SUCCESS' }} - uses: peter-evans/create-or-update-comment@v3 + uses: peter-evans/create-or-update-comment@v4 with: issue-number: ${{ env.pr_number }} body: | @@ -101,7 +101,7 @@ jobs: - name: Create Comment Flaky if: ${{ github.event_name == 'pull_request_target' && success() && env.result != 'SUCCESS' }} - uses: peter-evans/create-or-update-comment@v3 + uses: peter-evans/create-or-update-comment@v4 with: issue-number: ${{ env.pr_number }} body: | @@ -111,7 +111,7 @@ jobs: - name: Create Comment Failure if: ${{ github.event_name == 'pull_request_target' && failure() }} - uses: peter-evans/create-or-update-comment@v3 + uses: peter-evans/create-or-update-comment@v4 with: issue-number: ${{ env.pr_number }} body: | diff --git a/.github/workflows/poc-checklist.yml b/.github/workflows/poc-checklist.yml index 3d014e000a487..1b4f6b31e02f8 100644 --- a/.github/workflows/poc-checklist.yml +++ b/.github/workflows/poc-checklist.yml @@ -11,7 +11,7 @@ jobs: issues: write steps: - name: Add comment - uses: peter-evans/create-or-update-comment@v3 + uses: peter-evans/create-or-update-comment@v4 with: issue-number: ${{ github.event.issue.number }} body: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 41b113134ab66..56e185a181e50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.logging.log4j:log4j-core` from 2.22.1 to 2.23.0 ([#12464](https://github.com/opensearch-project/OpenSearch/pull/12464)) - Bump `antlr4` from 4.11.1 to 4.13.1 ([#12445](https://github.com/opensearch-project/OpenSearch/pull/12445)) - Bump `com.netflix.nebula.ospackage-base` from 11.8.0 to 11.8.1 ([#12461](https://github.com/opensearch-project/OpenSearch/pull/12461)) +- Bump `peter-evans/create-or-update-comment` from 3 to 4 ([#12462](https://github.com/opensearch-project/OpenSearch/pull/12462)) ### Changed - Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499)) From 8d84eea68d9ed7dedcccc8161569d5b4d2ee4eec Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Tue, 27 Feb 2024 06:46:54 +0800 Subject: [PATCH 39/41] Fix get task API does not refresh resource stats (#11531) * Fix get task API does not refresh resource stats Signed-off-by: Gao Binlong * modify change log Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- CHANGELOG.md | 1 + .../tasks/get/TransportGetTaskAction.java | 8 ++- .../node/tasks/ResourceAwareTasksTests.java | 55 ++++++++++++++++++- .../node/tasks/TaskManagerTestCase.java | 16 ++++++ 4 files changed, 77 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56e185a181e50..7f025d2304966 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -133,6 +133,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Prevent read beyond slice boundary in ByteArrayIndexInput ([#10481](https://github.com/opensearch-project/OpenSearch/issues/10481)) - Fix the "highlight.max_analyzer_offset" request parameter with "plain" highlighter ([#10919](https://github.com/opensearch-project/OpenSearch/pull/10919)) - Warn about deprecated and ignored index.mapper.dynamic index setting ([#11193](https://github.com/opensearch-project/OpenSearch/pull/11193)) +- Fix get task API does not refresh resource stats ([#11531](https://github.com/opensearch-project/OpenSearch/pull/11531)) ### Security diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java index e62c83490d810..ab6451382aa88 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java @@ -55,6 +55,7 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskInfo; +import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.tasks.TaskResult; import org.opensearch.tasks.TaskResultsService; import org.opensearch.threadpool.ThreadPool; @@ -84,6 +85,8 @@ public class TransportGetTaskAction extends HandledTransportAction 0); + }; + + taskTestContext.operationFinishedValidator = (task, threadId) -> { assertEquals(0, resourceTasks.size()); }; + + startResourceAwareNodesAction(testNodes[0], false, taskTestContext, new ActionListener() { + @Override + public void onResponse(NodesResponse listTasksResponse) { + responseReference.set(listTasksResponse); + taskTestContext.requestCompleteLatch.countDown(); + } + + @Override + public void onFailure(Exception e) { + throwableReference.set(e); + taskTestContext.requestCompleteLatch.countDown(); + } + }); + + // Waiting for whole request to complete and return successfully till client + taskTestContext.requestCompleteLatch.await(); + + assertTasksRequestFinishedSuccessfully(responseReference.get(), throwableReference.get()); + } + + public void testOnDemandRefreshWhileGetTask() throws InterruptedException { + setup(true, false); + + final AtomicReference throwableReference = new AtomicReference<>(); + final AtomicReference responseReference = new AtomicReference<>(); + + TaskTestContext taskTestContext = new TaskTestContext(); + + Map resourceTasks = testNodes[0].taskResourceTrackingService.getResourceAwareTasks(); + + taskTestContext.operationStartValidator = (task, threadId) -> { + assertFalse(resourceTasks.isEmpty()); + GetTaskResponse getTaskResponse = ActionTestUtils.executeBlocking( + testNodes[0].transportGetTaskAction, + new GetTaskRequest().setTaskId(new TaskId(testNodes[0].getNodeId(), new ArrayList<>(resourceTasks.values()).get(0).getId())) + ); + + TaskInfo taskInfo = getTaskResponse.getTask().getTask(); + + assertNotNull(taskInfo.getResourceStats()); + assertNotNull(taskInfo.getResourceStats().getResourceUsageInfo()); + assertNotNull(taskInfo.getResourceStats().getResourceUsageInfo().get("total")); + TaskResourceUsage taskResourceUsage = taskInfo.getResourceStats().getResourceUsageInfo().get("total"); assertCPUTime(taskResourceUsage.getCpuTimeInNanos()); assertTrue(taskResourceUsage.getMemoryInBytes() > 0); }; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java index a3fa0f9cb16e4..8d87fd5135663 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java @@ -34,6 +34,7 @@ import org.opensearch.Version; import org.opensearch.action.FailedNodeException; import org.opensearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction; +import org.opensearch.action.admin.cluster.node.tasks.get.TransportGetTaskAction; import org.opensearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.nodes.BaseNodeResponse; @@ -41,6 +42,7 @@ import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.action.support.nodes.TransportNodesAction; import org.opensearch.action.support.replication.ClusterStateCreationUtils; +import org.opensearch.client.Client; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.node.DiscoveryNode; @@ -57,6 +59,7 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.transport.BoundTransportAddress; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.tasks.TaskCancellationService; import org.opensearch.tasks.TaskManager; import org.opensearch.tasks.TaskResourceTrackingService; @@ -85,6 +88,7 @@ import static java.util.Collections.emptySet; import static org.opensearch.test.ClusterServiceUtils.createClusterService; import static org.opensearch.test.ClusterServiceUtils.setState; +import static org.mockito.Mockito.mock; /** * The test case for unit testing task manager and related transport actions @@ -249,6 +253,17 @@ protected TaskManager createTaskManager( taskResourceTrackingService ); transportCancelTasksAction = new TransportCancelTasksAction(clusterService, transportService, actionFilters); + Client mockClient = mock(Client.class); + NamedXContentRegistry namedXContentRegistry = mock(NamedXContentRegistry.class); + transportGetTaskAction = new TransportGetTaskAction( + threadPool, + transportService, + actionFilters, + clusterService, + mockClient, + namedXContentRegistry, + taskResourceTrackingService + ); transportService.acceptIncomingRequests(); } @@ -258,6 +273,7 @@ protected TaskManager createTaskManager( private final SetOnce discoveryNode = new SetOnce<>(); public final TransportListTasksAction transportListTasksAction; public final TransportCancelTasksAction transportCancelTasksAction; + public final TransportGetTaskAction transportGetTaskAction; @Override public void close() { From 79211755df1a39e292d6c77f39ef72b49bea67f2 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 27 Feb 2024 12:47:45 -0500 Subject: [PATCH 40/41] Bump Apache Lucene to 9.11.0-snapshot-8a555eb (#12478) Signed-off-by: Andriy Redko --- buildSrc/version.properties | 2 +- libs/core/licenses/lucene-core-9.10.0.jar.sha1 | 1 - libs/core/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + libs/core/src/main/java/org/opensearch/Version.java | 2 +- .../lang-expression/licenses/lucene-expressions-9.10.0.jar.sha1 | 1 - .../lucene-expressions-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + .../analysis-icu/licenses/lucene-analysis-icu-9.10.0.jar.sha1 | 1 - .../lucene-analysis-icu-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + .../licenses/lucene-analysis-kuromoji-9.10.0.jar.sha1 | 1 - .../lucene-analysis-kuromoji-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + .../analysis-nori/licenses/lucene-analysis-nori-9.10.0.jar.sha1 | 1 - .../lucene-analysis-nori-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + .../licenses/lucene-analysis-phonetic-9.10.0.jar.sha1 | 1 - .../lucene-analysis-phonetic-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + .../licenses/lucene-analysis-smartcn-9.10.0.jar.sha1 | 1 - .../lucene-analysis-smartcn-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + .../licenses/lucene-analysis-stempel-9.10.0.jar.sha1 | 1 - .../lucene-analysis-stempel-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + .../licenses/lucene-analysis-morfologik-9.10.0.jar.sha1 | 1 - .../lucene-analysis-morfologik-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-analysis-common-9.10.0.jar.sha1 | 1 - .../lucene-analysis-common-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-backward-codecs-9.10.0.jar.sha1 | 1 - .../lucene-backward-codecs-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-core-9.10.0.jar.sha1 | 1 - server/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-grouping-9.10.0.jar.sha1 | 1 - .../licenses/lucene-grouping-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-highlighter-9.10.0.jar.sha1 | 1 - .../lucene-highlighter-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-join-9.10.0.jar.sha1 | 1 - server/licenses/lucene-join-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-memory-9.10.0.jar.sha1 | 1 - server/licenses/lucene-memory-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-misc-9.10.0.jar.sha1 | 1 - server/licenses/lucene-misc-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-queries-9.10.0.jar.sha1 | 1 - server/licenses/lucene-queries-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-queryparser-9.10.0.jar.sha1 | 1 - .../lucene-queryparser-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-sandbox-9.10.0.jar.sha1 | 1 - server/licenses/lucene-sandbox-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-spatial-extras-9.10.0.jar.sha1 | 1 - .../lucene-spatial-extras-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-spatial3d-9.10.0.jar.sha1 | 1 - .../licenses/lucene-spatial3d-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + server/licenses/lucene-suggest-9.10.0.jar.sha1 | 1 - server/licenses/lucene-suggest-9.11.0-snapshot-8a555eb.jar.sha1 | 1 + 48 files changed, 25 insertions(+), 25 deletions(-) delete mode 100644 libs/core/licenses/lucene-core-9.10.0.jar.sha1 create mode 100644 libs/core/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.10.0.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.10.0.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.10.0.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.10.0.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.10.0.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.10.0.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.10.0.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.10.0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-join-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.11.0-snapshot-8a555eb.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.10.0.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.11.0-snapshot-8a555eb.jar.sha1 diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 4e315004584ef..39b0335c7ef55 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.10.0 +lucene = 9.11.0-snapshot-8a555eb bundled_jdk_vendor = adoptium bundled_jdk = 21.0.2+13 diff --git a/libs/core/licenses/lucene-core-9.10.0.jar.sha1 b/libs/core/licenses/lucene-core-9.10.0.jar.sha1 deleted file mode 100644 index 31b6bcd29f418..0000000000000 --- a/libs/core/licenses/lucene-core-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -64e5624754d59386be5d9159c68f81ff96298704 \ No newline at end of file diff --git a/libs/core/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1 b/libs/core/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..25beb34873c0c --- /dev/null +++ b/libs/core/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +fe1cf5663be8bdb6aa757fd4101bc551684c90fb \ No newline at end of file diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 1bc0583d5e7df..66ba446d4fc54 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -101,7 +101,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_9_2); public static final Version V_2_12_1 = new Version(2120199, org.apache.lucene.util.Version.LUCENE_9_9_2); public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_10_0); - public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_10_0); + public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_11_0); public static final Version CURRENT = V_3_0_0; public static Version fromId(int id) { diff --git a/modules/lang-expression/licenses/lucene-expressions-9.10.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.10.0.jar.sha1 deleted file mode 100644 index 4c85d77f41cd6..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e2daf2e568fe120e84567505f7c85643bb0b9dd2 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-8a555eb.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..82a17e2b79290 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +00759eaff8f62b38ba66a05f26ab784c268908d3 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.10.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.10.0.jar.sha1 deleted file mode 100644 index a6f523b96685c..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ed54097d13dbaae6e758c31f55a473ba6566ae8d \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..0643f16dc1052 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +44a4e095d7e047a9452d81b224905b72c830f8ae \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.10.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.10.0.jar.sha1 deleted file mode 100644 index 0572d78c75516..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a158d8a7d386c782a10ec15827ff3f0aa6fd8291 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..3f4d49a78791b --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +7f57fe7322e6d3a9b4edcc3da0b1ee0791a814ec \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.10.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.10.0.jar.sha1 deleted file mode 100644 index b81b33d7b28f7..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -79eb4c5e1c92b613341d05173dfcdce182fa5f80 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..1f110011ca9c6 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +9929da235100f8df323cfed165b8111fb2840093 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.10.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.10.0.jar.sha1 deleted file mode 100644 index 66245cc1d36d9..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b6ba8f0ecee48b94911c08ee12c20f059ec38ec5 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..8d6bf9fa0fa1b --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +8564c86d880c6ce002250002e2fd0936cbfff61d \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.10.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.10.0.jar.sha1 deleted file mode 100644 index 5f17e786f91d5..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ccf4512816ba9cd5c2cca9182db78a2e48e7e16f \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..cbe4aec98fae4 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +7e71777cfb5beb4ffd5b03030576d2f062eef13c \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.10.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.10.0.jar.sha1 deleted file mode 100644 index f7b19bc3b38cd..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b596817e70f00bc54d66e82c118f47ca91b9e401 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..e1c7aecc104d0 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +a597265bd6fb0a7e954e948a295d31507dd73cce \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.10.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.10.0.jar.sha1 deleted file mode 100644 index 7fb543858c1a7..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7acc74d35684269ffa74909395044e0e02581285 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..eefa2809f3540 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +c9e534845bb08985d7fa21e2e71a14bc68c46089 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.10.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.10.0.jar.sha1 deleted file mode 100644 index d33a965b3506c..0000000000000 --- a/server/licenses/lucene-analysis-common-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -92e559808a23f61c818ef90a9ccab3669a25caa0 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-analysis-common-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..9cab77f4e7394 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +6f0cfa07a5e4b36423e398cd1fd51c6825773d9c \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.10.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.10.0.jar.sha1 deleted file mode 100644 index 01b6cbb54f3e9..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6570ebf974d07025ad4cd9ffaa9927546b534704 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-backward-codecs-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..2d5d1a281a0f0 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +ab201b997c8449db1ecd2fa88bd42d2f457286fa \ No newline at end of file diff --git a/server/licenses/lucene-core-9.10.0.jar.sha1 b/server/licenses/lucene-core-9.10.0.jar.sha1 deleted file mode 100644 index 31b6bcd29f418..0000000000000 --- a/server/licenses/lucene-core-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -64e5624754d59386be5d9159c68f81ff96298704 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..25beb34873c0c --- /dev/null +++ b/server/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +fe1cf5663be8bdb6aa757fd4101bc551684c90fb \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.10.0.jar.sha1 b/server/licenses/lucene-grouping-9.10.0.jar.sha1 deleted file mode 100644 index 21f2062273355..0000000000000 --- a/server/licenses/lucene-grouping-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b82838299e4fa423fc574c1c07405410823f74ce \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-grouping-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..27d4f0f5874e9 --- /dev/null +++ b/server/licenses/lucene-grouping-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +a988f92842e48195c75a49377432533c9170d93d \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.10.0.jar.sha1 b/server/licenses/lucene-highlighter-9.10.0.jar.sha1 deleted file mode 100644 index 2148a8ef957f0..0000000000000 --- a/server/licenses/lucene-highlighter-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2f21ade4b4896f1ece2b3a823e1640c762c9d0cf \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-highlighter-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..2545822f2ac7b --- /dev/null +++ b/server/licenses/lucene-highlighter-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +7c669e2c01565d3bdf175cd61a1e4d0bdfc44311 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.10.0.jar.sha1 b/server/licenses/lucene-join-9.10.0.jar.sha1 deleted file mode 100644 index 93b8f50522800..0000000000000 --- a/server/licenses/lucene-join-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -541b722bb842817e2ec899cd4d0de2a136953f53 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-join-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..fe44ad772335f --- /dev/null +++ b/server/licenses/lucene-join-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +633a6d44b4cde8e149daa3407e8b8f644eece951 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.10.0.jar.sha1 b/server/licenses/lucene-memory-9.10.0.jar.sha1 deleted file mode 100644 index 82ca74ccdfc42..0000000000000 --- a/server/licenses/lucene-memory-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -06620029b6c7d84480fb7ad1ba0d72963111b765 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-memory-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..85bfbe066ff56 --- /dev/null +++ b/server/licenses/lucene-memory-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +25390259c9e5592354efbc2f250bb396402016b2 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.10.0.jar.sha1 b/server/licenses/lucene-misc-9.10.0.jar.sha1 deleted file mode 100644 index 2d69a4cfa1148..0000000000000 --- a/server/licenses/lucene-misc-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3621206c534d253a0205f8a0293c146ff34d8b03 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-misc-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..4dba5837b66de --- /dev/null +++ b/server/licenses/lucene-misc-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +056f87a3d30c223b08d2f45fe465ddf11210b85f \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.10.0.jar.sha1 b/server/licenses/lucene-queries-9.10.0.jar.sha1 deleted file mode 100644 index d7c38977e4fc8..0000000000000 --- a/server/licenses/lucene-queries-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bbe2360c60a43bf41fbf05d5f0b4f471547217ba \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-queries-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..b6b8c441eefb1 --- /dev/null +++ b/server/licenses/lucene-queries-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +e640f850b4fb13190be8422fe74c14c9d6603bb5 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.10.0.jar.sha1 b/server/licenses/lucene-queryparser-9.10.0.jar.sha1 deleted file mode 100644 index b02a3cd926375..0000000000000 --- a/server/licenses/lucene-queryparser-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c50f82d244ea5adac2d2d9295de85ddccc2d45cb \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-queryparser-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..d0e77b04db51a --- /dev/null +++ b/server/licenses/lucene-queryparser-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +8eb57762bf408fa51d7511f5e3b917627be61d1d \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.10.0.jar.sha1 b/server/licenses/lucene-sandbox-9.10.0.jar.sha1 deleted file mode 100644 index 74b9e925398b1..0000000000000 --- a/server/licenses/lucene-sandbox-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -80bc56b23b34cddab97557e5ada7cd0558dbf5e5 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-sandbox-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..179df9f07a594 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +615b4a90c5402084c2d5916a4c1fadc9d9177782 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.10.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.10.0.jar.sha1 deleted file mode 100644 index 76e9fea920bb8..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -055e6aa874b6ec9ddafc2e115055de1445c2df49 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-spatial-extras-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..9b88b24c21b12 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +19b42cdb5f76f63dece3ef5128207ebdd3741d48 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.10.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.10.0.jar.sha1 deleted file mode 100644 index 3268b8bd557d0..0000000000000 --- a/server/licenses/lucene-spatial3d-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -06945a8b2a703062ce4b286ecceae7182b4f5f44 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-spatial3d-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..789ab1d52ea8c --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +882691fe917e716fe6bcf8c0dd984b153495d015 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.10.0.jar.sha1 b/server/licenses/lucene-suggest-9.10.0.jar.sha1 deleted file mode 100644 index aaa10c449adb3..0000000000000 --- a/server/licenses/lucene-suggest-9.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d2068ad997165a414927e8c0072d0b88741af069 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-suggest-9.11.0-snapshot-8a555eb.jar.sha1 new file mode 100644 index 0000000000000..8cfaf60763724 --- /dev/null +++ b/server/licenses/lucene-suggest-9.11.0-snapshot-8a555eb.jar.sha1 @@ -0,0 +1 @@ +ef6d483960f776d5dbdd1009863786ee09ba5707 \ No newline at end of file From 3125b948029609f354d3153f8ca6391638daefc7 Mon Sep 17 00:00:00 2001 From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> Date: Tue, 27 Feb 2024 13:03:59 -0800 Subject: [PATCH 41/41] [Searchable Snapshot] Fix bug of Searchable Snapshot Dependency on repository chunk_size (#12277) * implement logic of fetching blocks from multiple chunks of snapshot file. Signed-off-by: Rishikesh1159 * Refactor and address comments. Signed-off-by: Rishikesh1159 * apply spotless check Signed-off-by: Rishikesh1159 * Address comments of using a different data structure to fetch blob parts. Signed-off-by: Rishikesh1159 * remove unnecessary code. Signed-off-by: Rishikesh1159 * Refactor outputstream usage. Signed-off-by: Rishikesh1159 * refactor blobpart logic into a separate method and add unit tests. Signed-off-by: Rishikesh1159 * Add new unit tests. Signed-off-by: Rishikesh1159 --------- Signed-off-by: Rishikesh1159 --- .../snapshots/SearchableSnapshotIT.java | 36 ++++++- .../file/OnDemandBlockSnapshotIndexInput.java | 46 ++++++--- .../store/remote/utils/BlobFetchRequest.java | 95 ++++++++++--------- .../store/remote/utils/TransferManager.java | 22 +++-- .../OnDemandBlockSnapshotIndexInputTests.java | 63 ++++++++++-- .../remote/utils/TransferManagerTests.java | 29 ++---- 6 files changed, 196 insertions(+), 95 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java index c89fef20aafb1..90bb2b501764e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java @@ -85,10 +85,10 @@ protected Settings.Builder randomRepositorySettings() { return settings; } - private Settings.Builder chunkedRepositorySettings() { + private Settings.Builder chunkedRepositorySettings(long chunkSize) { final Settings.Builder settings = Settings.builder(); settings.put("location", randomRepoPath()).put("compress", randomBoolean()); - settings.put("chunk_size", 2 << 23, ByteSizeUnit.BYTES); + settings.put("chunk_size", chunkSize, ByteSizeUnit.BYTES); return settings; } @@ -194,10 +194,10 @@ public void testSnapshottingSearchableSnapshots() throws Exception { } /** - * Tests a chunked repository scenario for searchable snapshots by creating an index, + * Tests a default 8mib chunked repository scenario for searchable snapshots by creating an index, * taking a snapshot, restoring it as a searchable snapshot index. */ - public void testCreateSearchableSnapshotWithChunks() throws Exception { + public void testCreateSearchableSnapshotWithDefaultChunks() throws Exception { final int numReplicasIndex = randomIntBetween(1, 4); final String indexName = "test-idx"; final String restoredIndexName = indexName + "-copy"; @@ -205,7 +205,33 @@ public void testCreateSearchableSnapshotWithChunks() throws Exception { final String snapshotName = "test-snap"; final Client client = client(); - Settings.Builder repositorySettings = chunkedRepositorySettings(); + Settings.Builder repositorySettings = chunkedRepositorySettings(2 << 23); + + internalCluster().ensureAtLeastNumSearchAndDataNodes(numReplicasIndex + 1); + createIndexWithDocsAndEnsureGreen(numReplicasIndex, 1000, indexName); + createRepositoryWithSettings(repositorySettings, repoName); + takeSnapshot(client, snapshotName, repoName, indexName); + + deleteIndicesAndEnsureGreen(client, indexName); + restoreSnapshotAndEnsureGreen(client, snapshotName, repoName); + assertRemoteSnapshotIndexSettings(client, restoredIndexName); + + assertDocCount(restoredIndexName, 1000L); + } + + /** + * Tests a small 1000 bytes chunked repository scenario for searchable snapshots by creating an index, + * taking a snapshot, restoring it as a searchable snapshot index. + */ + public void testCreateSearchableSnapshotWithSmallChunks() throws Exception { + final int numReplicasIndex = randomIntBetween(1, 4); + final String indexName = "test-idx"; + final String restoredIndexName = indexName + "-copy"; + final String repoName = "test-repo"; + final String snapshotName = "test-snap"; + final Client client = client(); + + Settings.Builder repositorySettings = chunkedRepositorySettings(1000); internalCluster().ensureAtLeastNumSearchAndDataNodes(numReplicasIndex + 1); createIndexWithDocsAndEnsureGreen(numReplicasIndex, 1000, indexName); diff --git a/server/src/main/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInput.java b/server/src/main/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInput.java index 7166e9aa482e3..8097fd08da50a 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInput.java +++ b/server/src/main/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInput.java @@ -15,6 +15,8 @@ import org.opensearch.index.store.remote.utils.TransferManager; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; /** * This is an implementation of {@link OnDemandBlockIndexInput} where this class provides the main IndexInput using shard snapshot files. @@ -136,25 +138,45 @@ protected IndexInput fetchBlock(int blockId) throws IOException { final long blockStart = getBlockStart(blockId); final long blockEnd = blockStart + getActualBlockSize(blockId); - // If the snapshot file is chunked, we must account for this by - // choosing the appropriate file part and updating the position - // accordingly. - final int part = (int) (blockStart / partSize); - final long partStart = part * partSize; - - final long position = blockStart - partStart; - final long length = blockEnd - blockStart; - + // Block may be present on multiple chunks of a file, so we need + // to fetch each chunk/blob part separately to fetch an entire block. BlobFetchRequest blobFetchRequest = BlobFetchRequest.builder() - .position(position) - .length(length) - .blobName(fileInfo.partName(part)) + .blobParts(getBlobParts(blockStart, blockEnd)) .directory(directory) .fileName(blockFileName) .build(); return transferManager.fetchBlob(blobFetchRequest); } + /** + * Returns list of blob parts/chunks in a file for a given block. + */ + protected List getBlobParts(long blockStart, long blockEnd) { + // If the snapshot file is chunked, we must account for this by + // choosing the appropriate file part and updating the position + // accordingly. + int partNum = (int) (blockStart / partSize); + long pos = blockStart; + long diff = (blockEnd - blockStart); + + List blobParts = new ArrayList<>(); + while (diff > 0) { + long partStart = pos % partSize; + long partEnd; + if ((partStart + diff) > partSize) { + partEnd = partSize; + } else { + partEnd = (partStart + diff); + } + long fetchBytes = partEnd - partStart; + blobParts.add(new BlobFetchRequest.BlobPart(fileInfo.partName(partNum), partStart, fetchBytes)); + partNum++; + pos = pos + fetchBytes; + diff = (blockEnd - pos); + } + return blobParts; + } + @Override public OnDemandBlockSnapshotIndexInput clone() { OnDemandBlockSnapshotIndexInput clone = buildSlice("clone", 0L, this.length); diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/BlobFetchRequest.java b/server/src/main/java/org/opensearch/index/store/remote/utils/BlobFetchRequest.java index d0508e9c6f4c7..f7e6545b5010e 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/utils/BlobFetchRequest.java +++ b/server/src/main/java/org/opensearch/index/store/remote/utils/BlobFetchRequest.java @@ -12,6 +12,7 @@ import org.apache.lucene.store.FSDirectory; import java.nio.file.Path; +import java.util.List; /** * The specification to fetch specific block from blob store @@ -20,37 +21,22 @@ */ public class BlobFetchRequest { - private final long position; - - private final long length; - - private final String blobName; - private final Path filePath; private final Directory directory; private final String fileName; + private final List blobParts; + + private final long blobLength; + private BlobFetchRequest(Builder builder) { - this.position = builder.position; - this.length = builder.length; - this.blobName = builder.blobName; this.fileName = builder.fileName; this.filePath = builder.directory.getDirectory().resolve(fileName); this.directory = builder.directory; - } - - public long getPosition() { - return position; - } - - public long getLength() { - return length; - } - - public String getBlobName() { - return blobName; + this.blobParts = builder.blobParts; + this.blobLength = builder.blobParts.stream().mapToLong(o -> o.getLength()).sum(); } public Path getFilePath() { @@ -65,6 +51,14 @@ public String getFileName() { return fileName; } + public List blobParts() { + return blobParts; + } + + public long getBlobLength() { + return blobLength; + } + public static Builder builder() { return new Builder(); } @@ -72,12 +66,8 @@ public static Builder builder() { @Override public String toString() { return "BlobFetchRequest{" - + "position=" - + position - + ", length=" - + length - + ", blobName='" - + blobName + + "blobParts=" + + blobParts + '\'' + ", filePath=" + filePath @@ -90,35 +80,45 @@ public String toString() { } /** - * Builder for BlobFetchRequest + * BlobPart represents a single chunk of a file */ - public static final class Builder { + public static class BlobPart { + private String blobName; private long position; private long length; - private String blobName; - private FSDirectory directory; - private String fileName; - - private Builder() {} - public Builder position(long position) { - this.position = position; - return this; - } - - public Builder length(long length) { + public BlobPart(String blobName, long position, long length) { + this.blobName = blobName; if (length <= 0) { - throw new IllegalArgumentException("Length for blob fetch request needs to be non-negative"); + throw new IllegalArgumentException("Length for blob part fetch request needs to be non-negative"); } this.length = length; - return this; + this.position = position; } - public Builder blobName(String blobName) { - this.blobName = blobName; - return this; + public String getBlobName() { + return blobName; + } + + public long getPosition() { + return position; } + public long getLength() { + return length; + } + } + + /** + * Builder for BlobFetchRequest + */ + public static final class Builder { + private List blobParts; + private FSDirectory directory; + private String fileName; + + private Builder() {} + public Builder directory(FSDirectory directory) { this.directory = directory; return this; @@ -129,6 +129,11 @@ public Builder fileName(String fileName) { return this; } + public Builder blobParts(List blobParts) { + this.blobParts = blobParts; + return this; + } + public BlobFetchRequest build() { return new BlobFetchRequest(this); } diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java b/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java index 9250e73e08509..98cad7bfadb09 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java +++ b/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java @@ -48,11 +48,12 @@ public TransferManager(final BlobContainer blobContainer, final FileCache fileCa } /** - * Given a blobFetchRequest, return it's corresponding IndexInput. + * Given a blobFetchRequestList, return it's corresponding IndexInput. * @param blobFetchRequest to fetch * @return future of IndexInput augmented with internal caching maintenance tasks */ public IndexInput fetchBlob(BlobFetchRequest blobFetchRequest) throws IOException { + final Path key = blobFetchRequest.getFilePath(); final CachedIndexInput cacheEntry = fileCache.compute(key, (path, cachedIndexInput) -> { @@ -85,15 +86,20 @@ private static FileCachedIndexInput createIndexInput(FileCache fileCache, BlobCo try { if (Files.exists(request.getFilePath()) == false) { try ( - InputStream snapshotFileInputStream = blobContainer.readBlob( - request.getBlobName(), - request.getPosition(), - request.getLength() - ); OutputStream fileOutputStream = Files.newOutputStream(request.getFilePath()); OutputStream localFileOutputStream = new BufferedOutputStream(fileOutputStream) ) { - snapshotFileInputStream.transferTo(localFileOutputStream); + for (BlobFetchRequest.BlobPart blobPart : request.blobParts()) { + try ( + InputStream snapshotFileInputStream = blobContainer.readBlob( + blobPart.getBlobName(), + blobPart.getPosition(), + blobPart.getLength() + ); + ) { + snapshotFileInputStream.transferTo(localFileOutputStream); + } + } } } final IndexInput luceneIndexInput = request.getDirectory().openInput(request.getFileName(), IOContext.READ); @@ -153,7 +159,7 @@ public IndexInput getIndexInput() throws IOException { @Override public long length() { - return request.getLength(); + return request.getBlobLength(); } @Override diff --git a/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java b/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java index 2204124f1de4f..a135802c5f49c 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java @@ -78,11 +78,31 @@ public void test4MBBlock() throws Exception { runAllTestsFor(22); } - public void testChunkedRepository() throws IOException { - final long blockSize = new ByteSizeValue(1, ByteSizeUnit.KB).getBytes(); - final long repositoryChunkSize = new ByteSizeValue(2, ByteSizeUnit.KB).getBytes(); - final long fileSize = new ByteSizeValue(3, ByteSizeUnit.KB).getBytes(); + public void testChunkedRepositoryWithBlockSizeGreaterThanChunkSize() throws IOException { + verifyChunkedRepository( + new ByteSizeValue(8, ByteSizeUnit.KB).getBytes(), // block Size + new ByteSizeValue(2, ByteSizeUnit.KB).getBytes(), // repository chunk size + new ByteSizeValue(15, ByteSizeUnit.KB).getBytes() // file size + ); + } + + public void testChunkedRepositoryWithBlockSizeLessThanChunkSize() throws IOException { + verifyChunkedRepository( + new ByteSizeValue(1, ByteSizeUnit.KB).getBytes(), // block Size + new ByteSizeValue(2, ByteSizeUnit.KB).getBytes(), // repository chunk size + new ByteSizeValue(3, ByteSizeUnit.KB).getBytes() // file size + ); + } + + public void testChunkedRepositoryWithBlockSizeEqualToChunkSize() throws IOException { + verifyChunkedRepository( + new ByteSizeValue(2, ByteSizeUnit.KB).getBytes(), // block Size + new ByteSizeValue(2, ByteSizeUnit.KB).getBytes(), // repository chunk size + new ByteSizeValue(15, ByteSizeUnit.KB).getBytes() // file size + ); + } + private void verifyChunkedRepository(long blockSize, long repositoryChunkSize, long fileSize) throws IOException { when(transferManager.fetchBlob(any())).thenReturn(new ByteArrayIndexInput("test", new byte[(int) blockSize])); try ( FSDirectory directory = new MMapDirectory(path, lockFactory); @@ -105,8 +125,9 @@ public void testChunkedRepository() throws IOException { // Seek to the position past the first repository chunk indexInput.seek(repositoryChunkSize); } - // Verify the second chunk is requested (i.e. ".part1") - verify(transferManager).fetchBlob(argThat(request -> request.getBlobName().equals("File_Name.part1"))); + + // Verify all the chunks related to block are added to the fetchBlob request + verify(transferManager).fetchBlob(argThat(request -> request.getBlobLength() == blockSize)); } private void runAllTestsFor(int blockSizeShift) throws Exception { @@ -115,6 +136,7 @@ private void runAllTestsFor(int blockSizeShift) throws Exception { TestGroup.testGetBlock(blockedSnapshotFile, blockSize, FILE_SIZE); TestGroup.testGetBlockOffset(blockedSnapshotFile, blockSize, FILE_SIZE); TestGroup.testGetBlockStart(blockedSnapshotFile, blockSize); + TestGroup.testGetBlobParts(blockedSnapshotFile); TestGroup.testCurrentBlockStart(blockedSnapshotFile, blockSize); TestGroup.testCurrentBlockPosition(blockedSnapshotFile, blockSize); TestGroup.testClone(blockedSnapshotFile, blockSize); @@ -252,6 +274,35 @@ public static void testGetBlockStart(OnDemandBlockSnapshotIndexInput blockedSnap assertEquals(blockSize * 2, blockedSnapshotFile.getBlockStart(2)); } + public static void testGetBlobParts(OnDemandBlockSnapshotIndexInput blockedSnapshotFile) { + // block id 0 + int blockId = 0; + long blockStart = blockedSnapshotFile.getBlockStart(blockId); + long blockEnd = blockStart + blockedSnapshotFile.getActualBlockSize(blockId); + assertEquals( + (blockEnd - blockStart), + blockedSnapshotFile.getBlobParts(blockStart, blockEnd).stream().mapToLong(o -> o.getLength()).sum() + ); + + // block 1 + blockId = 1; + blockStart = blockedSnapshotFile.getBlockStart(blockId); + blockEnd = blockStart + blockedSnapshotFile.getActualBlockSize(blockId); + assertEquals( + (blockEnd - blockStart), + blockedSnapshotFile.getBlobParts(blockStart, blockEnd).stream().mapToLong(o -> o.getLength()).sum() + ); + + // block 2 + blockId = 2; + blockStart = blockedSnapshotFile.getBlockStart(blockId); + blockEnd = blockStart + blockedSnapshotFile.getActualBlockSize(blockId); + assertEquals( + (blockEnd - blockStart), + blockedSnapshotFile.getBlobParts(blockStart, blockEnd).stream().mapToLong(o -> o.getLength()).sum() + ); + } + public static void testCurrentBlockStart(OnDemandBlockSnapshotIndexInput blockedSnapshotFile, int blockSize) throws IOException { // block 0 blockedSnapshotFile.seek(blockSize - 1); diff --git a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java index d42e614302658..7ae3944eb6944 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java @@ -163,17 +163,11 @@ public void testUsageExceedsCapacity() throws Exception { public void testDownloadFails() throws Exception { doThrow(new IOException("Expected test exception")).when(blobContainer).readBlob(eq("failure-blob"), anyLong(), anyLong()); + List blobParts = new ArrayList<>(); + blobParts.add(new BlobFetchRequest.BlobPart("failure-blob", 0, EIGHT_MB)); expectThrows( IOException.class, - () -> transferManager.fetchBlob( - BlobFetchRequest.builder() - .blobName("failure-blob") - .position(0) - .fileName("file") - .directory(directory) - .length(EIGHT_MB) - .build() - ) + () -> transferManager.fetchBlob(BlobFetchRequest.builder().fileName("file").directory(directory).blobParts(blobParts).build()) ); MatcherAssert.assertThat(fileCache.usage().activeUsage(), equalTo(0L)); MatcherAssert.assertThat(fileCache.usage().usage(), equalTo(0L)); @@ -187,16 +181,13 @@ public void testFetchesToDifferentBlobsDoNotBlockOnEachOther() throws Exception latch.await(); return new ByteArrayInputStream(createData()); }).when(blobContainer).readBlob(eq("blocking-blob"), anyLong(), anyLong()); + List blobParts = new ArrayList<>(); + blobParts.add(new BlobFetchRequest.BlobPart("blocking-blob", 0, EIGHT_MB)); + final Thread blockingThread = new Thread(() -> { try { transferManager.fetchBlob( - BlobFetchRequest.builder() - .blobName("blocking-blob") - .position(0) - .fileName("blocking-file") - .directory(directory) - .length(EIGHT_MB) - .build() + BlobFetchRequest.builder().fileName("blocking-file").directory(directory).blobParts(blobParts).build() ); } catch (IOException e) { throw new RuntimeException(e); @@ -216,9 +207,9 @@ public void testFetchesToDifferentBlobsDoNotBlockOnEachOther() throws Exception } private IndexInput fetchBlobWithName(String blobname) throws IOException { - return transferManager.fetchBlob( - BlobFetchRequest.builder().blobName("blob").position(0).fileName(blobname).directory(directory).length(EIGHT_MB).build() - ); + List blobParts = new ArrayList<>(); + blobParts.add(new BlobFetchRequest.BlobPart("blob", 0, EIGHT_MB)); + return transferManager.fetchBlob(BlobFetchRequest.builder().fileName(blobname).directory(directory).blobParts(blobParts).build()); } private static void assertIndexInputIsFunctional(IndexInput indexInput) throws IOException {