Update Hdfs3 fixture

- fix startup
- stop failing silently
- handle jarhell in shadowed commons

Author: breskeby

gradle/verification-metadata.xml

@@ -939,6 +939,11 @@
             <sha256 value="a1e25d1b71e805233daefabba72a21566580354a192539cdab43bbf191dbf951" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.re2j" name="re2j" version="1.0">
+         <artifact name="re2j-1.0.jar">
+            <sha256 value="5406ddfec247c0db50a11b2fe6f4d881980fc3b2c3d03fce7b258c4b014be3e0" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.re2j" name="re2j" version="1.1">
          <artifact name="re2j-1.1.jar">
             <sha256 value="24ada84d1b5de584e3e84b06f0c7dd562cee6eafe8dea8083bd8eb123823bbe7" origin="Generated by Gradle"/>
@@ -1474,6 +1479,11 @@
             <sha256 value="fd63b583fd3e8baeae22efacbd5a4f91c1fd97f56248e62e2615efa7b81daeaa" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="commons-daemon" name="commons-daemon" version="1.4.0">
+         <artifact name="commons-daemon-1.4.0.jar">
+            <sha256 value="6b8e35b13f27f265fed0caa7c4ce9d13b790662af5f48cb7d41df394de759dbd" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="commons-digester" name="commons-digester" version="1.8">
          <artifact name="commons-digester-1.8.jar">
             <sha256 value="05662373044f3dff112567b7bb5dfa1174e91e074c0c727b4412788013f49d56" origin="Generated by Gradle"/>

test/fixtures/hdfs-fixture/build.gradle

@@ -172,6 +172,11 @@ dependencies {
     hdfs3("commons-io:commons-io:2.8.0")
     hdfs3("org.apache.commons:commons-configuration2:2.1.1")
     hdfs3("org.apache.commons:commons-lang3:3.7")
+    hdfs3("org.eclipse.jetty:jetty-xml:9.4.40.v20210413")
+    hdfs3("commons-daemon:commons-daemon:1.4.0")
+    hdfs3("com.google.protobuf:protobuf-java:4.32.0")
+    hdfs3("com.google.re2j:re2j:1.0")
+    hdfs3("org.apache.commons:commons-math3:3.6.1")
     hdfs3("org.apache.commons:commons-text:1.4")
     hdfs3("com.fasterxml.woodstox:woodstox-core:6.7.0")
     hdfs3("org.codehaus.woodstox:stax2-api:4.2.2")
@@ -194,6 +199,8 @@ tasks.named("shadowJar").configure {
     exclude "org.apache.hadoop.ipc.StandbyException"
     exclude "org.apache.hadoop.application-classloader.properties"
   }
+  relocate("org.apache.commons.lang", "fixture.hdfs2.org.apache.commons.lang")
+  relocate("org.apache.commons.io", "fixture.hdfs2.org.apache.commons.io")
   configurations.add(project.configurations.hdfs3)
 }
 
@@ -203,6 +210,8 @@ def hdfs2Jar = tasks.register("hdfs2jar", ShadowJar) {
     exclude "org.apache.hadoop.ipc.StandbyException"
     exclude "org.apache.hadoop.application-classloader.properties"
   }
+  relocate("org.apache.commons.lang", "fixture.hdfs2.org.apache.commons.lang")
+  relocate("org.apache.commons.io", "fixture.hdfs2.org.apache.commons.io")
   archiveClassifier.set("hdfs2")
   from sourceSets.main.output
   configurations.add(project.configurations.hdfs2)
@@ -212,6 +221,7 @@ tasks.withType(ShadowJar).configureEach {
   dependencies {
     exclude(dependency("com.carrotsearch.randomizedtesting:randomizedtesting-runner:.*"))
     exclude(dependency("junit:junit:.*"))
+    exclude(dependency("org.slf4j:slf4j-api:.*"))
     exclude(dependency("org.apache.httpcomponents:httpcore:.*"))
     exclude(dependency("org.apache.logging.log4j:log4j-1.2-api:.*"))
     exclude(dependency("net.java.dev.jna:jna:.*"))

test/fixtures/hdfs-fixture/src/main/java/org/elasticsearch/test/fixtures/hdfs/HdfsFixture.java

@@ -199,7 +199,7 @@ private void startMinHdfs() throws Exception {
                 // If the maximum number of attempts is reached, rethrow the exception
                 FileUtils.deleteDirectory(baseDir.toFile());
                 if (attempt == maxAttempts) {
-                    Assume.assumeTrue("Unable to start HDFS cluster", false);
+                    throw e;
                 }
             }
         }

x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle

@@ -12,16 +12,15 @@ apply plugin: 'elasticsearch.internal-available-ports'
 configurations {
   javaRestTestRuntimeClasspath {
     attributes {
-      attribute(Attribute.of('hdfs.major.version', Integer), 2)
+      attribute(Attribute.of('hdfs.major.version', Integer), 3)
     }
   }
 }
 
 dependencies {
   clusterPlugins project(':plugins:repository-hdfs')
   javaRestTestImplementation(testArtifact(project(xpackModule('searchable-snapshots'))))
-  javaRestTestCompileOnly project(path: ':test:fixtures:hdfs-fixture')
-  javaRestTestRuntimeOnly project(path: ':test:fixtures:hdfs-fixture')
+  javaRestTestImplementation project(path: ':test:fixtures:hdfs-fixture')
   javaRestTestImplementation project(':test:fixtures:krb5kdc-fixture')
   javaRestTestRuntimeOnly "com.google.guava:guava:16.0.1"
   javaRestTestRuntimeOnly "commons-cli:commons-cli:1.2"