upgrade otel version to 1.24 (#378)

* upgrade otel version to 1.24

* comment some failing tests, and try to build the agent jar

* fix classloading issues

* remove jaxrs instrumentation

* nit

* try to fix muzzle and smoke tests

* fix/update smoke tests

* temp: Upgrade otel smoketest (#381)

* fix tomee test

* fix Play test

* fix liberty test

* add sock host port as well

* fix grpc test

* fix spring boot tests

* format

* fix wildify test

Author: shashank11p

build.gradle.kts

@@ -40,20 +40,16 @@ subprojects {
     extra.set("versions", mapOf(
             // when updating these values, some values must also be updated in buildSrc as this map
             // cannot be accessed there
-            // 7/23/22  version 1.11.1-alpha is specified for opentelemetry_java_agent_jaxrs
-            // gradle can't seem to load a more current version than that
-            "opentelemetry" to "1.13.0",
-            "opentelemetry_semconv" to "1.13.0-alpha",
+            "opentelemetry" to "1.24.0",
+            "opentelemetry_semconv" to "1.24.0-alpha",
             "opentelemetry_proto" to "0.11.0-alpha",
-            "opentelemetry_java_agent" to "1.13.1-alpha",
-            "opentelemetry_java_agent_all" to "1.13.1",
-            "opentelemetry_java_agent_jaxrs" to "1.11.1-alpha",
-            "opentelemetry_java_agent_netty" to "1.13.1-alpha",
-            "opentelemetry_java_agent-tooling" to "1.13.1-alpha",
+            "opentelemetry_java_agent" to "1.24.0-alpha",
+            "opentelemetry_java_agent_all" to "1.24.0",
+            "opentelemetry_java_agent-tooling" to "1.24.0-alpha",
 
-            "opentelemetry_gradle_plugin" to "1.16.0-alpha",
+            "opentelemetry_gradle_plugin" to "1.24.0-alpha",
             "byte_buddy" to "1.12.10",
-                "slf4j" to "1.7.32"
+                "slf4j" to "2.0.7"
     ))
 
     apply<JavaPlugin>()

buildSrc/build.gradle.kts

@@ -27,7 +27,7 @@ repositories {
 dependencies {
   implementation(gradleApi())
   implementation(localGroovy())
-  val otelInstrumentationVersion = "1.13.0-alpha"
+  val otelInstrumentationVersion = "1.24.0-alpha"
   implementation("io.opentelemetry.javaagent:opentelemetry-muzzle:$otelInstrumentationVersion")
   implementation("io.opentelemetry.instrumentation.muzzle-generation:io.opentelemetry.instrumentation.muzzle-generation.gradle.plugin:$otelInstrumentationVersion")
   implementation("io.opentelemetry.instrumentation.muzzle-check:io.opentelemetry.instrumentation.muzzle-check.gradle.plugin:$otelInstrumentationVersion")

buildSrc/src/main/groovy/MuzzlePlugin.groovy

@@ -89,7 +89,7 @@ class MuzzlePlugin implements Plugin<Project> {
           project.getLogger().info('No muzzle pass directives configured. Asserting pass against instrumentation compile-time dependencies')
           ClassLoader userCL = createCompileDepsClassLoader(project, bootstrapProject)
           ClassLoader instrumentationCL = createInstrumentationClassloader(project, toolingProject)
-          MuzzleGradlePluginUtil.@Companion.assertInstrumentationMuzzled(instrumentationCL, userCL, true)
+          MuzzleGradlePluginUtil.@Companion.assertInstrumentationMuzzled(instrumentationCL, userCL, muzzleDirective.excludedInstrumentationNames, true)
         }
         println "Muzzle executing for $project"
       }
@@ -276,6 +276,7 @@ class MuzzlePlugin implements Plugin<Project> {
       inverseDirective.versions = version
       inverseDirective.assertPass = !muzzleDirective.assertPass
       inverseDirectives.add(inverseDirective)
+      inverseDirective.excludedInstrumentationNames.addAll(muzzleDirective.excludedInstrumentationNames)
     }
 
     return inverseDirectives
@@ -393,7 +394,7 @@ class MuzzlePlugin implements Plugin<Project> {
         }
         ClassLoader userCL = createClassLoaderForTask(instrumentationProject, bootstrapProject, taskName, shadowMuzzleArchiveFile)
         try {
-          MuzzleGradlePluginUtil.@Companion.assertInstrumentationMuzzled(instrumentationCL, userCL, muzzleDirective.assertPass)
+          MuzzleGradlePluginUtil.@Companion.assertInstrumentationMuzzled(instrumentationCL, userCL, muzzleDirective.excludedInstrumentationNames, muzzleDirective.assertPass)
         } finally {
           Thread.currentThread().contextClassLoader = ccl
         }
@@ -457,6 +458,7 @@ class MuzzleDirective {
   String versions
   Set<String> skipVersions = new HashSet<>()
   List<String> additionalDependencies = new ArrayList<>()
+  Set<String> excludedInstrumentationNames = new HashSet<>()
   boolean assertPass
   boolean assertInverse = false
   boolean coreJdk = false
@@ -474,6 +476,10 @@ class MuzzleDirective {
     additionalDependencies.add(compileString)
   }
 
+  void excludeInstrumentationName(String excludedInstName) {
+    excludedInstrumentationNames.add(excludedInstName)
+  }
+
   /**
    * Slug of directive name.
    *

buildSrc/src/main/java/io/opentelemetry/instrumentation/gradle/AutoInstrumentationPlugin.java

@@ -64,11 +64,11 @@ private void addDependencies(Project project) {
         "io.opentelemetry.javaagent:opentelemetry-javaagent-extension-api:" + versions
             .get("opentelemetry_java_agent"));
     dependencies.add("implementation",
-        "io.opentelemetry.javaagent:opentelemetry-javaagent-instrumentation-api:" + versions
-            .get("opentelemetry_java_agent"));
+        "io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:" + versions
+            .get("opentelemetry"));
     dependencies.add("implementation",
         "io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:" + versions
-            .get("opentelemetry_java_agent"));
+            .get("opentelemetry"));
 
     dependencies.add("implementation", dependencies.project(Map.of("path", ":javaagent-core")));
     dependencies.add("implementation", dependencies.project(Map.of("path", ":filter-api")));

filter-api/src/main/java/org/hypertrace/agent/filter/FilterRegistry.java

@@ -55,7 +55,11 @@ public static Filter getFilter() {
         if (filter == null) {
           try {
             FilterProviderConfig providerConfig = new FilterProviderConfig();
-            filter = load(providerConfig, Collections.emptyList());
+            filter =
+                load(
+                    providerConfig,
+                    Collections.emptyList(),
+                    Thread.currentThread().getContextClassLoader());
           } catch (Throwable t) {
             logger.error("Throwable thrown while loading filter jars", t);
           }
@@ -72,17 +76,19 @@ public static Filter getFilter() {
    * @param providerConfig config needed by the filter (Eg. service name)
    * @param jarPaths paths to filter jar files.
    */
-  public static void initialize(FilterProviderConfig providerConfig, List<String> jarPaths) {
+  public static void initialize(
+      FilterProviderConfig providerConfig, List<String> jarPaths, ClassLoader cl) {
     try {
-      filter = load(providerConfig, jarPaths);
+      filter = load(providerConfig, jarPaths, cl);
     } catch (Throwable t) {
       logger.error("Throwable thrown while loading filter jars", t);
     }
   }
 
-  private static Filter load(FilterProviderConfig providerConfig, List<String> jarPaths) {
-    ClassLoader cl = loadJars(jarPaths);
-    ServiceLoader<FilterProvider> providers = ServiceLoader.load(FilterProvider.class, cl);
+  private static Filter load(
+      FilterProviderConfig providerConfig, List<String> jarPaths, ClassLoader cl) {
+    ClassLoader newCl = loadJars(jarPaths, cl);
+    ServiceLoader<FilterProvider> providers = ServiceLoader.load(FilterProvider.class, newCl);
     List<Filter> filters = new ArrayList<>();
 
     for (FilterProvider provider : providers) {
@@ -100,7 +106,7 @@ private static Filter load(FilterProviderConfig providerConfig, List<String> jar
     return new MultiFilter(filters);
   }
 
-  private static ClassLoader loadJars(List<String> jarPaths) {
+  private static ClassLoader loadJars(List<String> jarPaths, ClassLoader cl) {
     URL[] urls = new URL[jarPaths.size()];
     int i = 0;
     for (String jarPath : jarPaths) {
@@ -112,7 +118,7 @@ private static ClassLoader loadJars(List<String> jarPaths) {
         logger.warn(String.format("Malformed URL exception for jar on path: %s", jarPath), e);
       }
     }
-    return new URLClassLoader(urls, Thread.currentThread().getContextClassLoader());
+    return new URLClassLoader(urls, cl);
   }
 
   public static String getProviderDisabledPropertyName(Class<?> clazz) {

gradle/java.gradle

@@ -51,10 +51,10 @@ ext {
             ],
             groovy     : "org.codehaus.groovy:groovy-all:2.5.11",
             testLogging: [
-                    dependencies.create(group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3'),
-                    dependencies.create(group: 'org.slf4j', name: 'log4j-over-slf4j', version: '1.7.30'),
-                    dependencies.create(group: 'org.slf4j', name: 'jcl-over-slf4j', version: '1.7.30'),
-                    dependencies.create(group: 'org.slf4j', name: 'jul-to-slf4j', version: '1.7.30'),
+                    dependencies.create(group: 'ch.qos.logback', name: 'logback-classic', version: '1.4.6'),
+                    dependencies.create(group: 'org.slf4j', name: 'log4j-over-slf4j', version: '2.0.7'),
+                    dependencies.create(group: 'org.slf4j', name: 'jcl-over-slf4j', version: '2.0.7'),
+                    dependencies.create(group: 'org.slf4j', name: 'jul-to-slf4j', version: '2.0.7'),
             ]
     ]
 }

instrumentation/apache-httpasyncclient-4.1/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/apachehttpasyncclient/ApacheAsyncClientInstrumentationModule.java

@@ -16,9 +16,9 @@
 
 package io.opentelemetry.javaagent.instrumentation.hypertrace.apachehttpasyncclient;
 
+import static io.opentelemetry.javaagent.bootstrap.Java8BytecodeBridge.currentContext;
 import static io.opentelemetry.javaagent.extension.matcher.AgentElementMatchers.hasClassesNamed;
 import static io.opentelemetry.javaagent.extension.matcher.AgentElementMatchers.implementsInterface;
-import static io.opentelemetry.javaagent.instrumentation.api.Java8BytecodeBridge.currentContext;
 import static net.bytebuddy.matcher.ElementMatchers.isMethod;
 import static net.bytebuddy.matcher.ElementMatchers.named;
 import static net.bytebuddy.matcher.ElementMatchers.takesArgument;

instrumentation/apache-httpclient-4.0/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/apachehttpclient/v4_0/ApacheClientInstrumentationModule.java

@@ -25,10 +25,10 @@
 import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
 
 import com.google.auto.service.AutoService;
+import io.opentelemetry.javaagent.bootstrap.Java8BytecodeBridge;
 import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;
-import io.opentelemetry.javaagent.instrumentation.api.Java8BytecodeBridge;
 import java.util.Arrays;
 import java.util.List;
 import net.bytebuddy.asm.Advice;

instrumentation/apache-httpclient-4.0/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/apachehttpclient/v4_0/HttpEntityInstrumentation.java

@@ -23,7 +23,7 @@
 import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
 import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
 
-import io.opentelemetry.instrumentation.api.field.VirtualField;
+import io.opentelemetry.instrumentation.api.util.VirtualField;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;
 import io.opentelemetry.javaagent.instrumentation.hypertrace.apachehttpclient.v4_0.ApacheHttpClientObjectRegistry.SpanAndAttributeKey;

instrumentation/build.gradle.kts

@@ -26,8 +26,15 @@ tasks {
     // Keep in sync with https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/f893ca540b72a895fbf18c14d2df8d1cabaf2c7f/instrumentation/instrumentation.gradle#L51
     shadowJar {
         dependencies{
-            // exclude core, it lives in the bootstrap classloader
+            // exclude packages that live in the bootstrap classloader
             exclude(project(":javaagent-core"))
+            exclude(project(":filter-api"))
+            exclude("io/opentelemetry/semconv/**")
+            exclude("io/opentelemetry/context/**")
+            exclude(dependency("io.opentelemetry:opentelemetry-api"))
+            exclude("io/opentelemetry/instrumentation/api/**")
+            // exclude bootstrap part of javaagent-extension-api
+            exclude("io/opentelemetry/javaagent/bootstrap/**")
         }
 
         mergeServiceFiles()

instrumentation/grpc-1.6/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/grpc/v1_6/NettyHttp2HeadersInstrumentationModule.java

@@ -25,10 +25,10 @@
 import io.grpc.Metadata;
 import io.netty.handler.codec.http2.Http2Headers;
 import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.javaagent.bootstrap.Java8BytecodeBridge;
 import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;
-import io.opentelemetry.javaagent.instrumentation.api.Java8BytecodeBridge;
 import io.opentelemetry.javaagent.instrumentation.hypertrace.grpc.GrpcSemanticAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;

instrumentation/grpc-shaded-netty-1.9/src/main/java/io/opentelemetry/javaagent/instrumentation/shaded/netty/ShadedNettyUtilsInstrumentation.java

@@ -24,9 +24,9 @@
 import io.grpc.Metadata;
 import io.grpc.netty.shaded.io.netty.handler.codec.http2.Http2Headers;
 import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.javaagent.bootstrap.Java8BytecodeBridge;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;
-import io.opentelemetry.javaagent.instrumentation.api.Java8BytecodeBridge;
 import io.opentelemetry.javaagent.instrumentation.shaded.netty.utils.NettyUtils;
 import net.bytebuddy.asm.Advice;
 import net.bytebuddy.description.type.TypeDescription;

instrumentation/java-streams/build.gradle.kts

@@ -22,6 +22,7 @@ afterEvaluate{
 val versions: Map<String, String> by extra
 
 dependencies {
+    implementation("io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:${versions["opentelemetry"]}")
     testImplementation(testFixtures(project(":testing-common")))
     testImplementation("io.opentelemetry.javaagent:opentelemetry-muzzle:${versions["opentelemetry_java_agent"]}")
 }

instrumentation/java-streams/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/java/inputstream/InputStreamInstrumentationModule.java

@@ -26,7 +26,7 @@
 import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
 
 import com.google.auto.service.AutoService;
-import io.opentelemetry.instrumentation.api.field.VirtualField;
+import io.opentelemetry.instrumentation.api.util.VirtualField;
 import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;

instrumentation/java-streams/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/java/inputstream/InputStreamUtils.java

@@ -21,7 +21,7 @@
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.api.trace.Tracer;
 import io.opentelemetry.context.Context;
-import io.opentelemetry.instrumentation.api.field.VirtualField;
+import io.opentelemetry.instrumentation.api.util.VirtualField;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;

instrumentation/java-streams/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/java/outputstream/OutputStreamInstrumentationModule.java

@@ -24,7 +24,7 @@
 import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
 
 import com.google.auto.service.AutoService;
-import io.opentelemetry.instrumentation.api.field.VirtualField;
+import io.opentelemetry.instrumentation.api.util.VirtualField;
 import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;
@@ -40,10 +40,10 @@
 
 /**
  * {@link OutputStream} instrumentation. The type matcher applies to all implementations. However
- * only streams that are in the {@link io.opentelemetry.instrumentation.api.field.VirtualField} are
+ * only streams that are in the {@link io.opentelemetry.instrumentation.api.util.VirtualField} are
  * instrumented, otherwise the instrumentation is noop.
  *
- * <p>If the stream is in the {@link io.opentelemetry.instrumentation.api.field.VirtualField} then
+ * <p>If the stream is in the {@link io.opentelemetry.instrumentation.api.util.VirtualField} then
  * arguments to write methods are also passed to the buffered stream (value) from the map. The
  * buffered stream is then used by other instrumentations to capture body.
  */

instrumentation/java-streams/src/test/java/io/opentelemetry/javaagent/instrumentation/hypertrace/java/ContextAccessorInstrumentationModule.java

@@ -20,7 +20,7 @@
 import static net.bytebuddy.matcher.ElementMatchers.named;
 import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
 
-import io.opentelemetry.instrumentation.api.field.VirtualField;
+import io.opentelemetry.instrumentation.api.util.VirtualField;
 import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;

instrumentation/jaxrs-client-2.0/build.gradle.kts

@@ -32,12 +32,27 @@ val versions: Map<String, String> by extra
 
 dependencies {
     api(project(":instrumentation:java-streams"))
-    testImplementation("io.opentelemetry.javaagent.instrumentation:opentelemetry-javaagent-jaxrs-client-2.0-common:${versions["opentelemetry_java_agent_jaxrs"]}")
+    api("io.opentelemetry.javaagent.instrumentation:opentelemetry-javaagent-http-url-connection:1.24.0-alpha")
 
     compileOnly("javax.ws.rs:javax.ws.rs-api:2.0.1")
 
-     testImplementation(testFixtures(project(":testing-common")))
+    testImplementation(testFixtures(project(":testing-common")))
     testImplementation("org.glassfish.jersey.core:jersey-client:2.27")
     testImplementation("org.glassfish.jersey.inject:jersey-hk2:2.27")
     testImplementation("io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-semconv:${versions["opentelemetry_semconv"]}")
+
+    testImplementation(project(":instrumentation:apache-httpclient-4.0"))
+    testImplementation("io.opentelemetry.javaagent.instrumentation:opentelemetry-javaagent-apache-httpclient-4.0:${versions["opentelemetry_java_agent"]}")
+    testImplementation("org.jboss.resteasy:resteasy-client:3.0.5.Final")
+    // ^ This version has timeouts https://issues.redhat.com/browse/RESTEASY-975
+    testImplementation("org.apache.cxf:cxf-rt-rs-client:3.1.0")
+    // Doesn't work with CXF 3.0.x because their context is wrong:
+    // https://github.com/apache/cxf/commit/335c7bad2436f08d6d54180212df5a52157c9f21
+
+    testImplementation("javax.xml.bind:jaxb-api:2.2.3")
+
+    testImplementation("org.glassfish.jersey.inject:jersey-hk2:2.+")
+    testImplementation("org.glassfish.jersey.core:jersey-client:2.+")
+    testImplementation("org.jboss.resteasy:resteasy-client:3.0.26.Final")
+    testImplementation("org.apache.cxf:cxf-rt-rs-client:3.+")
 }