Merge branch 'master' into feature/client-gen

Miles-Garnsey · web-flow · commit 3df9295e9b8d · 2023-10-12T18:01:17.000+09:00
diff --git a/.github/workflows/docker-release.yaml b/.github/workflows/docker-release.yaml
@@ -10,10 +10,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dse-version: [6.8.25, 6.8.26, 6.8.28, 6.8.29, 6.8.30, 6.8.31, 6.8.32, 6.8.33, 6.8.34, 6.8.35, 6.8.36, 6.8.37]
+        dse-version: [6.8.25, 6.8.26, 6.8.28, 6.8.29, 6.8.30, 6.8.31, 6.8.32, 6.8.33, 6.8.34, 6.8.35, 6.8.36, 6.8.37, 6.8.38]
         image-base: [jdk8, jdk11]
         include:
-          - dse-version: 6.8.37
+          - dse-version: 6.8.38
             latest: true
     runs-on: ubuntu-latest
     steps:
@@ -106,10 +106,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dse-version: [6.8.25, 6.8.26, 6.8.28, 6.8.29, 6.8.30, 6.8.31, 6.8.32, 6.8.33, 6.8.34, 6.8.35, 6.8.36, 6.8.37]
+        dse-version: [6.8.25, 6.8.26, 6.8.28, 6.8.29, 6.8.30, 6.8.31, 6.8.32, 6.8.33, 6.8.34, 6.8.35, 6.8.36, 6.8.37, 6.8.38]
         image-base: [ubi8]
         include:
-          - dse-version: 6.8.37
+          - dse-version: 6.8.38
             latest: true
     runs-on: ubuntu-latest
     steps:
@@ -174,9 +174,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        cassandra-version: [3.11.7, 3.11.8, 3.11.11, 3.11.12, 3.11.13, 3.11.14, 3.11.15]
+        cassandra-version: [3.11.7, 3.11.8, 3.11.11, 3.11.12, 3.11.13, 3.11.14, 3.11.15, 3.11.16]
         include:
-          - cassandra-version: 3.11.15
+          - cassandra-version: 3.11.16
             latest: true
     runs-on: ubuntu-latest
     steps:
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,12 +10,18 @@ Changelog for Management API, new PRs should update the `main / unreleased` sect
 ```
 
 ## unreleased
-[CHANGE]  [#323](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/323) Rename Maven artifact groupId from com.datastax to io.k8ssandra
-[FEATURE] [#323](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/323) Add OpenAPI Java client generation
-[FEATURE] [#337](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/337) Publish Maven artifacts to Cloudsmith.io
-[FEATURE] [#326](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/326) Provide topology endpoints in OpenAPI client
+
+## v0.1.67 (2023-09-29)
+* [CHANGE]  [#323](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/323) Rename Maven artifact groupId from com.datastax to io.k8ssandra
+* [FEATURE] [#323](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/323) Add OpenAPI Java client generation
+* [FEATURE] [#337](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/337) Publish Maven artifacts to Cloudsmith.io
+* [FEATURE] [#375](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/375) Add SSL/TLS hot reloading and allow TLS v1.3
+* [FEATURE] [#349](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/349) Metrics endpoint should allow querying a subset
+* [FEATURE] [#395](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/395) Add Cassandra version 3.11.16
+* [FEATURE] [#380](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/380) Add DSE 6.8.38
+* [FEATURE] [#326](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/326) Provide topology endpoints in OpenAPI client
 [FEATURE] [#345](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/345) Implement host related methods
-[BUGFIX]  [#339](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/339) OpenAPI client publish does not also publish other artifacts
+* [BUGFIX]  [#339](https://github.com/k8ssandra/management-api-for-apache-cassandra/issues/339) OpenAPI client publish does not also publish other artifacts
 
 
 ## Releases older than this point did not maintain a Changelog in the format above. The changes below were generated by the gen_changelog.sh script.
diff --git a/Dockerfile-oss b/Dockerfile-oss
@@ -1,4 +1,4 @@
-ARG CASSANDRA_VERSION=3.11.15
+ARG CASSANDRA_VERSION=3.11.16
 
 FROM --platform=$BUILDPLATFORM maven:3.8.7-eclipse-temurin-11 as builder
 
diff --git a/Dockerfile-oss.ubi8 b/Dockerfile-oss.ubi8
@@ -1,6 +1,6 @@
 ARG UBI_MAJOR=8
 ARG UBI_BASETAG=latest
-ARG CASSANDRA_VERSION=3.11.15
+ARG CASSANDRA_VERSION=3.11.16
 FROM registry.access.redhat.com/ubi${UBI_MAJOR}/ubi-minimal:${UBI_BASETAG} AS builder
 
 ARG URL_PREFIX=https://dlcdn.apache.org/cassandra
diff --git a/README.md b/README.md
@@ -63,11 +63,12 @@ The following versions of Cassandra and DSE are published to Docker and supporte
 | 3.11.13          | 4.0.5           |                 | 6.8.30    |
 | 3.11.14          | 4.0.6           |                 | 6.8.31    |
 | 3.11.15          | 4.0.7           |                 | 6.8.32    |
-|                  | 4.0.8           |                 | 6.8.33    |
+| 3.11.16          | 4.0.8           |                 | 6.8.33    |
 |                  | 4.0.9           |                 | 6.8.34    |
 |                  | 4.0.10          |                 | 6.8.35    |
 |                  |                 |                 | 6.8.36    |
 |                  |                 |                 | 6.8.37    |
+|                  |                 |                 | 6.8.38    |
 
 - All supported images are available in `linux/amd64` or `linux/arm64` formats.
 - All images (with the exception of Cassandra trunk) are available as an Ubuntu based image or a RedHat UBI 8 based image.
diff --git a/dse-68/Dockerfile.jdk11 b/dse-68/Dockerfile.jdk11
@@ -10,7 +10,7 @@ ENV DSE_HOME /opt/dse
 ENV DSE_AGENT_HOME /opt/agent
 
 # Get commandline parameters
-ARG DSE_VERSION=6.8.37
+ARG DSE_VERSION=6.8.38
 ARG URL_PREFIX=https://downloads.datastax.com/enterprise
 ARG TARBALL=dse-${DSE_VERSION}-bin.tar.gz
 ARG DOWNLOAD_URL=${URL_PREFIX}/${TARBALL}
diff --git a/dse-68/Dockerfile.jdk8 b/dse-68/Dockerfile.jdk8
@@ -10,7 +10,7 @@ ENV DSE_HOME /opt/dse
 ENV DSE_AGENT_HOME /opt/agent
 
 # Get commandline parameters
-ARG DSE_VERSION=6.8.37
+ARG DSE_VERSION=6.8.38
 ARG URL_PREFIX=https://downloads.datastax.com/enterprise
 ARG TARBALL=dse-${DSE_VERSION}-bin.tar.gz
 ARG DOWNLOAD_URL=${URL_PREFIX}/${TARBALL}
diff --git a/dse-68/Dockerfile.ubi8 b/dse-68/Dockerfile.ubi8
@@ -1,4 +1,4 @@
-ARG DSE_VERSION=6.8.37
+ARG DSE_VERSION=6.8.38
 ARG UBI_MAJOR=8
 ARG UBI_BASETAG=latest
 
diff --git a/management-api-agent-common/pom.xml b/management-api-agent-common/pom.xml
@@ -64,6 +64,9 @@
           <groupId>org.assertj</groupId>
           <artifactId>assertj-core</artifactId>
           <version>${assertj.version}</version>
+          <groupId>org.mockito</groupId>
+          <artifactId>mockito-core</artifactId>
+          <version>5.5.0</version>
           <scope>test</scope>
         </dependency>
         <dependency>
diff --git a/management-api-agent-common/src/main/java/com/datastax/mgmtapi/util/JobExecutor.java b/management-api-agent-common/src/main/java/com/datastax/mgmtapi/util/JobExecutor.java
@@ -11,11 +11,12 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadPoolExecutor;
 import org.apache.cassandra.utils.Pair;
 
 public class JobExecutor {
   ExecutorService executorService = Executors.newFixedThreadPool(1);
-  Cache<String, Job> jobCache = CacheBuilder.newBuilder().maximumSize(1000).build();
+  Cache<String, Job> jobCache = CacheBuilder.newBuilder().recordStats().maximumSize(1000).build();
 
   public Pair<String, CompletableFuture<Void>> submit(String jobType, Runnable runnable) {
     // Where do I create the job details? Here? Add it to the Cache first?
@@ -57,4 +58,12 @@ public void updateJob(Job job) {
   public Job getJobWithId(String jobId) {
     return jobCache.getIfPresent(jobId);
   }
+
+  public int runningTasks() {
+    return ((ThreadPoolExecutor) executorService).getActiveCount();
+  }
+
+  public int queuedTasks() {
+    return ((ThreadPoolExecutor) executorService).getQueue().size();
+  }
 }
diff --git a/management-api-agent-common/src/main/java/io/k8ssandra/metrics/builder/CassandraMetricNameParser.java b/management-api-agent-common/src/main/java/io/k8ssandra/metrics/builder/CassandraMetricNameParser.java
@@ -42,6 +42,13 @@ public CassandraMetricNameParser(
     }
   }
 
+  public static CassandraMetricNameParser getDefaultParser(Configuration config) {
+    return new CassandraMetricNameParser(
+        CassandraMetricsTools.DEFAULT_LABEL_NAMES,
+        CassandraMetricsTools.DEFAULT_LABEL_VALUES,
+        config);
+  }
+
   private void parseEnvVariablesAsLabels(Map<String, String> envSettings) {
     for (Map.Entry<String, String> entry : envSettings.entrySet()) {
       String envValue = System.getenv(entry.getValue());
diff --git a/management-api-agent-common/src/main/java/io/k8ssandra/metrics/builder/CassandraMetricRegistryListener.java b/management-api-agent-common/src/main/java/io/k8ssandra/metrics/builder/CassandraMetricRegistryListener.java
@@ -77,11 +77,7 @@ public class CassandraMetricRegistryListener implements MetricRegistryListener {
   public CassandraMetricRegistryListener(
       ConcurrentHashMap<String, RefreshableMetricFamilySamples> familyCache, Configuration config)
       throws NoSuchMethodException {
-    parser =
-        new CassandraMetricNameParser(
-            CassandraMetricsTools.DEFAULT_LABEL_NAMES,
-            CassandraMetricsTools.DEFAULT_LABEL_VALUES,
-            config);
+    parser = CassandraMetricNameParser.getDefaultParser(config);
     cache = new ConcurrentHashMap<>();
 
     this.familyCache = familyCache;
diff --git a/management-api-agent-common/src/main/java/io/k8ssandra/metrics/config/Configuration.java b/management-api-agent-common/src/main/java/io/k8ssandra/metrics/config/Configuration.java
@@ -21,6 +21,9 @@ public class Configuration {
   @JsonProperty("labels")
   private LabelConfiguration labels;
 
+  @JsonProperty("extended_metrics_disabled")
+  private boolean extendedDisabled;
+
   public Configuration() {
     relabels = new ArrayList<>();
   }
@@ -44,4 +47,12 @@ public void setRelabels(List<RelabelSpec> relabels) {
   public void setLabels(LabelConfiguration labels) {
     this.labels = labels;
   }
+
+  public boolean isExtendedDisabled() {
+    return extendedDisabled;
+  }
+
+  public void setExtendedDisabled(boolean extendedDisabled) {
+    this.extendedDisabled = extendedDisabled;
+  }
 }
diff --git a/management-api-agent-common/src/main/java/io/k8ssandra/metrics/interceptors/MetricsInterceptor.java b/management-api-agent-common/src/main/java/io/k8ssandra/metrics/interceptors/MetricsInterceptor.java
@@ -9,6 +9,7 @@
 import io.k8ssandra.metrics.config.Configuration;
 import io.k8ssandra.metrics.http.NettyMetricsHttpServer;
 import io.k8ssandra.metrics.prometheus.CassandraDropwizardExports;
+import io.k8ssandra.metrics.prometheus.CassandraTasksExports;
 import io.netty.channel.EventLoopGroup;
 import io.netty.channel.epoll.EpollEventLoopGroup;
 import io.prometheus.client.hotspot.DefaultExports;
@@ -72,6 +73,11 @@ public static void intercept(@SuperCall Callable<Void> zuper) throws Exception {
       // Add JVM metrics
       DefaultExports.initialize();
 
+      // Add task metrics
+      if (!config.isExtendedDisabled()) {
+        new CassandraTasksExports(CassandraMetricsRegistry.Metrics, config).register();
+      }
+
       // Create /metrics handler. Note, this doesn't support larger than nThreads=1
       final EventLoopGroup httpGroup = new EpollEventLoopGroup(1);
 
@@ -81,12 +87,7 @@ public static void intercept(@SuperCall Callable<Void> zuper) throws Exception {
 
       logger.info("Metrics collector started");
 
-      Runtime.getRuntime()
-          .addShutdownHook(
-              new Thread(
-                  () -> {
-                    httpGroup.shutdownGracefully();
-                  }));
+      Runtime.getRuntime().addShutdownHook(new Thread(httpGroup::shutdownGracefully));
     } catch (Throwable t) {
       logger.error("Unable to start metrics endpoint", t);
     }
diff --git a/management-api-agent-common/src/main/java/io/k8ssandra/metrics/prometheus/CassandraTasksExports.java b/management-api-agent-common/src/main/java/io/k8ssandra/metrics/prometheus/CassandraTasksExports.java
diff --git a/management-api-agent-common/src/main/resources/default-metric-settings.yaml b/management-api-agent-common/src/main/resources/default-metric-settings.yaml
diff --git a/management-api-agent-common/src/test/java/io/k8ssandra/metrics/config/ConfigReaderTest.java b/management-api-agent-common/src/test/java/io/k8ssandra/metrics/config/ConfigReaderTest.java
diff --git a/management-api-agent-common/src/test/java/io/k8ssandra/metrics/prometheus/TaskExportsTests.java b/management-api-agent-common/src/test/java/io/k8ssandra/metrics/prometheus/TaskExportsTests.java
diff --git a/management-api-agent-common/src/test/resources/collector_tls.yaml b/management-api-agent-common/src/test/resources/collector_tls.yaml
diff --git a/management-api-agent-dse-6.8/pom.xml b/management-api-agent-dse-6.8/pom.xml
diff --git a/pom.xml b/pom.xml

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-ARG CASSANDRA_VERSION=3.11.15`
	`1`	`+ARG CASSANDRA_VERSION=3.11.16`
`2`	`2`
`3`	`3`	`FROM --platform=$BUILDPLATFORM maven:3.8.7-eclipse-temurin-11 as builder`
`4`	`4`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-ARG DSE_VERSION=6.8.37`
	`1`	`+ARG DSE_VERSION=6.8.38`
`2`	`2`	`ARG UBI_MAJOR=8`
`3`	`3`	`ARG UBI_BASETAG=latest`
`4`	`4`