Beam SDK 2.55.0 (#772)

* Update Github maven workflow

* Upgrade to codeql v3

* Upgrade e2e

* Update README

* Keep hamcrest

* Beam 2.55.0

* Update Libraries BOM

* Bump byte-buddy for Java 21 support

Author: labianchin

.github/workflows/codeql-analysis.yml

@@ -18,7 +18,6 @@ jobs:
   analyze:
     name: Analyze
     runs-on: ubuntu-latest
-
     strategy:
       fail-fast: false
       matrix:
@@ -27,14 +26,12 @@ jobs:
         language: ['java']
         # Learn more...
         # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
-
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
-
+      uses: actions/checkout@v4
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -45,7 +42,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -59,4 +56,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v3

.github/workflows/maven.yml

@@ -17,22 +17,22 @@ on:
 
 jobs:
   test:
-    name: CI on JDK ${{ matrix.java_version }} and ${{ matrix.os }}
+    name: mvn JDK ${{ matrix.java_version }} and ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        java_version: ['11', '17']
+        java_version: ['11', '17', '21']
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java_version }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java_version }}
           distribution: 'adopt'  # use Corretto once supported https://github.com/actions/setup-java/issues/68
       # https://docs.github.com/en/actions/guides/building-and-testing-java-with-maven#caching-dependencies
       - name: Cache Maven packages
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2
           key: ${{ runner.os }}-java${{ matrix.java_version }}-m2-${{ hashFiles('**/pom.xml') }}
@@ -41,7 +41,7 @@ jobs:
         run: |
           mvn clean verify --activate-profiles=coverage,pack
       - name: Codecov upload
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./dbeam-core/target/site/jacoco/jacoco.xml
@@ -50,20 +50,15 @@ jobs:
           fail_ci_if_error: false # optional (default = false)
           verbose: true # optional (default = false)
       - name: End to end tests
+        if: matrix.java_version == '17'
         run: ./e2e/e2e.sh
       - run: mkdir staging && cp /tmp/debeam_e2e.log ./staging
-      - uses: actions/upload-artifact@v3
-        if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main') && matrix.java_version == '11'
+        if: matrix.java_version == '17'
+      - uses: actions/upload-artifact@v4
+        if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main') && matrix.java_version == '17'
         with:
           name: Package
           path: staging
-  update_release_draft:
-    runs-on: ubuntu-latest
-    steps:
-      # Drafts your next Release notes as Pull Requests are merged into "main"
-      - uses: release-drafter/release-drafter@v5
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   deploy:
     # https://docs.github.com/en/actions/guides/publishing-java-packages-with-maven#publishing-packages-to-github-packages
     name: Deploy
@@ -73,18 +68,19 @@ jobs:
     permissions:
       contents: read
       packages: write
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main')
+    # Disable it, since it was failing on sonatype deploy. TODO: fix
+    if: false && github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main')
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       # https://docs.github.com/en/actions/guides/building-and-testing-java-with-maven#caching-dependencies
       - name: Cache Maven packages
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2
           key: Linux-java11-m2-${{ hashFiles('**/pom.xml') }}
           restore-keys: Linux-java11-m2
       - name: Set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '11'
           distribution: 'adopt'  # use Corretto once supported https://github.com/actions/setup-java/issues/68

README.md

@@ -287,6 +287,14 @@ You can check the deployment in the following links:
 - https://github.com/spotify/dbeam/actions
 - https://oss.sonatype.org/#nexus-search;quick~dbeam-core
 
+## Future roadmap
+
+DBeam is mature, maintained and used in production since August 2017. No major features or development planned.
+Like Redis/[Redict](https://andrewkelley.me/post/redis-renamed-to-redict.html), DBeam can be considered a finished product.
+
+> It can be maintained for decades to come with minimal effort. It can continue to provide a high amount of value for a low amount of labor.
+
+
 ---
 
 ## License

e2e/e2e.sh

@@ -10,12 +10,12 @@ readonly PROJECT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null && pw
 
 # This file contatins psql views with complex types to validate and troubleshoot dbeam
 
-PSQL_DOCKER_IMAGE=postgres:10
+PSQL_DOCKER_IMAGE=postgres:16
 PSQL_USER=postgres
-PSQL_PASSWORD=mysecretpassword
+PSQL_PASSWORD=tempandnotasecret
 PSQL_DB=dbeam_test
 DOCKER_NETWORK=dbeam1-network
-JAVA_DOCKER_IMAGE=gcr.io/distroless/java:11
+JAVA_DOCKER_IMAGE=gcr.io/distroless/java17-debian12
 
 startPostgres() {
   set -o xtrace
@@ -26,7 +26,7 @@ startPostgres() {
   docker run --detach --name dbeam-postgres \
     --net "$DOCKER_NETWORK" \
     --env "POSTGRES_DB=dbeam_test" \
-    --env "POSTGRES_PASSWORD=mysecretpassword" \
+    --env "POSTGRES_PASSWORD=$PSQL_PASSWORD" \
     --mount="type=bind,source=/tmp/pgdata,target=/var/lib/postgresql/data" \
     --publish="54321:5432/tcp" "$PSQL_DOCKER_IMAGE" || docker start dbeam-postgres
   sleep 1
@@ -35,13 +35,13 @@ startPostgres() {
   # https://stackoverflow.com/questions/35069027/docker-wait-for-postgresql-to-be-running
   time docker run --interactive --rm \
     --net "$DOCKER_NETWORK" \
-    --env "PGPASSWORD=mysecretpassword" \
+    --env "PGPASSWORD=$PSQL_PASSWORD" \
     "$PSQL_DOCKER_IMAGE" \
     timeout 45s bash -xc 'until psql -h dbeam-postgres -U postgres dbeam_test -c "select 1"; do sleep 1; done; echo "psql up and running.."'
   sleep 3
   time docker run --interactive --rm \
     --net "$DOCKER_NETWORK" \
-    --env "PGPASSWORD=mysecretpassword" \
+    --env "PGPASSWORD=$PSQL_PASSWORD" \
     "$PSQL_DOCKER_IMAGE" \
     timeout 30s psql -h dbeam-postgres -U postgres dbeam_test < "$SCRIPT_PATH/ddl.sql"
   timeout 1 bash -c "cat < /dev/null > /dev/tcp/0.0.0.0/54321" && echo "success"

pom.xml

@@ -104,11 +104,11 @@
     <avro.version>1.11.3</avro.version>
     <bouncycastle.version>1.77</bouncycastle.version>
     <!--Ensure Beam SDK compatibility-->
-    <!-- https://github.com/apache/beam/blob/release-2.54.0/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy#L600 -->
-    <beam.version>2.54.0</beam.version>
-    <!--https://storage.googleapis.com/cloud-opensource-java-dashboard/com.google.cloud/libraries-bom/26.32.0/index.html -->
-    <google-cloud-libraries-bom.version>26.32.0</google-cloud-libraries-bom.version>
-    <guava.version>33.0.0-jre</guava.version>
+    <!-- https://github.com/apache/beam/blob/release-2.55.0/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy#L600 -->
+    <beam.version>2.55.0</beam.version>
+    <!--https://storage.googleapis.com/cloud-opensource-java-dashboard/com.google.cloud/libraries-bom/26.34.0/index.html -->
+    <google-cloud-libraries-bom.version>26.34.0</google-cloud-libraries-bom.version>
+    <guava.version>33.1.0-jre</guava.version>
     <hamcrest.version>2.2</hamcrest.version>
     <junit.version>4.13.2</junit.version>
     <jackson.version>2.16.2</jackson.version>
@@ -161,13 +161,28 @@
       <dependency>
         <groupId>com.google.apis</groupId>
         <artifactId>google-api-services-storage</artifactId>
-        <version>v1-rev20240202-2.0.0</version>
+        <version>v1-rev20240209-2.0.0</version>
       </dependency>
       <dependency>
         <groupId>commons-codec</groupId>
         <artifactId>commons-codec</artifactId>
         <version>1.16.1</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.commons</groupId>
+        <artifactId>commons-compress</artifactId>
+        <version>1.26.1</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.commons</groupId>
+        <artifactId>commons-lang3</artifactId>
+        <version>3.14.0</version>
+      </dependency>
+      <dependency>
+        <groupId>net.bytebuddy</groupId>
+        <artifactId>byte-buddy</artifactId>
+        <version>1.14.12</version>
+      </dependency>
       <!-- overrides to resolve dependency conflicts - end-->
       <!-- overrides with vulnerability fixes - start-->
       <dependency>
@@ -249,12 +264,6 @@
         <groupId>org.apache.beam</groupId>
         <artifactId>beam-runners-direct-java</artifactId>
         <version>${beam.version}</version>
-        <exclusions>
-          <exclusion>
-            <groupId>org.hamcrest</groupId>
-            <artifactId>hamcrest</artifactId>
-          </exclusion>
-        </exclusions>
       </dependency>
       <dependency>
         <groupId>org.apache.beam</groupId>
@@ -265,10 +274,6 @@
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
           </exclusion>
-          <exclusion>
-            <groupId>org.hamcrest</groupId>
-            <artifactId>hamcrest</artifactId>
-          </exclusion>
           <exclusion>
             <groupId>com.google.api.grpc</groupId>
             <artifactId>grpc-google-cloud-pubsub-v1</artifactId>
@@ -380,6 +385,14 @@
         <artifactId>mysql-socket-factory-connector-j-8</artifactId>
         <version>${socket-factory.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.hamcrest</groupId>
+        <artifactId>hamcrest</artifactId>
+        <version>${hamcrest.version}</version>
+        <!-- Needed for PipelineOptionRegistrar auto-service that registered TestPipelineOptions interfaces-->
+        <!--https://github.com/apache/beam/issues/18593-->
+        <!--https://github.com/apache/beam/pull/30637/files-->
+      </dependency>
 
       <!-- Test Deps -->