[CYB-207] Support CSA 1.12.0 CDH 7.1.9 (#74)

* Datahub 1.10.0 pom changes * [CYB-172] kafka topic created from tableapi kafka connector need ssb properties * Datahub 1.10.0 pom changes * [CYB-190] user can specify names of CDP cloud artifacts in example create_datahub_config.sh setup script * [CYB-190] user can specify names of CDP cloud artifacts in example create_datahub_config.sh setup script * [CYB-191] schemas produced for topics can't be consumed by SMM because of watermark properties * [CYB-206] dev log removed * [CYB-207] cybersec toolkit supports CSA 1.12.0 with CDP 7.1.9 * [CYB-207] fix indexing job serialization error * [CYB-207] add missing jupiter dependency. use jupiter for all tests * [CYB-207] remove deprecated forkMode for rest enrichments module * [CYB-207] add junit dependency * [CYB-207] use https for mock server for all tests. * [CYB-207] temporary change to run just problem module * [CYB-207] update mockserver dependency * Revert "[CYB-207] temporary change to run just problem module" This reverts commit ba0a0a9. * Revert "[CYB-207] use https for mock server for all tests." This reverts commit 1fe5223. * [CYB-207] Update mockserver dependency to remove conflicts * typo fix --------- Co-authored-by: cduby <[email protected]> Co-authored-by: Stas Panasiuk <[email protected]> Co-authored-by: stas-panasiuk <[email protected]>
cloudera · Oct 9, 2024 · 5199d12 · 5199d12
1 parent 0512dcd
commit 5199d12
Show file tree

Hide file tree

Showing 37 changed files with 459 additions and 179 deletions.
diff --git a/flink-cyber/caracal-generator/pom.xml b/flink-cyber/caracal-generator/pom.xml
@@ -33,6 +33,13 @@
             <artifactId>flink-connector-kafka</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-clients</artifactId>
+            <scope>provided</scope>
+            <version>${flink.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>com.hortonworks.smm</groupId>
             <artifactId>monitoring-interceptors</artifactId>

diff --git a/flink-cyber/caracal-parser/pom.xml b/flink-cyber/caracal-parser/pom.xml
@@ -126,8 +126,12 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
-            <version>${flink.version}</version>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.hortonworks.registries</groupId>
+            <artifactId>schema-registry-serdes</artifactId>
         </dependency>
 
         <dependency>
@@ -164,6 +168,11 @@
             <artifactId>flink-orc</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.orc</groupId>
+            <artifactId>orc-core</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.parquet</groupId>
             <artifactId>parquet-avro</artifactId>

diff --git a/...cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java b/...cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java
@@ -54,7 +54,7 @@ public long getTs() {
         return message.getTs();
     }
 
-    public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(ScoredMessage.class.getPackage().getName(), ScoredMessage.class.getName())
+    public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(ScoredMessage.class.getPackage().getName(), ScoredMessage.class.getName(), null)
             .fields()
             .name("message").type(Message.SCHEMA$).noDefault()
             .name("cyberScoresDetails").type(Schema.createArray(Scores.SCHEMA$)).noDefault()

diff --git a/flink-cyber/flink-alert-scoring/pom.xml b/flink-cyber/flink-alert-scoring/pom.xml
@@ -95,6 +95,11 @@
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-statebackend-rocksdb</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <scope>test</scope>
+        </dependency>
 
     </dependencies>
     <build>

diff --git a/flink-cyber/flink-commands/json-commands/pom.xml b/flink-cyber/flink-commands/json-commands/pom.xml
@@ -70,15 +70,14 @@
             <version>${log4j.version}</version>
             <scope>provided</scope>
         </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>${guava.version}</version>
-        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>${jackson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <scope>test</scope>
         </dependency>
     </dependencies>
 

diff --git a/flink-cyber/flink-commands/scoring-commands/pom.xml b/flink-cyber/flink-commands/scoring-commands/pom.xml
@@ -60,6 +60,14 @@
             <version>${jackson.datatype.version}</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>com.hortonworks.registries</groupId>
+            <artifactId>schema-registry-serdes</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
         <!-- Logging and Metrics -->
         <dependency>
             <groupId>org.slf4j</groupId>
@@ -90,6 +98,10 @@
             <artifactId>avro</artifactId>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-java</artifactId>
+        </dependency>
     </dependencies>
     <build>
         <plugins>
@@ -128,6 +140,12 @@
                                     </excludes>
                                 </filter>
                             </filters>
+                            <relocations>
+                                <relocation>
+                                    <pattern>com.google.</pattern>
+                                    <shadedPattern>com.cloudera.cyber.com.google.</shadedPattern>
+                                </relocation>
+                            </relocations>
                             <transformers>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                     <mainClass>com.cloudera.cyber.scoring.ScoringJobKafka</mainClass>

diff --git a/flink-cyber/flink-common/pom.xml b/flink-cyber/flink-common/pom.xml
@@ -43,9 +43,22 @@
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-connector-kafka</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka-clients</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.hortonworks.registries</groupId>
+            <artifactId>schema-registry-serdes</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-base</artifactId>
+            <version>${flink.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.avro</groupId>

diff --git a/flink-cyber/flink-cyber-api/pom.xml b/flink-cyber/flink-cyber-api/pom.xml
@@ -32,6 +32,19 @@
         </plugins>
     </build>
 
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <!-- Import dependency management from Spring Boot -->
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-dependencies</artifactId>
+                <version>${spring-boot.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+     </dependencyManagement>
+
     <dependencies>
         <dependency>
             <groupId>org.projectlombok</groupId>
@@ -55,10 +68,15 @@
             <artifactId>flink-table-common</artifactId>
         </dependency>
 
+
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-core</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.avro</groupId>
             <artifactId>avro</artifactId>
-            <version>${avro.version}</version>
         </dependency>
 
         <dependency>

diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java
@@ -18,6 +18,7 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.NonNull;
+import org.apache.avro.LogicalTypes;
 import org.apache.avro.Schema;
 import org.apache.avro.SchemaBuilder;
 import org.apache.avro.specific.SpecificRecord;
@@ -56,7 +57,7 @@ public class Message extends SpecificRecordBase implements SpecificRecord, Ident
     public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(Message.class.getPackage().getName(), Message.class.getName())
             .fields()
             .requiredString("id")
-            .requiredLong("ts")
+            .name("ts").type(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG))).noDefault()
             .name("originalSource").type(SignedSourceKey.SCHEMA$).noDefault()
             .requiredString("message")
             .name("threats").type().optional().type(SchemaBuilder.map().values(SchemaBuilder.array().items(ThreatIntelligence.SCHEMA$)))

diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java
@@ -5,8 +5,16 @@
 public class AvroSchemas {
 
     public static SchemaBuilder.RecordBuilder<org.apache.avro.Schema> createRecordBuilder(String namespace, String recordName) {
-        return SchemaBuilder.record(recordName).namespace(namespace)
-                .prop("ssb.rowtimeAttribute", "ts")
-                .prop("ssb.watermarkExpression", "`ts` - INTERVAL '30' SECOND");
+        return createRecordBuilder(namespace, recordName, "ts");
+    }
+
+    public static SchemaBuilder.RecordBuilder<org.apache.avro.Schema> createRecordBuilder(String namespace, String recordName, String tsFieldName) {
+        SchemaBuilder.RecordBuilder<org.apache.avro.Schema> recordBuilder = SchemaBuilder.record(recordName).namespace(namespace);
+        if (tsFieldName != null) {
+            recordBuilder
+                    .prop("ssb.rowtimeAttribute", tsFieldName)
+                    .prop("ssb.watermarkExpression", String.format("`%s` - INTERVAL '30' SECOND", tsFieldName));
+        }
+        return recordBuilder;
     }
 }
diff --git a/flink-cyber/flink-dedupe/pom.xml b/flink-cyber/flink-dedupe/pom.xml
@@ -45,7 +45,12 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.hortonworks.registries</groupId>
+            <artifactId>schema-registry-serdes</artifactId>
         </dependency>
 
         <dependency>

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml
@@ -90,6 +90,11 @@
             <classifier>tests</classifier>
         </dependency>
 
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.hortonworks.smm</groupId>
             <artifactId>monitoring-interceptors</artifactId>

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml
@@ -156,6 +156,11 @@
             <version>${commons-validator.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.assertj</groupId>
             <artifactId>assertj-core</artifactId>

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-load/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-load/pom.xml
@@ -72,25 +72,19 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-connector-hbase-${flink.hbase.version}</artifactId>
-            <version>${flink.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.glassfish</groupId>
-                    <artifactId>javax.el</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.hbase</groupId>
-                    <artifactId>hbase-client</artifactId>
-                </exclusion>
-            </exclusions>
         </dependency>
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
             <scope>compile</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.hortonworks.registries</groupId>
+            <artifactId>schema-registry-serdes</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.cloudera.cyber</groupId>
             <artifactId>flink-cyber-api</artifactId>

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/pom.xml
@@ -39,6 +39,14 @@
             <artifactId>lombok</artifactId>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
 
     </dependencies>
 

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/pom.xml
@@ -52,21 +52,19 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-connector-hbase-${flink.hbase.version}</artifactId>
-            <version>${flink.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.glassfish</groupId>
-                    <artifactId>javax.el</artifactId>
-                </exclusion>
-            </exclusions>
         </dependency>
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
             <scope>provided</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.hortonworks.registries</groupId>
+            <artifactId>schema-registry-serdes</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.cloudera.cyber</groupId>
             <artifactId>flink-cyber-api</artifactId>

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/pom.xml
@@ -50,10 +50,15 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
             <scope>provided</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.hortonworks.registries</groupId>
+            <artifactId>schema-registry-serdes</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.cloudera.cyber</groupId>
             <artifactId>flink-cyber-api</artifactId>

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/pom.xml
@@ -51,10 +51,15 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
             <scope>provided</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.hortonworks.registries</groupId>
+            <artifactId>schema-registry-serdes</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.cloudera.cyber</groupId>
             <artifactId>flink-cyber-api</artifactId>
@@ -118,10 +123,14 @@
             <version>${global.httpclient.version}</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.mock-server</groupId>
-            <artifactId>mockserver-netty</artifactId>
-            <version>5.11.1</version>
+            <artifactId>mockserver-netty-no-dependencies</artifactId>
+            <version>RELEASE</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -196,8 +205,8 @@
                 <artifactId>maven-surefire-plugin</artifactId>
                 <version>${maven-surefire-plugin.version}</version>
                 <configuration>
-                    <!-- use separate jvm to run each test.  mockserver secure and insecure get confused if tests run in the same jvm -->
-                    <forkMode>always</forkMode>
+                    <forkCount>1</forkCount>
+                    <reuseForks>false</reuseForks>
                 </configuration>
             </plugin>
         </plugins>