Skip to content

Commit 57922d0

Browse files
conker84jexp
conker84
authored and
jexp
committed
fixes #88 (#89)
1 parent 935fa92 commit 57922d0

File tree

20 files changed

+169
-33
lines changed

20 files changed

+169
-33
lines changed

‎assembly/assembly.xml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
<assembly
2+
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
5+
<id>streams-assembly-all</id>
6+
<formats>
7+
<format>jar</format>
8+
</formats>
9+
<includeBaseDirectory>false</includeBaseDirectory> <!-- strip the module prefixes -->
10+
<dependencySets>
11+
<dependencySet>
12+
<unpack>true</unpack> <!-- unpack , then repack the jars -->
13+
<useTransitiveDependencies>false</useTransitiveDependencies> <!-- do not pull in any transitive dependencies -->
14+
</dependencySet>
15+
</dependencySets>
16+
<containerDescriptorHandlers>
17+
<containerDescriptorHandler>
18+
<handlerName>metaInf-services</handlerName>
19+
</containerDescriptorHandler>
20+
</containerDescriptorHandlers>
21+
</assembly>

‎consumer/pom.xml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66

77
<groupId>org.neo4j</groupId>
88
<artifactId>neo4j-streams-consumer</artifactId>
9-
<version>3.4.7.1</version>
109
<name>Neo4j Streams - Consumer</name>
1110
<description>Neo4j Streams - Kafka Consumer</description>
1211
<packaging>jar</packaging>
@@ -17,5 +16,4 @@
1716
<version>3.4.7.1</version>
1817
</parent>
1918

20-
2119
</project>

‎consumer/src/main/kotlin/streams/StreamsEventSink.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,16 @@ abstract class StreamsEventSink(private val config: Config,
1111

1212
abstract fun stop()
1313

14+
abstract fun start()
15+
1416
override fun unavailable() {
1517
stop()
1618
}
1719

20+
override fun available() {
21+
start()
22+
}
23+
1824
}
1925

2026
object StreamsEventSinkFactory {

‎consumer/src/main/kotlin/streams/StreamsEventSinkExtensionFactory.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ class StreamsEventSinkExtensionFactory : KernelExtensionFactory<StreamsEventSink
2929
private val configuration = dependencies.config()
3030
private var streamsLog = log.getUserLog(StreamsEventLifecycle::class.java)
3131

32-
lateinit var eventSink: StreamsEventSink
32+
private lateinit var eventSink: StreamsEventSink
3333

3434
override fun start() {
3535
try {

‎consumer/src/main/kotlin/streams/StreamsSinkConfiguration.kt‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,27 @@ import org.apache.commons.lang3.StringUtils
44
import org.neo4j.kernel.configuration.Config
55

66

7-
private const val streamsConfigPrefix = "streams."
8-
private const val streamsSinkTopicPrefix = "sink.topic.cypher."
7+
private object StreamsSinkConfigurationConstants {
8+
const val STREAMS_CONFIG_PREFIX: String = "streams."
9+
const val STREAMS_SINK_TOPIC_CYPHER_PREFIX: String = "sink.topic.cypher."
10+
const val ENABLED = "sink.enabled"
11+
}
912

10-
data class StreamsSinkConfiguration(val sinkPollingInterval: Long = Long.MAX_VALUE,
13+
data class StreamsSinkConfiguration(val enabled: Boolean = true,
14+
val sinkPollingInterval: Long = Long.MAX_VALUE,
1115
val topics: Map<String, String> = emptyMap()) {
1216

1317
companion object {
1418
fun from(cfg: Config): StreamsSinkConfiguration {
1519
val default = StreamsSinkConfiguration()
1620
val config = cfg.raw
17-
.filterKeys { it.startsWith(streamsConfigPrefix) }
18-
.mapKeys { it.key.substring(streamsConfigPrefix.length) }
21+
.filterKeys { it.startsWith(StreamsSinkConfigurationConstants.STREAMS_CONFIG_PREFIX) }
22+
.mapKeys { it.key.substring(StreamsSinkConfigurationConstants.STREAMS_CONFIG_PREFIX.length) }
1923
val topics = config
20-
.filterKeys { it.startsWith(streamsSinkTopicPrefix) }
21-
.mapKeys { it.key.replace(streamsSinkTopicPrefix, StringUtils.EMPTY) }
22-
return default.copy(sinkPollingInterval = config.getOrDefault("sink.polling.interval", default.sinkPollingInterval).toString().toLong(),
24+
.filterKeys { it.startsWith(StreamsSinkConfigurationConstants.STREAMS_SINK_TOPIC_CYPHER_PREFIX) }
25+
.mapKeys { it.key.replace(StreamsSinkConfigurationConstants.STREAMS_SINK_TOPIC_CYPHER_PREFIX, StringUtils.EMPTY) }
26+
return default.copy(enabled = config.getOrDefault(StreamsSinkConfigurationConstants.ENABLED, default.enabled).toString().toBoolean(),
27+
sinkPollingInterval = config.getOrDefault("sink.polling.interval", default.sinkPollingInterval).toString().toLong(),
2328
topics = topics)
2429
}
2530
}

‎consumer/src/main/kotlin/streams/kafka/KafkaEventSink.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,15 +40,15 @@ class KafkaEventSink: StreamsEventSink {
4040
this.db = db
4141
}
4242

43-
override fun available() {
43+
override fun start() {
4444
this.streamsTopicService = StreamsTopicService(this.db, kafkaConfig.streamsSinkConfiguration)
4545
if (streamsTopicService!!.getTopics().isEmpty()) {
4646
log.info("No topic configuration found under streams.sink.topic.*, Kafka Sink will not stared")
4747
return
4848
}
4949

5050
this.queryExecution = StreamsEventSinkQueryExecution(this.streamsTopicService!!, db, log)
51-
createConsumer();
51+
createConsumer()
5252

5353
job = createJob()
5454
log.info("Kafka Sink Connector started.")

‎consumer/src/main/kotlin/streams/kafka/KafkaSinkConfiguration.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,8 @@ data class KafkaSinkConfiguration(val zookeeperConnect: String = "localhost:2181
6161
.mapKeys { it.key.toPointCase() }
6262
props.putAll(map)
6363
props.putAll(extraProperties)
64-
props.putAll(addDeserializers())
65-
props[ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG] = true
64+
props.putAll(addDeserializers()) // Fixed deserializers
65+
props[ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG] = true // Fixed autocommit
6666
return props
6767
}
6868

‎distribution/pom.xml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
7+
<groupId>org.neo4j</groupId>
8+
<artifactId>neo4j-streams-distribution</artifactId>
9+
<name>Neo4j Streams - Distribution</name>
10+
<description>Neo4j Streams - Distribution Package</description>
11+
<packaging>pom</packaging>
12+
13+
<parent>
14+
<groupId>org.neo4j</groupId>
15+
<artifactId>neo4j-streams-parent</artifactId>
16+
<version>3.4.7.1</version>
17+
</parent>
18+
19+
<!-- Include here all the required dependencies of the final package -->
20+
<dependencies>
21+
<dependency>
22+
<groupId>org.neo4j</groupId>
23+
<artifactId>neo4j-streams-producer</artifactId>
24+
<version>${project.version}</version>
25+
</dependency>
26+
<dependency>
27+
<groupId>org.neo4j</groupId>
28+
<artifactId>neo4j-streams-consumer</artifactId>
29+
<version>${project.version}</version>
30+
</dependency>
31+
</dependencies>
32+
33+
<build>
34+
<plugins>
35+
<plugin>
36+
<artifactId>maven-assembly-plugin</artifactId>
37+
<version>2.4</version>
38+
<executions>
39+
<execution>
40+
<id>streams-assembly</id>
41+
<phase>package</phase> <!-- create assembly in package phase (invoke 'single' goal on assemby plugin)-->
42+
<goals>
43+
<goal>single</goal>
44+
</goals>
45+
<configuration>
46+
<descriptors>
47+
<descriptor>assembly/assembly.xml</descriptor>
48+
</descriptors>
49+
<appendAssemblyId>false</appendAssemblyId>
50+
<finalName>neo4j-streams-${project.version}</finalName>
51+
<appendAssemblyId>false</appendAssemblyId>
52+
<outputDirectory>../target</outputDirectory>
53+
</configuration>
54+
</execution>
55+
</executions>
56+
</plugin>
57+
</plugins>
58+
</build>
59+
</project>

‎doc/asciidoc/consumer/configuration.adoc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ kafka.zookeeper.connect=localhost:2181
66
kafka.bootstrap.servers=localhost:9092
77
kafka.auto.offset.reset=earliest
88
kafka.group.id=neo4j
9+
910
streams.sink.polling.interval=<The time, in milliseconds, spent waiting in poll if data is not available in the buffer. default=Long.MAX_VALUE>
1011
streams.sink.topic.cypher.<TOPIC_NAME>=<CYPHER_QUERY>
12+
streams.sink.enable=<true/false, default=true>
1113
----
1214

1315
See the https://kafka.apache.org/documentation/#brokerconfigs[Apache Kafka documentation] for details on these settings.

‎doc/asciidoc/producer/configuration.adoc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ kafka.transaction.id=
1818
1919
streams.source.topic.nodes.<TOPIC_NAME>=<PATTERN>
2020
streams.source.topic.relationships.<TOPIC_NAME>=<PATTERN>
21+
streams.source.enable=<true/false, default=true>
22+
streams.procedures.enable=<true/false, default=true>
2123
----
2224

2325
Note: To use the Kafka transactions please set `kafka.transaction.id` and `kafka.acks` properly

0 commit comments

Comments
 (0)