-
Notifications
You must be signed in to change notification settings - Fork 257
/
Copy pathMongoSourceConfig.java
1553 lines (1397 loc) · 67.2 KB
/
MongoSourceConfig.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* Copyright 2008-present MongoDB, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.mongodb.kafka.connect.source;
import static com.mongodb.kafka.connect.source.MongoSourceTask.LOGGER;
import static com.mongodb.kafka.connect.source.SourceConfigSoftValidator.logIncompatibleProperties;
import static com.mongodb.kafka.connect.source.SourceConfigSoftValidator.logObsoleteProperties;
import static com.mongodb.kafka.connect.source.schema.AvroSchemaDefaults.DEFAULT_AVRO_KEY_SCHEMA;
import static com.mongodb.kafka.connect.source.schema.AvroSchemaDefaults.DEFAULT_AVRO_VALUE_SCHEMA;
import static com.mongodb.kafka.connect.util.Assertions.assertFalse;
import static com.mongodb.kafka.connect.util.Assertions.assertNotNull;
import static com.mongodb.kafka.connect.util.Assertions.fail;
import static com.mongodb.kafka.connect.util.ClassHelper.createInstance;
import static com.mongodb.kafka.connect.util.ConfigHelper.collationFromJson;
import static com.mongodb.kafka.connect.util.ConfigHelper.fullDocumentBeforeChangeFromString;
import static com.mongodb.kafka.connect.util.ConfigHelper.fullDocumentFromString;
import static com.mongodb.kafka.connect.util.ConfigHelper.jsonArrayFromString;
import static com.mongodb.kafka.connect.util.ServerApiConfig.addServerApiConfig;
import static com.mongodb.kafka.connect.util.SslConfigs.addSslConfigDef;
import static com.mongodb.kafka.connect.util.Validators.emptyString;
import static com.mongodb.kafka.connect.util.Validators.errorCheckingPasswordValueValidator;
import static com.mongodb.kafka.connect.util.Validators.errorCheckingValueValidator;
import static com.mongodb.kafka.connect.util.VisibleForTesting.AccessModifier.PACKAGE;
import static java.lang.String.format;
import static java.util.Arrays.asList;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static java.util.Collections.unmodifiableList;
import static org.apache.kafka.common.config.ConfigDef.Width;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.kafka.common.config.AbstractConfig;
import org.apache.kafka.common.config.ConfigDef;
import org.apache.kafka.common.config.ConfigDef.Importance;
import org.apache.kafka.common.config.ConfigDef.Type;
import org.apache.kafka.common.config.ConfigValue;
import org.bson.BsonTimestamp;
import org.bson.Document;
import org.bson.json.JsonWriterSettings;
import com.mongodb.ConnectionString;
import com.mongodb.annotations.Immutable;
import com.mongodb.client.model.Collation;
import com.mongodb.client.model.changestream.FullDocument;
import com.mongodb.client.model.changestream.FullDocumentBeforeChange;
import com.mongodb.lang.Nullable;
import com.mongodb.kafka.connect.source.MongoSourceConfig.StartupConfig.StartupMode;
import com.mongodb.kafka.connect.source.json.formatter.JsonWriterSettingsProvider;
import com.mongodb.kafka.connect.source.producer.BinaryOutboxSchemaAndValueProducer;
import com.mongodb.kafka.connect.source.schema.AvroSchema;
import com.mongodb.kafka.connect.source.topic.mapping.BinaryOutboxTopicMapper;
import com.mongodb.kafka.connect.source.topic.mapping.TopicMapper;
import com.mongodb.kafka.connect.util.ConfigHelper;
import com.mongodb.kafka.connect.util.ConnectConfigException;
import com.mongodb.kafka.connect.util.Validators;
import com.mongodb.kafka.connect.util.VisibleForTesting;
import com.mongodb.kafka.connect.util.config.BsonTimestampParser;
public class MongoSourceConfig extends AbstractConfig {
private static final String EMPTY_STRING = "";
private static final Pattern CLASS_NAME =
Pattern.compile("\\p{javaJavaIdentifierStart}\\p{javaJavaIdentifierPart}*");
private static final Pattern FULLY_QUALIFIED_CLASS_NAME =
Pattern.compile("(" + CLASS_NAME + "\\.)*" + CLASS_NAME);
public static final String CONNECTION_URI_CONFIG = "connection.uri";
private static final String CONNECTION_URI_DEFAULT =
"mongodb://localhost:27017,localhost:27018,localhost:27019";
private static final String CONNECTION_URI_DISPLAY = "MongoDB Connection URI";
private static final String CONNECTION_URI_DOC =
"The connection URI as supported by the official drivers. "
+ "eg: ``mongodb://user@pass@locahost/``.";
public static final String OUTPUT_FORMAT_KEY_CONFIG = "output.format.key";
private static final String OUTPUT_FORMAT_KEY_DEFAULT =
OutputFormat.JSON.name().toLowerCase(Locale.ROOT);
private static final String OUTPUT_FORMAT_KEY_DISPLAY = "The key output format";
private static final String OUTPUT_FORMAT_KEY_DOC =
"The output format of the data produced by the connector for the key. Supported formats are:\n"
+ " * `json` - Raw Json strings\n"
+ " * `bson` - Bson byte array\n"
+ " * `schema` - Schema'd output\n";
public static final String OUTPUT_FORMAT_VALUE_CONFIG = "output.format.value";
private static final String OUTPUT_FORMAT_VALUE_DEFAULT =
OutputFormat.JSON.name().toLowerCase(Locale.ROOT);
private static final String OUTPUT_FORMAT_VALUE_DISPLAY = "The value output format";
private static final String OUTPUT_FORMAT_VALUE_DOC =
"The output format of the data produced by the connector for the value. Supported formats are:\n"
+ " * `json` - Raw Json strings\n"
+ " * `bson` - Bson byte array\n"
+ " * `schema` - Schema'd output\n"
+ " * `binary` - Binary output\n";
public static final String OUTPUT_JSON_FORMATTER_CONFIG = "output.json.formatter";
private static final String OUTPUT_JSON_FORMATTER_DEFAULT =
"com.mongodb.kafka.connect.source.json.formatter.DefaultJson";
private static final String OUTPUT_JSON_FORMATTER_DISPLAY = "The json formatter class";
private static final String OUTPUT_JSON_FORMATTER_DOC =
"The output format of json strings can be configured to be either:\n"
+ " * com.mongodb.kafka.connect.source.json.formatter.DefaultJson: The legacy strict json formatter.\n"
+ " * com.mongodb.kafka.connect.source.json.formatter.ExtendedJson: The fully type safe extended json formatter.\n"
+ " * com.mongodb.kafka.connect.source.json.formatter.SimplifiedJson: Simplified Json, "
+ "with ObjectId, Decimals, Dates and Binary values represented as strings.\n\n"
+ "Users can provide their own implementation of the com.mongodb.kafka.connect.source.json.formatter.";
public static final String OUTPUT_SCHEMA_KEY_CONFIG = "output.schema.key";
private static final String OUTPUT_SCHEMA_KEY_DEFAULT = DEFAULT_AVRO_KEY_SCHEMA;
private static final String OUTPUT_SCHEMA_KEY_DISPLAY = "The Avro schema definition for the key.";
private static final String OUTPUT_SCHEMA_KEY_DOC =
"The Avro schema definition for the key value of the SourceRecord.";
public static final String OUTPUT_SCHEMA_VALUE_CONFIG = "output.schema.value";
private static final String OUTPUT_SCHEMA_VALUE_DEFAULT = DEFAULT_AVRO_VALUE_SCHEMA;
private static final String OUTPUT_SCHEMA_VALUE_DISPLAY =
"The Avro schema definition for the value.";
private static final String OUTPUT_SCHEMA_VALUE_DOC =
"The Avro schema definition for the value of the SourceRecord.";
public static final String OUTPUT_SCHEMA_INFER_VALUE_CONFIG = "output.schema.infer.value";
public static final boolean OUTPUT_SCHEMA_INFER_VALUE_DEFAULT = false;
private static final String OUTPUT_SCHEMA_INFER_VALUE_DOC =
"Infer the schema for the value. "
+ "Each Document will be processed in isolation, which may lead to multiple schema definitions "
+ "for the data. Only applied when "
+ OUTPUT_FORMAT_VALUE_CONFIG
+ " is set to schema.";
private static final String OUTPUT_SCHEMA_INFER_VALUE_DISPLAY =
"Enable Infer Schemas for the value";
public static final String TOPIC_MAPPER_CONFIG = "topic.mapper";
private static final String TOPIC_MAPPER_DISPLAY = "The topic mapper class";
private static final String TOPIC_MAPPER_DOC =
"The class that determines the topic to write the source data to. "
+ "By default this will be based on the 'ns' field in the change stream document, "
+ "along with any configured prefix and suffix.";
private static final String TOPIC_MAPPER_DEFAULT =
"com.mongodb.kafka.connect.source.topic.mapping.DefaultTopicMapper";
public static final String TOPIC_SEPARATOR_CONFIG = "topic.separator";
public static final String TOPIC_SEPARATOR_DEFAULT = ".";
private static final String TOPIC_SEPARATOR_DISPLAY = "The topic separator";
private static final String TOPIC_SEPARATOR_DOC =
"Separator to use when joining prefix, database & collection names, suffix to generate the name of the Kafka "
+ "topic to publish data to. Used by the 'DefaultTopicMapper'.";
public static final String TOPIC_PREFIX_CONFIG = "topic.prefix";
private static final String TOPIC_PREFIX_DOC =
"Prefix to prepend to database & collection names to generate the name of the Kafka "
+ "topic to publish data to. Used by the 'DefaultTopicMapper'.";
private static final String TOPIC_PREFIX_DISPLAY = "The topic prefix";
private static final String TOPIC_PREFIX_DEFAULT = EMPTY_STRING;
public static final String TOPIC_SUFFIX_CONFIG = "topic.suffix";
private static final String TOPIC_SUFFIX_DOC =
"Suffix to append to database & collection names to generate the name of the Kafka "
+ "topic to publish data to. Used by the 'DefaultTopicMapper'.";
private static final String TOPIC_SUFFIX_DISPLAY = "The topic suffix";
private static final String TOPIC_SUFFIX_DEFAULT = EMPTY_STRING;
public static final String TOPIC_NAMESPACE_MAP_CONFIG = "topic.namespace.map";
private static final String TOPIC_NAMESPACE_MAP_DISPLAY = "The namespace to topic map";
private static final String TOPIC_NAMESPACE_MAP_DOC =
"A JSON object specifying how to map a MongoDB change stream document namespace"
+ " to a Kafka topic name. Used by the `DefaultTopicMapper`."
+ " MongoDB change stream document namespace is"
+ " a database name optionally concatenated with a collection name, separated by full stop '.'."
+ "\nThe name in each JSON name/value pair is a namespace pattern,"
+ " the value is a string representing the corresponding topic name template."
+ " Pairs are ordered. When there are multiple pairs with equal namespace patterns (duplicates),"
+ " they are deduplicated and only one pair is taken into account:"
+ " its position is taken from the first pair among duplicates,"
+ " its topic name template is taken from the last pair among duplicates."
+ " After deduplication, pairs with an empty topic name template are ignored when computing topic names."
+ " Note that a topic name computed based on this configuration is then decorated"
+ " using the `topic.prefix` and `topic.suffix` configuration properties."
+ "\nThere are three kinds of pairs:"
+ "\n- Simple. The namespace pattern must not contain solidus '/' and can be either of the following:"
+ "\n - A namespace with a collection name, in which case it matches only that namespace."
+ " The topic name template is interpreted as the topic name."
+ "\n - A namespace with only a database name, in which case it matches any namespace having that database name."
+ " The matching namespace may either have a collection name, or not:"
+ "\n - If there is a collection name, then the topic name is computed"
+ " by concatenating the topic name template and the collection name from the matching namespace, separated by `topic.separator`."
+ "\n - If there is no collection name, then the topic name template is interpreted as the topic name."
+ "\n- Regex. The namespace pattern starts with solidus '/',"
+ " followed by a regular expression with the syntax and behavior as per `java.util.regex.Pattern`."
+ " The topic name is computed by doing variable expansion on the topic name template."
+ " The following variables are supported:"
+ "\n - `db` The database name from the matching namespace."
+ "\n - `sep` The value of the `topic.separator` configuration property."
+ "\n - `coll` The collection name from the matching namespace, or an empty string if there is no collection name."
+ "\n - `sep_coll` The value of the `coll` variable"
+ " prefixed with the value of `sep` if and only if the value of `coll` is not empty."
+ "\n - `coll_sep` The value of the `coll` variable"
+ " suffixed with the value of `sep` if and only if the value of `coll` is not empty."
+ "\n - `sep_coll_sep` The value of the `coll` variable"
+ " prefixed and suffixed with the value of `sep` if and only if the value of `coll` is not empty."
+ "\n To be expanded, a variable must be enclosed between curly brackets '{' and '}', for example '{db}'."
+ " The characters '{' and '}' are not allowed to be used in the topic name template for any other purpose."
+ " Note that variable names are case-sensitive."
+ "\nBe careful when creating a namespace pattern with characters that need escaping according to the JSON syntax."
+ " For example, if you want to match full stop '.', the regex syntax requires escaping it as '\\.'."
+ " However, reverse solidus '\\' itself must be escaped as '\\\\' according to the JSON syntax."
+ " Consequently, to match '.' you need to write '\\\\.'."
+ "\n- Wildcard. The namespace pattern is asterisk '*' and matches any namespace."
+ " The topic name template is interpreted as the topic name."
+ "\n The matching order:"
+ "\n1. Simple pairs with a collection name in the namespace pattern."
+ "\n2. Simple pairs without a collection name in the namespace pattern."
+ "\n3. Regex pairs in order."
+ "\n4. The wildcard pair."
+ "\n Matching stops as soon as the first match is found. If no matches are found,"
+ " the topic name is computed solely based on the namespace. The namespace may either have a collection name, or not:"
+ "\n- If there is a collection name, then the topic name is computed"
+ " by concatenating the database name and the collection name, separated by `topic.separator`."
+ "\n- If there is no collection name, then the database name is used as the topic name."
+ "\nExamples (`topic.separator` is assumed to be '-'):"
+ "\n1. '{\"myDb\": \"topicTwo\", \"myDb.myColl\": \"topicOne\"}'"
+ " The 'myDb.myColl' namespace is mapped to the 'topicOne' topic name."
+ " All other namespaces with the 'myDb' database name are mapped"
+ " either to the 'topicTwo-<collection name>' topic name, if they have a collection name,"
+ " or to the 'topicTwo' topic otherwise."
+ " All other namespaces are mapped"
+ " either to the '<database name>-<collection name>' topic name, if they have a collection name,"
+ " or to the '<database name>' topic otherwise."
+ "\n2. '{\"/myDb(?:\\\\..*)?\": \"topicTwo{sep_coll}\", \"*\": \"topicThree\", \"myDb.myColl\": \"topicOne\"}'"
+ " The regex namespace pattern matches any namespace with the 'myDb' database name."
+ " The 'myDb.myColl' namespace is mapped to the 'topicOne' topic name."
+ " All other namespaces with the 'myDb' database name are mapped"
+ " either to the 'topicTwo-<collection name>' topic name, if they have a collection name,"
+ " or to the 'topicTwo' topic otherwise."
+ " All other namespaces are mapped to the 'topicThree' topic name.";
private static final String TOPIC_NAMESPACE_MAP_DEFAULT = EMPTY_STRING;
public static final String PIPELINE_CONFIG = "pipeline";
private static final String PIPELINE_DISPLAY = "The pipeline to apply to the change stream";
private static final String PIPELINE_DOC =
"An inline JSON array with objects describing the pipeline operations to run.\n"
+ "Example: `[{\"$match\": {\"operationType\": \"insert\"}}, {\"$addFields\": {\"Kafka\": \"Rules!\"}}]`";
private static final String PIPELINE_DEFAULT = "[]";
public static final String BATCH_SIZE_CONFIG = "batch.size";
private static final String BATCH_SIZE_DISPLAY = "The cursor batch size";
private static final String BATCH_SIZE_DOC = "The cursor batch size.";
private static final int BATCH_SIZE_DEFAULT = 0;
public static final String PUBLISH_FULL_DOCUMENT_ONLY_CONFIG = "publish.full.document.only";
private static final String PUBLISH_FULL_DOCUMENT_ONLY_DISPLAY =
"Publish only the `fullDocument` field";
private static final String PUBLISH_FULL_DOCUMENT_ONLY_DOC =
"Only publish the actual changed document rather than the full change "
+ "stream document. Automatically, sets `change.stream.full.document=updateLookup` so updated documents will be included.";
private static final boolean PUBLISH_FULL_DOCUMENT_ONLY_DEFAULT = false;
public static final String PUBLISH_FULL_DOCUMENT_ONLY_TOMBSTONE_ON_DELETE_CONFIG =
"publish.full.document.only.tombstone.on.delete";
private static final boolean PUBLISH_FULL_DOCUMENT_ONLY_TOMBSTONE_ON_DELETE_DEFAULT = false;
private static final String PUBLISH_FULL_DOCUMENT_ONLY_TOMBSTONE_ON_DELETE_DISPLAY =
format(
"Send a null value on a delete event. Requires %s=true.",
PUBLISH_FULL_DOCUMENT_ONLY_CONFIG);
private static final String PUBLISH_FULL_DOCUMENT_ONLY_TOMBSTONE_ON_DELETE_DOC =
format(
"Send a null value on a delete event. Requires %s=true. Defaults to: %s",
PUBLISH_FULL_DOCUMENT_ONLY_CONFIG,
PUBLISH_FULL_DOCUMENT_ONLY_TOMBSTONE_ON_DELETE_DEFAULT);
public static final String FULL_DOCUMENT_BEFORE_CHANGE_CONFIG =
"change.stream.full.document.before.change";
private static final String FULL_DOCUMENT_BEFORE_CHANGE_DISPLAY =
"The `fullDocumentBeforeChange` configuration.";
private static final String FULL_DOCUMENT_BEFORE_CHANGE_DOC =
"Specifies the pre-image configuration when creating a Change Stream.\n"
+ "The pre-image is not available in source records published while copying existing data as a result of"
+ " enabling `copy.existing`, and the pre-image configuration has no effect on copying.\n"
+ " Requires MongoDB 6.0 or above."
+ "See https://www.mongodb.com/docs/manual/reference/method/db.collection.watch/ for more details and possible values.";
private static final String FULL_DOCUMENT_BEFORE_CHANGE_DEFAULT = EMPTY_STRING;
public static final String FULL_DOCUMENT_CONFIG = "change.stream.full.document";
private static final String FULL_DOCUMENT_DISPLAY = "The `fullDocument` configuration.";
private static final String FULL_DOCUMENT_DOC =
"Determines what to return for update operations when using a Change Stream.\n"
+ "When set to 'updateLookup', the change stream for partial updates will include both a delta "
+ "describing the changes to the document as well as a copy of the entire document that was changed from *some time* after "
+ "the change occurred.\n"
+ "See https://www.mongodb.com/docs/manual/reference/method/db.collection.watch/ for more details and possible values.";
private static final String FULL_DOCUMENT_DEFAULT = EMPTY_STRING;
public static final String COLLATION_CONFIG = "collation";
private static final String COLLATION_DISPLAY = "The collation options";
private static final String COLLATION_DOC =
"The json representation of the Collation options to use for the change stream.\n"
+ "Use the `Collation.asDocument().toJson()` to create the specific json representation.";
private static final String COLLATION_DEFAULT = EMPTY_STRING;
public static final String POLL_MAX_BATCH_SIZE_CONFIG = "poll.max.batch.size";
private static final String POLL_MAX_BATCH_SIZE_DISPLAY = "The maximum batch size";
private static final String POLL_MAX_BATCH_SIZE_DOC =
"Maximum number of change stream documents to include in a single batch when "
+ "polling for new data. This setting can be used to limit the amount of data buffered internally in the connector.";
private static final int POLL_MAX_BATCH_SIZE_DEFAULT = 1000;
public static final String POLL_AWAIT_TIME_MS_CONFIG = "poll.await.time.ms";
private static final String POLL_AWAIT_TIME_MS_DOC =
"The maximum amount of time in milliseconds the server waits for new data changes to report to the change stream cursor "
+ "before returning an empty batch.";
private static final long POLL_AWAIT_TIME_MS_DEFAULT = 5000;
private static final String POLL_AWAIT_TIME_MS_DISPLAY = "Poll await time (ms)";
public static final String DATABASE_CONFIG = "database";
private static final String DATABASE_DISPLAY = "The database to watch.";
private static final String DATABASE_DOC =
"The database to watch. If not set then all databases will be watched.";
private static final String DATABASE_DEFAULT = EMPTY_STRING;
public static final String COLLECTION_CONFIG = "collection";
private static final String COLLECTION_DISPLAY = "The collection to watch.";
private static final String COLLECTION_DOC =
"The collection in the database to watch. If not set then all collections will be "
+ "watched.";
private static final String COLLECTION_DEFAULT = EMPTY_STRING;
public static final String STARTUP_MODE_CONFIG = "startup.mode";
private static final String STARTUP_MODE_CONFIG_DISPLAY =
"The start up behavior when there is no source offset available.";
private static final String STARTUP_MODE_CONFIG_DOC =
format(
"Specifies how the connector should start up when there is no source offset available."
+ "\nResuming a change stream requires a resume token,"
+ " which the connector stores as / reads from the source offset."
+ " If no source offset is available, the connector may either ignore all/some existing source data,"
+ " or may at first copy all existing source data and then continue with processing new data."
+ " Possible values are %s."
+ "\n- 'latest' is the default value."
+ " The connector creates a new change stream, processes change events from it and stores resume tokens from them,"
+ " thus ignoring all existing source data."
+ "\n- 'timestamp' actuates 'startup.mode.timestamp.*' properties."
+ " If no such properties are configured, then 'timestamp' is equivalent to 'latest'."
+ "\n- 'copy_existing' actuates 'startup.mode.copy.existing.*' properties."
+ " The connector creates a new change stream and stores its resume token,"
+ " copies all existing data from all the collections being used as the source,"
+ " then processes new data starting from the stored resume token."
+ " It should be noted that the reading of all the data during the copy"
+ " and then the subsequent change stream events may produce duplicated events."
+ " During the copy, clients can make changes to the source data,"
+ " which may be represented both by the copying process and the change stream."
+ " However, as the change stream events are idempotent, it's possible to apply them multiple times"
+ " with the effect being the same as if they were applied once."
+ " Renaming a collection during the copying process is not supported."
+ "\nIt is an equivalent replacement for the deprecated 'copy.existing = true'.",
Stream.of(StartupMode.publicValues())
.map(StartupMode::propertyValue)
.map(v -> "'" + v + "'")
.collect(Collectors.joining(", ")));
static final StartupMode STARTUP_MODE_CONFIG_DEFAULT = StartupMode.DEFAULT_INTERNAL;
static final String STARTUP_MODE_TIMESTAMP_START_AT_OPERATION_TIME_CONFIG =
"startup.mode.timestamp.start.at.operation.time";
private static final String STARTUP_MODE_TIMESTAMP_START_AT_OPERATION_TIME_DISPLAY =
"The `startAtOperationTime` configuration.";
private static final String STARTUP_MODE_TIMESTAMP_START_AT_OPERATION_TIME_DOC =
"Actuated only if 'startup.mode = timestamp'."
+ " Specifies the starting point for the change stream. "
+ BsonTimestampParser.FORMAT_DESCRIPTION
+ " You may specify '0' to start at the beginning of the oplog."
+ " Requires MongoDB 4.0 or above."
+ " See https://www.mongodb.com/docs/current/reference/operator/aggregation/changeStream/.";
static final String STARTUP_MODE_TIMESTAMP_START_AT_OPERATION_TIME_DEFAULT = EMPTY_STRING;
static final String STARTUP_MODE_COPY_EXISTING_MAX_THREADS_CONFIG =
"startup.mode.copy.existing.max.threads";
private static final String STARTUP_MODE_COPY_EXISTING_MAX_THREADS_DISPLAY =
"Copy existing max number of threads";
private static final String STARTUP_MODE_COPY_EXISTING_MAX_THREADS_DOC =
"The number of threads to use when performing the data copy. "
+ "Defaults to the number of processors."
+ "\nIt is an equivalent replacement for the deprecated 'copy.existing.max.threads'.";
static final int STARTUP_MODE_COPY_EXISTING_MAX_THREADS_DEFAULT =
Runtime.getRuntime().availableProcessors();
static final String STARTUP_MODE_COPY_EXISTING_QUEUE_SIZE_CONFIG =
"startup.mode.copy.existing.queue.size";
private static final String STARTUP_MODE_COPY_EXISTING_QUEUE_SIZE_DISPLAY =
"Copy existing queue size";
private static final String STARTUP_MODE_COPY_EXISTING_QUEUE_SIZE_DOC =
"The max size of the queue to use when copying data."
+ "\nIt is an equivalent replacement for the deprecated 'copy.existing.queue.size'.";
static final int STARTUP_MODE_COPY_EXISTING_QUEUE_SIZE_DEFAULT = 16000;
static final String STARTUP_MODE_COPY_EXISTING_PIPELINE_CONFIG =
"startup.mode.copy.existing.pipeline";
private static final String STARTUP_MODE_COPY_EXISTING_PIPELINE_DISPLAY =
"Copy existing initial pipeline";
private static final String STARTUP_MODE_COPY_EXISTING_PIPELINE_DOC =
"An inline JSON array with objects describing the pipeline operations to run when copying existing data.\n"
+ "This can improve the use of indexes by the copying manager and make copying more efficient.\n"
+ "Use if there is any filtering of collection data in the `pipeline` configuration to speed up the copying process.\n"
+ "Example: `[{\"$match\": {\"closed\": \"false\"}}]`."
+ "\nIt is an equivalent replacement for the deprecated 'copy.existing.pipeline'.";
static final String STARTUP_MODE_COPY_EXISTING_PIPELINE_DEFAULT = EMPTY_STRING;
public static final String STARTUP_MODE_COPY_EXISTING_NAMESPACE_REGEX_CONFIG =
"startup.mode.copy.existing.namespace.regex";
private static final String STARTUP_MODE_COPY_EXISTING_NAMESPACE_REGEX_DISPLAY =
"Copy existing namespace regex";
private static final String STARTUP_MODE_COPY_EXISTING_NAMESPACE_REGEX_DOC =
"Use a regular expression to define from which existing namespaces data should be copied from."
+ " A namespace is the database name and collection separated by a period e.g. `database.collection`.\n"
+ " Example: The following regular expression will only include collections starting with `a` "
+ "in the `demo` database: `demo\\.a.*`."
+ "\nIt is an equivalent replacement for the deprecated 'copy.existing.namespace.regex'.";
static final String STARTUP_MODE_COPY_EXISTING_NAMESPACE_REGEX_DEFAULT = EMPTY_STRING;
static final String STARTUP_MODE_COPY_EXISTING_ALLOW_DISK_USE_CONFIG =
"startup.mode.copy.existing.allow.disk.use";
private static final String STARTUP_MODE_COPY_EXISTING_ALLOW_DISK_USE_DISPLAY =
"Copy existing allow disk use with the copying aggregation";
private static final String STARTUP_MODE_COPY_EXISTING_ALLOW_DISK_USE_DOC =
"Copy existing data uses an aggregation pipeline that mimics change stream events. In certain contexts this can require"
+ "writing to disk if the aggregation process runs out of memory."
+ "\nIt is an equivalent replacement for the deprecated 'copy.existing.allow.disk.use'.";
public static final boolean STARTUP_MODE_COPY_EXISTING_ALLOW_DISK_USE_DEFAULT = true;
static final String COPY_EXISTING_CONFIG = "copy.existing";
private static final String COPY_EXISTING_DISPLAY = "Copy existing data";
private static final String COPY_EXISTING_DOC =
"Deprecated, use 'startup.mode = copy_existing' instead; deprecated properties are overridden by normal ones if there is a conflict. "
+ "Copy existing data from all the collections being used as the source then add "
+ "any changes after. It should be noted that the reading of all the data during the copy and then the subsequent change "
+ "stream events may produce duplicated events. "
+ "During the copy, clients can make changes to the data in MongoDB, which may be "
+ "represented both by the copying process and the change stream. However, as the change stream events are idempotent the "
+ "changes can be applied so that the data is eventually consistent. Renaming a collection during the copying process is not "
+ "supported.";
static final boolean COPY_EXISTING_DEFAULT = false;
static final String COPY_EXISTING_MAX_THREADS_CONFIG = "copy.existing.max.threads";
private static final String COPY_EXISTING_MAX_THREADS_DISPLAY =
"Copy existing max number of threads";
private static final String COPY_EXISTING_MAX_THREADS_DOC =
"Deprecated, use 'startup.mode.copy.existing.max.threads' instead; "
+ "deprecated properties are overridden by normal ones if there is a conflict. "
+ "The number of threads to use when performing the data copy. "
+ "Defaults to the number of processors";
static final int COPY_EXISTING_MAX_THREADS_DEFAULT = Runtime.getRuntime().availableProcessors();
static final String COPY_EXISTING_QUEUE_SIZE_CONFIG = "copy.existing.queue.size";
private static final String COPY_EXISTING_QUEUE_SIZE_DISPLAY = "Copy existing queue size";
private static final String COPY_EXISTING_QUEUE_SIZE_DOC =
"Deprecated, use 'startup.mode.copy.existing.queue.size' instead; "
+ "deprecated properties are overridden by normal ones if there is a conflict. "
+ "The max size of the queue to use when copying data.";
static final int COPY_EXISTING_QUEUE_SIZE_DEFAULT = 16000;
static final String COPY_EXISTING_PIPELINE_CONFIG = "copy.existing.pipeline";
private static final String COPY_EXISTING_PIPELINE_DISPLAY = "Copy existing initial pipeline";
private static final String COPY_EXISTING_PIPELINE_DOC =
"Deprecated, use 'startup.mode.copy.existing.pipeline' instead; "
+ "deprecated properties are overridden by normal ones if there is a conflict. "
+ "An inline JSON array with objects describing the pipeline operations to run when copying existing data.\n"
+ "This can improve the use of indexes by the copying manager and make copying more efficient.\n"
+ "Use if there is any filtering of collection data in the `pipeline` configuration to speed up the copying process.\n"
+ "Example: `[{\"$match\": {\"closed\": \"false\"}}]`";
static final String COPY_EXISTING_PIPELINE_DEFAULT = EMPTY_STRING;
public static final String COPY_EXISTING_NAMESPACE_REGEX_CONFIG = "copy.existing.namespace.regex";
private static final String COPY_EXISTING_NAMESPACE_REGEX_DISPLAY =
"Copy existing namespace regex";
private static final String COPY_EXISTING_NAMESPACE_REGEX_DOC =
"Deprecated, use 'startup.mode.copy.existing.namespace.regex' instead; "
+ "deprecated properties are overridden by normal ones if there is a conflict. "
+ "Use a regular expression to define from which existing namespaces data should be copied from."
+ " A namespace is the database name and collection separated by a period e.g. `database.collection`.\n"
+ " Example: The following regular expression will only include collections starting with `a` "
+ "in the `demo` database: `demo\\.a.*`";
static final String COPY_EXISTING_NAMESPACE_REGEX_DEFAULT = EMPTY_STRING;
static final String COPY_EXISTING_ALLOW_DISK_USE_CONFIG = "copy.existing.allow.disk.use";
private static final String COPY_EXISTING_ALLOW_DISK_USE_DISPLAY =
"Copy existing allow disk use with the copying aggregation";
private static final String COPY_EXISTING_ALLOW_DISK_USE_DOC =
"Deprecated, use 'startup.mode.copy.existing.allow.disk.use' instead; "
+ "deprecated properties are overridden by normal ones if there is a conflict. "
+ "Copy existing data uses an aggregation pipeline that mimics change stream events. In certain contexts this can require"
+ "writing to disk if the aggregation process runs out of memory.";
public static final boolean COPY_EXISTING_ALLOW_DISK_USE_DEFAULT = true;
public static final String ERRORS_TOLERANCE_CONFIG = "errors.tolerance";
public static final String ERRORS_TOLERANCE_DISPLAY = "Error Tolerance";
public static final ErrorTolerance ERRORS_TOLERANCE_DEFAULT = ErrorTolerance.NONE;
public static final String ERRORS_TOLERANCE_DOC =
"Behavior for tolerating errors during connector operation. 'none' is the default value "
+ "and signals that any error will result in an immediate connector task failure; 'all' "
+ "changes the behavior to skip over problematic records.";
public static final String OVERRIDE_ERRORS_TOLERANCE_CONFIG = "mongo.errors.tolerance";
public static final String OVERRIDE_ERRORS_TOLERANCE_DOC =
"Use this property if you would like to configure the connector's error handling behavior differently from the Connect framework's.";
public static final String ERRORS_LOG_ENABLE_CONFIG = "errors.log.enable";
public static final String ERRORS_LOG_ENABLE_DISPLAY = "Log Errors";
public static final boolean ERRORS_LOG_ENABLE_DEFAULT = false;
public static final String ERRORS_LOG_ENABLE_DOC =
"If true, write each error and the details of the failed operation and problematic record "
+ "to the Connect application log. This is 'false' by default, so that only errors that are not tolerated are reported.";
public static final String OVERRIDE_ERRORS_LOG_ENABLE_CONFIG = "mongo.errors.log.enable";
public static final String OVERRIDE_ERRORS_LOG_ENABLE_DOC =
"Use this property if you would like to configure the connector's error handling behavior differently from the Connect framework's.";
public static final String ERRORS_DEAD_LETTER_QUEUE_TOPIC_NAME_CONFIG =
"errors.deadletterqueue.topic.name";
public static final String ERRORS_DEAD_LETTER_QUEUE_TOPIC_NAME_DISPLAY =
"Output errors to the dead letter queue";
public static final String ERRORS_DEAD_LETTER_QUEUE_TOPIC_NAME_DEFAULT = EMPTY_STRING;
public static final String ERRORS_DEAD_LETTER_QUEUE_TOPIC_NAME_DOC =
"Whether to output conversion errors to the dead letter queue. "
+ "Stops poison messages when using schemas, any message will be outputted as extended json on the specified topic. "
+ "By default messages are not outputted to the dead letter queue. "
+ "Also requires `errors.tolerance=all`.";
public static final String OVERRIDE_ERRORS_DEAD_LETTER_QUEUE_TOPIC_NAME_CONFIG =
"mongo.errors.deadletterqueue.topic.name";
public static final String LEGACY_ERRORS_DEAD_LETTER_QUEUE_TOPIC_NAME_DOC =
"Use this property if you would like to configure the connector's error handling behavior differently from the Connect framework's.";
public static final String HEARTBEAT_INTERVAL_MS_CONFIG = "heartbeat.interval.ms";
private static final String HEARTBEAT_INTERVAL_MS_DISPLAY = "Heartbeat interval";
private static final String HEARTBEAT_INTERVAL_MS_DOC =
"The length of time between sending heartbeat messages to record the post batch resume token"
+ " when no source records have been published. Improves the resumability of the connector"
+ " for low volume namespaces. Use 0 to disable.";
private static final int HEARTBEAT_INTERVAL_MS_DEFAULT = 0;
public static final String HEARTBEAT_TOPIC_NAME_CONFIG = "heartbeat.topic.name";
private static final String HEARTBEAT_TOPIC_NAME_DISPLAY = "The heartbeat topic name";
private static final String HEARTBEAT_TOPIC_NAME_DEFAULT = "__mongodb_heartbeats";
private static final String HEARTBEAT_TOPIC_NAME_DOC =
"The name of the topic to publish heartbeats to."
+ " Defaults to '"
+ HEARTBEAT_TOPIC_NAME_DEFAULT
+ "'.";
public static final String OFFSET_PARTITION_NAME_CONFIG = "offset.partition.name";
public static final String OFFSET_PARTITION_NAME_DISPLAY = "Offset partition name";
public static final String OFFSET_PARTITION_NAME_DEFAULT = EMPTY_STRING;
public static final String OFFSET_PARTITION_NAME_DOC =
"Use a custom offset partition name. If blank the default partition name based on the "
+ "connection details will be used.";
static final String PROVIDER_CONFIG = "provider";
public static final ConfigDef CONFIG = createConfigDef();
private static final List<Consumer<MongoSourceConfig>> INITIALIZERS =
asList(MongoSourceConfig::validateCollection, MongoSourceConfig::getTopicMapper);
@VisibleForTesting(otherwise = PACKAGE)
@Immutable
public static final class StartupConfig {
private final StartupMode startupMode;
@Nullable private final TimestampConfig timestampConfig;
@Nullable private final CopyExistingConfig copyExistingConfig;
private StartupConfig(
final StartupMode startupMode,
@Nullable final TimestampConfig timestampConfig,
@Nullable final CopyExistingConfig copyExistingConfig) {
assertFalse(startupMode == StartupMode.DEFAULT_INTERNAL);
this.startupMode = startupMode;
this.timestampConfig = timestampConfig;
this.copyExistingConfig = copyExistingConfig;
}
static StartupConfig latest() {
return new StartupConfig(StartupMode.LATEST, null, null);
}
static StartupConfig timestamp(final TimestampConfig cfg) {
return new StartupConfig(StartupMode.TIMESTAMP, cfg, null);
}
static StartupConfig copyExisting(final CopyExistingConfig cfg) {
return new StartupConfig(StartupMode.COPY_EXISTING, null, cfg);
}
/** Never returns {@link StartupMode#DEFAULT_INTERNAL}. */
StartupMode startupMode() {
return startupMode;
}
TimestampConfig timestampConfig() {
return assertNotNull(timestampConfig);
}
CopyExistingConfig copyExistingConfig() {
return assertNotNull(copyExistingConfig);
}
@VisibleForTesting(otherwise = PACKAGE)
@Immutable
public enum StartupMode {
/**
* Is equivalent to {@link #LATEST}, and is used to discriminate a situation when the default
* behavior is configured explicitly and, therefore, overrides {@link #COPY_EXISTING_CONFIG}.
*/
DEFAULT_INTERNAL,
/** @see #DEFAULT_INTERNAL */
LATEST,
TIMESTAMP,
COPY_EXISTING;
/** @see #parse(String) */
@VisibleForTesting(otherwise = PACKAGE)
public String propertyValue() {
return this == DEFAULT_INTERNAL ? EMPTY_STRING : name().toLowerCase(Locale.ROOT);
}
/** @see #propertyValue() */
private static StartupMode parse(final String propertyValue) {
return propertyValue.equals(EMPTY_STRING)
? DEFAULT_INTERNAL
: StartupMode.valueOf(propertyValue.toUpperCase());
}
private static StartupMode[] publicValues() {
return Stream.of(StartupMode.values())
.filter(v -> v != DEFAULT_INTERNAL)
.toArray(StartupMode[]::new);
}
}
@Immutable
static final class TimestampConfig {
@Nullable private final BsonTimestamp startAtOperationTime;
private TimestampConfig(@Nullable final BsonTimestamp startAtOperationTime) {
this.startAtOperationTime = startAtOperationTime;
}
Optional<BsonTimestamp> startAtOperationTime() {
return Optional.ofNullable(startAtOperationTime);
}
}
/** {@link Immutable} provided that we don't mutate {@link Document} in {@link #pipeline()}. */
@Immutable
static final class CopyExistingConfig {
private final int maxThreads;
private final int queueSize;
@Nullable private final List<Document> pipeline;
private final String namespaceRegex;
private final boolean allowDiskUse;
private CopyExistingConfig(
final int maxThreads,
final int queueSize,
@Nullable final List<Document> pipeline,
final String namespaceRegex,
final boolean allowDiskUse) {
this.maxThreads = maxThreads;
this.queueSize = queueSize;
this.pipeline = pipeline == null ? null : unmodifiableList(new ArrayList<>(pipeline));
this.namespaceRegex = namespaceRegex;
this.allowDiskUse = allowDiskUse;
}
int maxThreads() {
return maxThreads;
}
int queueSize() {
return queueSize;
}
Optional<List<Document>> pipeline() {
return Optional.ofNullable(pipeline);
}
String namespaceRegex() {
return namespaceRegex;
}
boolean allowDiskUse() {
return allowDiskUse;
}
}
}
public enum OutputFormat {
JSON,
BSON,
SCHEMA,
BINARY_OUTBOX
}
public enum ErrorTolerance {
/** Tolerate no errors. */
NONE,
/** Tolerate all errors. */
ALL;
public String value() {
return name().toLowerCase(Locale.ROOT);
}
}
private final ConnectionString connectionString;
private TopicMapper topicMapper;
@Nullable private StartupConfig startupConfig;
public MongoSourceConfig(final Map<?, ?> originals) {
this(originals, true);
}
private MongoSourceConfig(final Map<?, ?> originals, final boolean validateAll) {
super(CONFIG, originals, false);
connectionString = new ConnectionString(getPassword(CONNECTION_URI_CONFIG).value());
if (validateAll) {
INITIALIZERS.forEach(i -> i.accept(this));
}
}
public ConnectionString getConnectionString() {
return connectionString;
}
public OutputFormat getKeyOutputFormat() {
return OutputFormat.valueOf(getString(OUTPUT_FORMAT_KEY_CONFIG).toUpperCase());
}
public OutputFormat getValueOutputFormat() {
return OutputFormat.valueOf(getString(OUTPUT_FORMAT_VALUE_CONFIG).toUpperCase());
}
Optional<List<Document>> getPipeline() {
return getPipeline(PIPELINE_CONFIG);
}
private Optional<List<Document>> getPipeline(final String configName) {
return jsonArrayFromString(getString(configName));
}
Optional<Collation> getCollation() {
return collationFromJson(getString(COLLATION_CONFIG));
}
Optional<FullDocumentBeforeChange> getFullDocumentBeforeChange() {
return fullDocumentBeforeChangeFromString(getString(FULL_DOCUMENT_BEFORE_CHANGE_CONFIG));
}
Optional<FullDocument> getFullDocument() {
if (getBoolean(PUBLISH_FULL_DOCUMENT_ONLY_CONFIG)) {
return Optional.of(FullDocument.UPDATE_LOOKUP);
} else {
return fullDocumentFromString(getString(FULL_DOCUMENT_CONFIG));
}
}
StartupConfig getStartupConfig() {
StartupConfig result = startupConfig;
if (result != null) {
return result;
}
StartupMode startupMode = StartupMode.parse(getString(STARTUP_MODE_CONFIG));
if (startupMode == STARTUP_MODE_CONFIG_DEFAULT) {
StartupMode defaultBehavior = StartupMode.LATEST;
startupMode = getBoolean(COPY_EXISTING_CONFIG) ? StartupMode.COPY_EXISTING : defaultBehavior;
}
switch (startupMode) {
case LATEST:
result = StartupConfig.latest();
break;
case TIMESTAMP:
final String startAtOperationTime =
getString(STARTUP_MODE_TIMESTAMP_START_AT_OPERATION_TIME_CONFIG);
result =
StartupConfig.timestamp(
new StartupConfig.TimestampConfig(
startAtOperationTime.isEmpty()
? null
: BsonTimestampParser.parse(
STARTUP_MODE_TIMESTAMP_START_AT_OPERATION_TIME_CONFIG,
startAtOperationTime,
null)));
break;
case COPY_EXISTING:
result =
StartupConfig.copyExisting(
new StartupConfig.CopyExistingConfig(
ConfigHelper.getOverrideOrFallback(
this,
AbstractConfig::getInt,
STARTUP_MODE_COPY_EXISTING_MAX_THREADS_CONFIG,
COPY_EXISTING_MAX_THREADS_CONFIG),
ConfigHelper.getOverrideOrFallback(
this,
AbstractConfig::getInt,
STARTUP_MODE_COPY_EXISTING_QUEUE_SIZE_CONFIG,
COPY_EXISTING_QUEUE_SIZE_CONFIG),
ConfigHelper.getOverrideOrFallback(
this,
MongoSourceConfig::getPipeline,
STARTUP_MODE_COPY_EXISTING_PIPELINE_CONFIG,
COPY_EXISTING_PIPELINE_CONFIG)
.orElse(null),
ConfigHelper.getOverrideOrFallback(
this,
AbstractConfig::getString,
STARTUP_MODE_COPY_EXISTING_NAMESPACE_REGEX_CONFIG,
COPY_EXISTING_NAMESPACE_REGEX_CONFIG),
ConfigHelper.getOverrideOrFallback(
this,
AbstractConfig::getBoolean,
STARTUP_MODE_COPY_EXISTING_ALLOW_DISK_USE_CONFIG,
COPY_EXISTING_ALLOW_DISK_USE_CONFIG)));
break;
case DEFAULT_INTERNAL:
default:
throw fail();
}
startupConfig = assertNotNull(result);
return result;
}
private void validateCollection() {
String database = getString(DATABASE_CONFIG);
String collection = getString(COLLECTION_CONFIG);
if (!collection.isEmpty() && database.isEmpty()) {
throw new ConnectConfigException(
COLLECTION_CONFIG,
collection,
format("Missing database configuration `%s`", DATABASE_CONFIG));
}
}
public TopicMapper getTopicMapper() {
if (topicMapper == null) {
topicMapper =
configureInstance(
createInstance(
TOPIC_MAPPER_CONFIG, getString(TOPIC_MAPPER_CONFIG), TopicMapper.class));
}
return topicMapper;
}
public JsonWriterSettings getJsonWriterSettings() {
return createInstance(
OUTPUT_JSON_FORMATTER_CONFIG,
getString(OUTPUT_JSON_FORMATTER_CONFIG),
JsonWriterSettingsProvider.class)
.getJsonWriterSettings();
}
public boolean tolerateErrors() {
String errorsTolerance =
ConfigHelper.getOverrideOrFallback(
this,
AbstractConfig::getString,
OVERRIDE_ERRORS_TOLERANCE_CONFIG,
ERRORS_TOLERANCE_CONFIG);
return ErrorTolerance.valueOf(errorsTolerance.toUpperCase()).equals(ErrorTolerance.ALL);
}
public boolean logErrors() {
return !tolerateErrors()
|| ConfigHelper.getOverrideOrFallback(
this,
AbstractConfig::getBoolean,
OVERRIDE_ERRORS_LOG_ENABLE_CONFIG,
ERRORS_LOG_ENABLE_CONFIG);
}
public String getDlqTopic() {
return ConfigHelper.getOverrideOrFallback(
this,
AbstractConfig::getString,
OVERRIDE_ERRORS_DEAD_LETTER_QUEUE_TOPIC_NAME_CONFIG,
ERRORS_DEAD_LETTER_QUEUE_TOPIC_NAME_CONFIG);
}
private <T extends Configurable> T configureInstance(final T instance) {
instance.configure(this);
return instance;
}
private static ConfigDef createConfigDef() {
ConfigDef configDef =
new ConfigDef() {
@Override
public Map<String, ConfigValue> validateAll(final Map<String, String> props) {
logObsoleteProperties(props.keySet(), LOGGER::warn);
logIncompatibleProperties(props, LOGGER::warn);
Map<String, ConfigValue> results = super.validateAll(props);
// Don't validate child configs if the top level configs are broken
if (results.values().stream().anyMatch((c) -> !c.errorMessages().isEmpty())) {
return results;
}
// Validate database & collection
MongoSourceConfig cfg = new MongoSourceConfig(props, false);
INITIALIZERS.forEach(
i -> {
try {
i.accept(cfg);
} catch (ConnectConfigException t) {
results.put(
t.getName(),
new ConfigValue(
t.getName(), t.getValue(), emptyList(), singletonList(t.getMessage())));
}
});
return results;
}
};
String group = "Connection";
int orderInGroup = 0;
configDef.define(
CONNECTION_URI_CONFIG,
Type.PASSWORD,
CONNECTION_URI_DEFAULT,
errorCheckingPasswordValueValidator("A valid connection string", ConnectionString::new),
Importance.HIGH,
CONNECTION_URI_DOC,
group,
++orderInGroup,
Width.MEDIUM,
CONNECTION_URI_DISPLAY);
configDef.define(
DATABASE_CONFIG,
Type.STRING,
DATABASE_DEFAULT,
Importance.MEDIUM,
DATABASE_DOC,
group,
++orderInGroup,
Width.MEDIUM,
DATABASE_DISPLAY);
configDef.define(
COLLECTION_CONFIG,
Type.STRING,
COLLECTION_DEFAULT,
Importance.MEDIUM,
COLLECTION_DOC,
group,
++orderInGroup,
Width.MEDIUM,
COLLECTION_DISPLAY);
addServerApiConfig(configDef);
addSslConfigDef(configDef);
group = "Change stream";
orderInGroup = 0;
configDef.define(
PIPELINE_CONFIG,
Type.STRING,
PIPELINE_DEFAULT,
errorCheckingValueValidator("A valid JSON array", ConfigHelper::jsonArrayFromString),
Importance.MEDIUM,
PIPELINE_DOC,
group,
++orderInGroup,
Width.MEDIUM,
PIPELINE_DISPLAY);