Skip to content

Commit 9ea03b5

Browse files
lovromazgonlovromazgon
authored
Upgrade builtin connectors, SDK and commons (#1515)
* upgrade builtin connectors * rename deprecated parameter * fix metadata constants test * make generate * clone config map when sending it to builtin plugin
1 parent 3993bff commit 9ea03b5

13 files changed

Lines changed: 40 additions & 59 deletions

File tree

‎.golangci.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
run:
22
timeout: 3m
3-
skip-dirs-use-default: false
4-
skip-dirs:
3+
exclude-dirs-use-default: false
4+
exclude-dirs:
55
- ^examples/
66
- ^ui/
77
- ^pkg/plugin/processor/builtin/internal/diff # external code

‎go.mod‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,15 @@ require (
88
github.com/Masterminds/sprig/v3 v3.2.3
99
github.com/NYTimes/gziphandler v1.1.1
1010
github.com/bufbuild/buf v1.31.0
11-
github.com/conduitio/conduit-commons v0.1.1
11+
github.com/conduitio/conduit-commons v0.2.0
1212
github.com/conduitio/conduit-connector-file v0.6.0
13-
github.com/conduitio/conduit-connector-generator v0.5.0
14-
github.com/conduitio/conduit-connector-kafka v0.7.2
13+
github.com/conduitio/conduit-connector-generator v0.6.0
14+
github.com/conduitio/conduit-connector-kafka v0.8.0
1515
github.com/conduitio/conduit-connector-log v0.3.0
16-
github.com/conduitio/conduit-connector-postgres v0.6.0
17-
github.com/conduitio/conduit-connector-protocol v0.5.1-0.20240104160905-e9e61586fb8d
16+
github.com/conduitio/conduit-connector-postgres v0.7.0
17+
github.com/conduitio/conduit-connector-protocol v0.6.0
1818
github.com/conduitio/conduit-connector-s3 v0.5.1
19-
github.com/conduitio/conduit-connector-sdk v0.8.0
19+
github.com/conduitio/conduit-connector-sdk v0.9.1
2020
github.com/conduitio/conduit-processor-sdk v0.1.1
2121
github.com/conduitio/yaml/v3 v3.3.0
2222
github.com/dgraph-io/badger/v4 v4.2.0
@@ -124,7 +124,7 @@ require (
124124
github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
125125
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
126126
github.com/curioswitch/go-reassign v0.2.0 // indirect
127-
github.com/daixiang0/gci v0.12.3 // indirect
127+
github.com/daixiang0/gci v0.13.4 // indirect
128128
github.com/davecgh/go-spew v1.1.1 // indirect
129129
github.com/denis-tingaikin/go-header v0.5.0 // indirect
130130
github.com/dgraph-io/ristretto v0.1.1 // indirect

‎go.sum‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -327,24 +327,24 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
327327
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
328328
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
329329
github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c=
330-
github.com/conduitio/conduit-commons v0.1.1 h1:ACgRexA85QIme5NE5AzrsO48RcqIBNSmwdBjWgXjxbg=
331-
github.com/conduitio/conduit-commons v0.1.1/go.mod h1:ejihG+J5Q6V3pucBPKlsdw3Kfzw0ggONn1SIm1RLzTc=
330+
github.com/conduitio/conduit-commons v0.2.0 h1:TMpVGXi0Wski537qLAyQWdGjuGHEhaZxOS5L90pZJSQ=
331+
github.com/conduitio/conduit-commons v0.2.0/go.mod h1:i7Q2jm7FBSi2zj1/4MCsFD1hIKAbvamlNtSQfkhUTiY=
332332
github.com/conduitio/conduit-connector-file v0.6.0 h1:8tsGeGhKvFwYQZztOOL5/tmOhVShsfo9lQ3b/0fX8kQ=
333333
github.com/conduitio/conduit-connector-file v0.6.0/go.mod h1:ju7PiB4kTJgqng4KVXDt/Gvw/53kFwSzi5Ez9EDXxNI=
334-
github.com/conduitio/conduit-connector-generator v0.5.0 h1:zpXHif89DCJ13nftKLv31uI2AJGicpY5H1V7SwldRNo=
335-
github.com/conduitio/conduit-connector-generator v0.5.0/go.mod h1:CQKGYQNAnO6+bJSurzwtmGf98XoyeU6zsMJI/OCrlJI=
336-
github.com/conduitio/conduit-connector-kafka v0.7.2 h1:2kiCYI6CbuSGdNGBbhtjPB4Zv5PjtFduf35EL6vv0gY=
337-
github.com/conduitio/conduit-connector-kafka v0.7.2/go.mod h1:dpkglwPd42tdgUJpYEFWM2WOLCwFDP9HC5EPqMuiVUU=
334+
github.com/conduitio/conduit-connector-generator v0.6.0 h1:GjeR3v+mVGqXfGXs7zJMOHJFsX/SUpY7x4h1UhxpMzk=
335+
github.com/conduitio/conduit-connector-generator v0.6.0/go.mod h1:VYJjGhPh3N8ka5tr7a/oPyGhfrVGxr0MjCS+ccfteDM=
336+
github.com/conduitio/conduit-connector-kafka v0.8.0 h1:NsiihUjhzl+PWrxTWrbtCEV6UDfvLiEEuv7BaABr8zc=
337+
github.com/conduitio/conduit-connector-kafka v0.8.0/go.mod h1:1fkIxtjojlTKNQ0RpMNPUkz2h1VNVtZHOyIsZVT1U2I=
338338
github.com/conduitio/conduit-connector-log v0.3.0 h1:J9CD/y86D/3i/C1MI0lrTiFnLwwnpJKQJu3M+jDiI4Y=
339339
github.com/conduitio/conduit-connector-log v0.3.0/go.mod h1:R0gHB21mw/9BXYAKfx9IKQLEsPuB05zJPJXB6JFxYGs=
340-
github.com/conduitio/conduit-connector-postgres v0.6.0 h1:D0YTZXkxm5FG/UvL80MOP59RqozaFXKwr7sGII6fmn4=
341-
github.com/conduitio/conduit-connector-postgres v0.6.0/go.mod h1:KASWjEASlHhMPV6xQCKMzP4qH2JRGarAto5AeLnBiCk=
342-
github.com/conduitio/conduit-connector-protocol v0.5.1-0.20240104160905-e9e61586fb8d h1:XkJkS63Qkf2vuIYZnWeQMOKsMxBCvqlhX+T+aDqh0kc=
343-
github.com/conduitio/conduit-connector-protocol v0.5.1-0.20240104160905-e9e61586fb8d/go.mod h1:zW3my/7U6dBCBwXZt4EdtmLHBUhhX/42fXMu/+4eRHU=
340+
github.com/conduitio/conduit-connector-postgres v0.7.0 h1:At6O5e4eFegESTgzxZeY1eh0i+1FO5L4R35PZ9s7te0=
341+
github.com/conduitio/conduit-connector-postgres v0.7.0/go.mod h1:d39rdzjiCFUaBbFsg+Lh0KorMIY6KbQDu8HrDu/5Zvs=
342+
github.com/conduitio/conduit-connector-protocol v0.6.0 h1:2gMOCOpa+c97CHIpZv7Niu3V4o5UgRr6fzj9kzfRV7o=
343+
github.com/conduitio/conduit-connector-protocol v0.6.0/go.mod h1:3mo59xYX9etFoR3n82R7J50La1iWK+Vm63H8z2wo4QM=
344344
github.com/conduitio/conduit-connector-s3 v0.5.1 h1:yRo8004ryCIZc/S3iWQ1rN6pm6bjySlXFCGZOl1rE1E=
345345
github.com/conduitio/conduit-connector-s3 v0.5.1/go.mod h1:nbxzsyS95gbFJ28Job9vFFB+byRFINSv70/13Yi4mKQ=
346-
github.com/conduitio/conduit-connector-sdk v0.8.0 h1:gvchqoj5d3AQsBoIosx4i32L8Ex9+5BuAyHi/IM9VD4=
347-
github.com/conduitio/conduit-connector-sdk v0.8.0/go.mod h1:nOz4K3X6fD8YMe5CPbULwSEE18Eu02ZrpT6o6KwQfxs=
346+
github.com/conduitio/conduit-connector-sdk v0.9.1 h1:DiMUn7udnjWvyaDsyeTZFHeYTEIdqUU6dqPunEEE3Kw=
347+
github.com/conduitio/conduit-connector-sdk v0.9.1/go.mod h1:cNoofumgDlsaThkxkNYg7zab4AkmRZt1V711aO7guGU=
348348
github.com/conduitio/conduit-processor-sdk v0.1.1 h1:C+5Z9pGKVTpdIf5QFNx4UxpvxuOylGRVkGidEpom7HQ=
349349
github.com/conduitio/conduit-processor-sdk v0.1.1/go.mod h1:StkbqQX1WxTjr9LOy7zY+e3DAbEDVvozeamELdzFqck=
350350
github.com/conduitio/yaml/v3 v3.3.0 h1:kbbaOSHcuH39gP4+rgbJGl6DSbLZcJgEaBvkEXJlCsI=
@@ -366,8 +366,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
366366
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
367367
github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo=
368368
github.com/curioswitch/go-reassign v0.2.0/go.mod h1:x6OpXuWvgfQaMGks2BZybTngWjT84hqJfKoO8Tt/Roc=
369-
github.com/daixiang0/gci v0.12.3 h1:yOZI7VAxAGPQmkb1eqt5g/11SUlwoat1fSblGLmdiQc=
370-
github.com/daixiang0/gci v0.12.3/go.mod h1:xtHP9N7AHdNvtRNfcx9gwTDfw7FRJx4bZUsiEfiNNAI=
369+
github.com/daixiang0/gci v0.13.4 h1:61UGkmpoAcxHM2hhNkZEf5SzwQtWJXTSws7jaPyqwlw=
370+
github.com/daixiang0/gci v0.13.4/go.mod h1:12etP2OniiIdP4q+kjUGrC/rUagga7ODbqsom5Eo5Yk=
371371
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
372372
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
373373
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -561,7 +561,6 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
561561
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
562562
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
563563
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
564-
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
565564
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
566565
github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
567566
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

‎pkg/plugin/connector/builtin/v1/internal/toplugin/destination.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
package toplugin
1616

1717
import (
18+
"maps"
19+
1820
"github.com/conduitio/conduit-connector-protocol/cpluginv1"
1921
"github.com/conduitio/conduit/pkg/record"
2022
)
@@ -23,7 +25,7 @@ func DestinationConfigureRequest(in map[string]string) cpluginv1.DestinationConf
2325
out := cpluginv1.DestinationConfigureRequest{}
2426
if len(in) > 0 {
2527
// gRPC sends `nil` if the map is empty, match behavior
26-
out.Config = in
28+
out.Config = maps.Clone(in)
2729
}
2830
return out
2931
}

‎pkg/plugin/connector/builtin/v1/internal/toplugin/source.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
package toplugin
1616

1717
import (
18+
"maps"
19+
1820
"github.com/conduitio/conduit-connector-protocol/cpluginv1"
1921
"github.com/conduitio/conduit/pkg/record"
2022
)
@@ -23,7 +25,7 @@ func SourceConfigureRequest(in map[string]string) cpluginv1.SourceConfigureReque
2325
out := cpluginv1.SourceConfigureRequest{}
2426
if len(in) > 0 {
2527
// gRPC sends `nil` if the map is empty, match behavior
26-
out.Config = in
28+
out.Config = maps.Clone(in)
2729
}
2830
return out
2931
}

‎pkg/plugin/connector/standalone/v1/metadata_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ package standalonev1
1717
import (
1818
"testing"
1919

20+
metadatav1 "github.com/conduitio/conduit-commons/proto/metadata/v1"
2021
opencdcv1 "github.com/conduitio/conduit-commons/proto/opencdc/v1"
21-
connectorv1 "github.com/conduitio/conduit-connector-protocol/proto/connector/v1"
2222
"github.com/conduitio/conduit/pkg/record"
2323
"google.golang.org/protobuf/proto"
2424
"google.golang.org/protobuf/runtime/protoimpl"
@@ -30,10 +30,10 @@ func TestMetadataConstants(t *testing.T) {
3030
record.MetadataOpenCDCVersion: opencdcv1.E_MetadataVersion,
3131
record.MetadataCreatedAt: opencdcv1.E_MetadataCreatedAt,
3232
record.MetadataReadAt: opencdcv1.E_MetadataReadAt,
33-
record.MetadataConduitSourcePluginName: connectorv1.E_MetadataConduitSourcePluginName,
34-
record.MetadataConduitSourcePluginVersion: connectorv1.E_MetadataConduitSourcePluginVersion,
35-
record.MetadataConduitDestinationPluginName: connectorv1.E_MetadataConduitDestinationPluginName,
36-
record.MetadataConduitDestinationPluginVersion: connectorv1.E_MetadataConduitDestinationPluginVersion,
33+
record.MetadataConduitSourcePluginName: metadatav1.E_MetadataConduitSourcePluginName,
34+
record.MetadataConduitSourcePluginVersion: metadatav1.E_MetadataConduitSourcePluginVersion,
35+
record.MetadataConduitDestinationPluginName: metadatav1.E_MetadataConduitDestinationPluginName,
36+
record.MetadataConduitDestinationPluginVersion: metadatav1.E_MetadataConduitDestinationPluginVersion,
3737
}
3838
for goConstant, extensionInfo := range wantMapping {
3939
protoConstant := proto.GetExtension(extensionInfo.TypeDescriptor().ParentFile().Options(), extensionInfo)

‎pkg/plugin/processor/builtin/internal/exampleutil/example.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,9 @@ import (
2020
"fmt"
2121
"log"
2222

23-
"github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal"
24-
2523
"github.com/conduitio/conduit-commons/opencdc"
2624
sdk "github.com/conduitio/conduit-processor-sdk"
25+
"github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal"
2726
"github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal/diff"
2827
"github.com/goccy/go-json"
2928
"github.com/google/go-cmp/cmp"

‎pkg/plugin/processor/builtin/internal/exampleutil/export.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,8 @@ import (
2323
"sort"
2424
"strings"
2525

26-
"github.com/conduitio/conduit/pkg/foundation/cerrors"
27-
2826
"github.com/conduitio/conduit-commons/opencdc"
27+
"github.com/conduitio/conduit/pkg/foundation/cerrors"
2928
"github.com/goccy/go-json"
3029
)
3130

‎pkg/plugin/processor/builtin/internal/exampleutil/specs/avro.decode.json‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,7 @@
6060
"summary": "Decode a record field in Avro format",
6161
"description": "This example shows the usage of the `avro.decode` processor.\nThe processor decodes the record's`.Key` field using the schema that is\ndownloaded from the schema registry and needs to exist under the subject`example-decode`.\nIn this example we use the following schema:\n\n```json\n{\n \"type\":\"record\",\n \"name\":\"record\",\n \"fields\":[\n {\"name\":\"myString\",\"type\":\"string\"},\n {\"name\":\"myInt\",\"type\":\"int\"}\n ]\n}\n```",
6262
"config": {
63-
"auth.basic.password": "",
64-
"auth.basic.username": "",
6563
"field": ".Key",
66-
"tls.ca.cert": "",
67-
"tls.client.cert": "",
68-
"tls.client.key": "",
6964
"url": "http://127.0.0.1:54322"
7065
},
7166
"have": {

‎pkg/plugin/processor/builtin/internal/exampleutil/specs/avro.encode.json‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -98,16 +98,9 @@
9898
"summary": "Auto-register schema",
9999
"description": "This example shows the usage of the `avro.encode` processor\nwith the `autoRegister` schema strategy. The processor encodes the record's\n`.Payload.After` field using the schema that is extracted from the data\nand registered on the fly under the subject `example-autoRegister`.",
100100
"config": {
101-
"auth.basic.password": "",
102-
"auth.basic.username": "",
103101
"field": ".Payload.After",
104102
"schema.autoRegister.subject": "example-autoRegister",
105-
"schema.preRegistered.subject": "",
106-
"schema.preRegistered.version": "",
107103
"schema.strategy": "autoRegister",
108-
"tls.ca.cert": "",
109-
"tls.client.cert": "",
110-
"tls.client.key": "",
111104
"url": "http://127.0.0.1:54322"
112105
},
113106
"have": {
@@ -151,16 +144,10 @@
151144
"summary": "Pre-register schema",
152145
"description": "This example shows the usage of the `avro.encode` processor\nwith the `preRegistered` schema strategy. When using this strategy, the\nschema has to be manually pre-registered. In this example we use the following schema:\n\n```json\n{\n \"type\":\"record\",\n \"name\":\"record\",\n \"fields\":[\n {\"name\":\"myString\",\"type\":\"string\"},\n {\"name\":\"myInt\",\"type\":\"int\"}\n ]\n}\n```\n\nThe processor encodes the record's`.Key` field using the above schema.",
153146
"config": {
154-
"auth.basic.password": "",
155-
"auth.basic.username": "",
156147
"field": ".Key",
157-
"schema.autoRegister.subject": "",
158148
"schema.preRegistered.subject": "example-preRegistered",
159149
"schema.preRegistered.version": "1",
160150
"schema.strategy": "preRegistered",
161-
"tls.ca.cert": "",
162-
"tls.client.cert": "",
163-
"tls.client.key": "",
164151
"url": "http://127.0.0.1:54322"
165152
},
166153
"have": {

0 commit comments

Comments
 (0)