Skip to content

Commit 7f97c97

Browse files
alarbadaGuillemguillemus
authored
housekeeping (#19)
* Update linter * Fix linter errors * Track tools in tools.go * Improve github workflow scripts * Run paramgen * Fix missing renames * Fix gci linter error * Fix failing test github workflow * Add install-tools script * Remove unnecessary workflow * Add missing docs to tls.enabled * Make delivery.contentType default to application/json * Move shared queue config into Config struct * Join docker tls and non tls rabbitmq services * Add pipeline.yml config example * Remove unused sdk.Position data * Add record structure docs --------- Co-authored-by: Guillem <[email protected]> Co-authored-by: Guillem <[email protected]>
1 parent 5f9e24e commit 7f97c97

19 files changed

+418
-279
lines changed

‎.github/pull_request_template.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,6 @@ Fixes # (issue)
88

99
### Quick checks:
1010

11-
- [ ] There is no other [pull request](https://github.com/alarbada/conduit-connector-rabbitmq/pulls) for the same update/change.
11+
- [ ] There is no other [pull request](https://github.com/conduitio-labs/conduit-connector-rabbitmq/pulls) for the same update/change.
1212
- [ ] I have written unit tests.
13-
- [ ] I have made sure that the PR is of reasonable size and can be easily reviewed.
13+
- [ ] I have made sure that the PR is of reasonable size and can be easily reviewed.

‎.github/workflows/validate-generated-files.yml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: validate-generated-files
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
8+
jobs:
9+
validate-generated-files:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: actions/checkout@v4
13+
14+
- name: Set up Go
15+
uses: actions/setup-go@v5
16+
with:
17+
go-version-file: 'go.mod'
18+
19+
- name: Check generated files
20+
run: |
21+
export PATH=$PATH:$(go env GOPATH)/bin
22+
make install-tools generate
23+
git diff
24+
git diff --exit-code --numstat

‎.golangci.yml

Lines changed: 58 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1+
run:
2+
timeout: 5m
3+
14
linters-settings:
2-
gofmt:
3-
simplify: false
4-
govet:
5-
check-shadowing: false
65
nolintlint:
76
allow-unused: false # report any unused nolint directives
87
require-explanation: true # require an explanation for nolint directives
@@ -23,84 +22,95 @@ issues:
2322
linters:
2423
- dogsled
2524
- gosec
26-
- revive
25+
- gocognit
26+
- errcheck
27+
- forcetypeassert
28+
- funlen
29+
- goerr113
30+
- dupl
31+
- prealloc
2732

2833
linters:
2934
# please, do not use `enable-all`: it's deprecated and will be removed soon.
3035
# inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
3136
disable-all: true
3237
enable:
38+
- asasalint
39+
- asciicheck
40+
- bidichk
3341
- bodyclose
42+
- containedctx
43+
- contextcheck
44+
- decorder
3445
# - depguard
3546
- dogsled
47+
- dupl
48+
- dupword
3649
- durationcheck
3750
- errcheck
51+
- errchkjson
3852
- errname
39-
# - errorlint
40-
# - exhaustive
41-
# - exhaustivestruct
53+
- errorlint
54+
- execinquery
55+
- exhaustive
4256
- exportloopref
4357
# - forbidigo
44-
# - forcetypeassert
45-
# - funlen
46-
# - gochecknoinits
58+
- forcetypeassert
59+
- funlen
60+
- gci
61+
- ginkgolinter
62+
- gocheckcompilerdirectives
63+
- gochecknoinits
64+
- gocognit
4765
- goconst
4866
- gocritic
49-
- gocyclo
50-
# - cyclop # not interested in package complexities at the moment
51-
# - godot
67+
- godot
68+
# - goerr113
5269
- gofmt
53-
# - gofumpt
70+
- gofumpt
5471
- goheader
5572
- goimports
56-
- revive
57-
# - gomnd
5873
- gomoddirectives
59-
- gomodguard
6074
- goprintffuncname
6175
- gosec
6276
- gosimple
77+
- gosmopolitan
6378
- govet
64-
# - ifshort
79+
- grouper
80+
- importas
6581
- ineffassign
66-
# - importas
67-
# - lll
68-
# - misspell
82+
- interfacebloat
83+
# - ireturn # Doesn't have correct support for generic types https://github.com/butuzov/ireturn/issues/37
84+
- loggercheck
85+
- maintidx
6986
- makezero
70-
# - nakedret
71-
# - nilerr
72-
# - nilnil
73-
# - nlreturn
87+
- mirror
88+
- misspell
89+
- musttag
90+
- nakedret
91+
- nestif
92+
- nilerr
93+
- nilnil
7494
- noctx
7595
- nolintlint
76-
# - paralleltest
96+
- nosprintfhostport
97+
- prealloc
7798
- predeclared
99+
- promlinter
100+
- reassign
101+
- revive
78102
- rowserrcheck
103+
- sqlclosecheck
79104
- staticcheck
80105
- stylecheck
81-
- sqlclosecheck
82-
# - tagliatelle
83-
# - tenv
106+
- tenv
107+
- testableexamples
84108
# - thelper
85-
# - tparallel
86-
- typecheck
87109
- unconvert
88-
# - unparam
110+
- unparam
89111
- unused
112+
- usestdlibvars
90113
- wastedassign
91114
- whitespace
92-
# - wrapcheck
93-
# - wsl
94-
95-
# don't enable:
96-
# - asciicheck
97-
# - dupl
98-
# - gochecknoglobals
99-
# - gocognit
100-
# - godox
101-
# - goerr113
102-
# - maligned
103-
# - nestif
104-
# - prealloc
105-
# - testpackage
106-
# - wsl
115+
- wrapcheck
116+
- zerologlint

‎.goreleaser.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ builds:
77
env:
88
- CGO_ENABLED=0
99
ldflags:
10-
- "-s -w -X 'github.com/alarbada/conduit-connector-rabbitmq.version={{ .Tag }}'"
10+
- "-s -w -X 'github.com/conduitio-labs/conduit-connector-rabbitmq.version={{ .Tag }}'"
1111
checksum:
1212
name_template: checksums.txt
1313
archives:

‎README.md

Lines changed: 76 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,20 @@
11
# Conduit Connector for RabbitMQ
2-
The RabbitMQ connector is one of [Conduit](https://github.com/ConduitIO/conduit) builtin plugins. It provides both, a
3-
source and a destination connector for [RabbitMQ](https://rabbitmq.com/).
2+
3+
The RabbitMQ connector is one of [Conduit](https://github.com/ConduitIO/conduit) builtin plugins. It provides both, a source and a destination connector for [RabbitMQ](https://rabbitmq.com/).
4+
5+
It uses the [AMQP 0-9-1 Model](https://www.rabbitmq.com/tutorials/amqp-concepts) to connect to RabbitMQ.
6+
7+
8+
## What data does the OpenCDC record consist of?
9+
10+
| Field | Description |
11+
|-------------------------|-----------------------------------------------------------------------------------------|
12+
| `record.Position` | json object with the delivery tag and the queue name from where the record was read from.|
13+
| `record.Operation` | currently fixed as "create". |
14+
| `record.Metadata` | a string to string map, with keys prefixed as `rabbitmq.{DELIVERY_PROPERTY}`. |
15+
| `record.Key` | the message id from the read message. |
16+
| `record.Payload.Before` | <empty> |
17+
| `record.Payload.After` | the message body |
418

519
## How to Build?
620

@@ -11,25 +25,15 @@ Run `make build` to compile the connector.
1125
Execute `make test` to perform all non-tls tests. Execute `make test-tls` for the TLS tests. Both commands use docker files located at `test/docker-compose.yml` and `test/docker-compose-tls.yml` respectively.
1226
Tests require docker-compose v2.
1327

14-
## Shared Configuration Parameters
15-
16-
Shared between source and destination connectors.
28+
## Source Configuration Parameters
1729

18-
| Name | Description | Required | Default Value |
19-
|------------------|-----------------------------------------------------|----------|---------------|
30+
| Name | Description | Required | Default Value |
31+
|------------------------|-----------------------------------------------------------------------------|----------|---------------|
2032
| `url` | The RabbitMQ server's URL. | Yes | |
33+
| `tls.enabled` | Flag to enable or disable TLS. | false | `false` |
2134
| `tls.clientCert` | Path to the client certificate for TLS. | No | |
2235
| `tls.clientKey` | Path to the client's key for TLS. | No | |
2336
| `tls.caCert` | Path to the CA (Certificate Authority) certificate for TLS. | No | |
24-
25-
## Source Connector
26-
27-
The source connector extracts data from RabbitMQ and sends it to downstream systems via Conduit.
28-
29-
### Source Configuration Parameters
30-
31-
| Name | Description | Required | Default Value |
32-
|------------------------|-----------------------------------------------------------------------------|----------|---------------|
3337
| `queue.name` | The name of the RabbitMQ queue to consume messages from. | Yes | |
3438
| `queue.durable` | Specifies whether the queue is durable. | No | `true` |
3539
| `queue.autoDelete` | If the queue will auto-delete. | No | `false` |
@@ -41,14 +45,15 @@ The source connector extracts data from RabbitMQ and sends it to downstream syst
4145
| `consumer.noLocal` | If the server should not deliver messages published by the same connection. | No | `false` |
4246
| `consumer.noWait` | If the consumer should be declared without waiting for server confirmation. | No | `false` |
4347

44-
## Destination Connector
48+
## Destination Configuration Parameters
4549

46-
The destination connector sends data from upstream systems to RabbitMQ via Conduit.
47-
48-
### Destination Configuration Parameters
49-
50-
| Name | Description | Required | Default Value |
51-
|----------------------------|---------------------------------------------------------------------|----------|---------------|
50+
| Name | Description | Required | Default Value |
51+
|------------------------|-----------------------------------------------------------------------------|----------|---------------|
52+
| `url` | The RabbitMQ server's URL. | Yes | |
53+
| `tls.enabled` | Flag to enable or disable TLS. | false | `false` |
54+
| `tls.clientCert` | Path to the client certificate for TLS. | No | |
55+
| `tls.clientKey` | Path to the client's key for TLS. | No | |
56+
| `tls.caCert` | Path to the CA (Certificate Authority) certificate for TLS. | No | |
5257
| `queue.name` | The name of the RabbitMQ queue where messages will be published to. | Yes | |
5358
| `queue.durable` | Specifies whether the queue is durable. | No | `true` |
5459
| `queue.autoDelete` | If the queue will auto-delete. | No | `false` |
@@ -73,3 +78,51 @@ The destination connector sends data from upstream systems to RabbitMQ via Condu
7378
| `exchange.internal` | If the exchange is internal. | No | `false` |
7479
| `exchange.noWait` | If the exchange is declared without waiting for server reply. | No | `false` |
7580
| `routingKey` | The routing key to use when publishing to an exchange. | No | |
81+
82+
83+
## Example pipeline.yml file
84+
85+
Here's an example of a `pipeline.yml` file using `file to RabbitMQ` and `RabbitMQ to file` pipelines:
86+
87+
```yaml
88+
version: 2.0
89+
pipelines:
90+
- id: file-to-rabbitmq
91+
status: running
92+
connectors:
93+
- id: file.in
94+
type: source
95+
plugin: builtin:file
96+
name: file-destination
97+
settings:
98+
path: ./file.in
99+
- id: rabbitmq.out
100+
type: destination
101+
plugin: standalone:rabbitmq
102+
name: rabbitmq-source
103+
settings:
104+
url: amqp://guest:guest@localhost:5672/
105+
queue.name: demo-queue
106+
sdk.record.format: template
107+
sdk.record.format.options: '{{ printf "%s" .Payload.After }}'
108+
109+
- id: rabbitmq-to-file
110+
status: running
111+
connectors:
112+
- id: rabbitmq.in
113+
type: source
114+
plugin: standalone:rabbitmq
115+
name: rabbitmq-source
116+
settings:
117+
url: amqp://guest:guest@localhost:5672/
118+
queue.name: demo-queue
119+
120+
- id: file.out
121+
type: destination
122+
plugin: builtin:file
123+
name: file-destination
124+
settings:
125+
path: ./file.out
126+
sdk.record.format: template
127+
sdk.record.format.options: '{{ printf "%s" .Payload.After }}'
128+
```

‎cmd/connector/main.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,8 @@
1515
package main
1616

1717
import (
18-
sdk "github.com/conduitio/conduit-connector-sdk"
19-
2018
rabbitmq "github.com/conduitio-labs/conduit-connector-rabbitmq"
19+
sdk "github.com/conduitio/conduit-connector-sdk"
2120
)
2221

2322
func main() {

‎config.go

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ type Config struct {
2828
// URL is the RabbitMQ server URL
2929
URL string `json:"url" validate:"required"`
3030

31-
TLS TLSConfig `json:"tls"`
31+
TLS TLSConfig `json:"tls"`
32+
Queue QueueConfig `json:"queue"`
3233
}
3334

3435
type TLSConfig struct {
@@ -62,7 +63,7 @@ type QueueConfig struct {
6263
NoWait bool `json:"noWait" default:"false"`
6364
}
6465

65-
// to use with ampq.Channel Consume method
66+
// to use with ampq.Channel Consume method.
6667
type ConsumerConfig struct {
6768
// Name is the name of the consumer
6869
Name string `json:"name"`
@@ -83,7 +84,6 @@ type ConsumerConfig struct {
8384
type SourceConfig struct {
8485
Config
8586

86-
Queue QueueConfig `json:"queue"`
8787
Consumer ConsumerConfig `json:"consumer"`
8888
}
8989

@@ -115,8 +115,8 @@ type DeliveryConfig struct {
115115
// ContentEncoding specifies the encoding of the message content.
116116
ContentEncoding string `json:"contentEncoding"`
117117

118-
// ContentType specifies the MIME type of the message content. Default is "text/plain".
119-
ContentType string `json:"contentType" default:"text/plain"`
118+
// ContentType specifies the MIME type of the message content. Defaults to "application/json".
119+
ContentType string `json:"contentType" default:"application/json"`
120120

121121
// DeliveryMode indicates the message delivery mode. Non-persistent (1) or persistent (2). Default is 2 (persistent).
122122
DeliveryMode uint8 `json:"deliveryMode" default:"2" validation:"inclusion=1|2"`
@@ -152,7 +152,6 @@ type DeliveryConfig struct {
152152
type DestinationConfig struct {
153153
Config
154154

155-
Queue QueueConfig `json:"queue"`
156155
Delivery DeliveryConfig `json:"delivery"`
157156
Exchange ExchangeConfig `json:"exchange"`
158157

‎destination.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ func (d *Destination) Write(ctx context.Context, records []sdk.Record) (int, err
154154
return len(records), nil
155155
}
156156

157-
func (d *Destination) Teardown(_ context.Context) error {
157+
func (d *Destination) Teardown(ctx context.Context) error {
158158
errs := make([]error, 0, 2)
159159
if d.ch != nil {
160160
if err := d.ch.Close(); err != nil {
@@ -172,7 +172,7 @@ func (d *Destination) Teardown(_ context.Context) error {
172172
return err
173173
}
174174

175-
sdk.Logger(context.Background()).Info().Msg("destination teardown complete")
175+
sdk.Logger(ctx).Info().Msg("destination teardown complete")
176176

177177
return nil
178178
}

0 commit comments

Comments
 (0)