Destination connector implementation.

.github/pull_request_template.md

@@ -8,6 +8,6 @@ Fixes # (issue)
 
 ### Quick checks:
 
-- [ ] There is no other [pull request](https://github.com/conduitio/conduit-connector-connectorname/pulls) for the same update/change.
+- [ ] There is no other [pull request](https://github.com/conduitio/conduit-connector-sftp/pulls) for the same update/change.
 - [ ] I have written unit tests.
 - [ ] I have made sure that the PR is of reasonable size and can be easily reviewed.

.gitignore

@@ -20,7 +20,7 @@
 .vscode
 
 # Binary, built with `make build`
-/conduit-connector-connectorname
+/conduit-connector-sftp
 
 ### OS ###
 .DS_Store

.golangci.goheader.template

@@ -0,0 +1,13 @@
+Copyright © {{ copyright-year }} Meroxa, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

.golangci.yml

@@ -20,7 +20,12 @@ linters-settings:
       - .WithMessagef(
       - .WithStack(
       - (context.Context).Err()
-
+  goheader:
+    template-path: '.golangci.goheader.template'
+    values:
+      regexp:
+        copyright-year: 20[2-9]\d
+
 issues:
   exclude-rules:
     - path: _test\.go

.goreleaser.yml

@@ -8,7 +8,7 @@ builds:
     env:
       - CGO_ENABLED=0
     ldflags:
-      - "-s -w -X 'github.com/conduitio/conduit-connector-connectorname.version={{ .Tag }}'"
+      - "-s -w -X 'github.com/conduitio/conduit-connector-sftp.version={{ .Tag }}'"
 checksum:
   name_template: checksums.txt
 archives:

Makefile

@@ -2,7 +2,7 @@ VERSION=$(shell git describe --tags --dirty --always)
 
 .PHONY: build
 build:
-	go build -ldflags "-X 'github.com/conduitio/conduit-connector-connectorname.version=${VERSION}'" -o conduit-connector-connectorname cmd/connector/main.go
+	go build -ldflags "-X 'github.com/conduitio/conduit-connector-sftp.version=${VERSION}'" -o conduit-connector-sftp cmd/connector/main.go
 
 .PHONY: test
 test:
@@ -16,6 +16,18 @@ test-integration:
 		docker compose -f test/docker-compose.yml down; \
 		exit $$ret
 
+.PHONY: gofumpt
+gofumpt:
+	go install mvdan.cc/gofumpt@latest
+
+.PHONY: fmt
+fmt: gofumpt
+	gofumpt -l -w .
+
+.PHONY: lint
+lint:
+	golangci-lint run -v
+
 .PHONY: generate
 generate:
 	go generate ./...
@@ -25,11 +37,3 @@ install-tools:
 	@echo Installing tools from tools.go
 	@go list -e -f '{{ join .Imports "\n" }}' tools.go | xargs -I % go list -f "%@{{.Module.Version}}" % | xargs -tI % go install %
 	@go mod tidy
-
-.PHONY: fmt
-fmt:
-	gofumpt -l -w .
-
-.PHONY: lint
-lint:
-	golangci-lint run

README.md

@@ -1,65 +1,48 @@
-# Conduit Connector Template
+# Conduit Connector SFTP
 
-This is a template project for building [Conduit](https://conduit.io) connectors in Go. It makes it possible to
-start working on a Conduit connector in a matter of seconds.
+The SFTP connector is one of [Conduit](https://github.com/ConduitIO/conduit) plugins. It
+provides both, a source and a destination SFTP connector.
 
-## Quick start
+## How to build it
 
-1. Click [_Use this template_](https://github.com/new?template_name=conduit-connector-template&template_owner=ConduitIO) and clone your new repository.
-2. Initialize the repository using [`setup.sh`](https://github.com/ConduitIO/conduit-connector-template/blob/main/setup.sh) and commit your changes.
-   ```sh
-   ./setup.sh github.com/myusername/conduit-connector-myconnector
-   git add -A
-   git commit -m "initialize repository"
-   ```
-3. Set up [automatic Dependabot PR merges](#automatically-merging-dependabot-prs).
+Run `make build`.
 
-With that, you're all set up and ready to start working on your connector! As a next step, we recommend that you 
-check out the [Conduit Connector SDK](https://github.com/ConduitIO/conduit-connector-sdk).
+## Testing
 
-## What's included?
+Run `make test` to run all the unit and integration tests.
 
-* Skeleton code for the connector's configuration, source and destination.
-* Example unit tests.
-* A [Makefile](/Makefile) with commonly used targets.
-* A [GitHub workflow](/.github/workflows/test.yml) to build the code and run the tests.
-* A [GitHub workflow](/.github/workflows/lint.yml) to run a pre-configured set of linters.
-* A [GitHub workflow](/.github/workflows/release.yml) which automatically creates a release when a tag is pushed.
-* A [Dependabot setup](/.github/dependabot.yml) which checks your dependencies for available updates and 
-[merges minor version upgrades](/.github/workflows/dependabot-auto-merge-go.yml) automatically.
-* [Issue](/.github/ISSUE_TEMPLATE) and [PR templates](/.github/pull_request_template.md).
-* A [README template](/README_TEMPLATE.md).
+## Source
 
-## Automatically merging Dependabot PRs
+The source SFTP connector monitors a directory on an SFTP server for files matching a specified pattern. It reads these files and converts them into `opencdc.Record` that can be processed by Conduit. For handling large files, it splits them into smaller chunks, enabling smooth data handling through the Conduit pipeline.
+The connector supports both password and private key authentication methods.
 
-> [!NOTE]
-> This applies only to public connector repositories, as branch protection rules are not enforced in private repositories.
+### Configuration Options
 
-The template makes it simple to keep your connector up-to-date using automatic merging of
-[Dependabot](https://github.com/dependabot) PRs. To make use of this setup, you need to adjust
-some repository settings.
+| name           | description                                                                                           | required | default value |
+| -------------- | ----------------------------------------------------------------------------------------------------- | -------- | -------- |
+| `address` | Address is the address of the sftp server to connect.| **true** |  |
+| `hostKey` | HostKey is the key used for host key callback validation.| **true** |  |
+| `username`| User is the username of the SFTP user. | **true** |  |
+| `password`| Password is the SFTP password (can be used as passphrase for private key). | false |  |
+| `privateKeyPath`| PrivateKeyPath is the private key for ssh login.| false |  |
+| `directoryPath` | DirectoryPath is the path to the directory to read data. | **true** |  |
+| `filePattern` | Pattern to match files that should be read (e.g., "*.txt") | false | `*` |
+| `pollingPeriod` | Duration for polling SFTP for fetching new records. | false | `5s` |
+| `fileChunkSizeBytes` | Maximum size of a file chunk in bytes to split large files. | false | `3145728` |
 
-1. Navigate to Settings -> General and allow auto-merge of PRs.
+## Destination
 
-   ![Allow auto-merge](https://github.com/ConduitIO/conduit-connector-template/assets/8320753/695b15f0-85b4-49cb-966d-649e9bf03455)
+Destination connects to a remote server. It takes an `opencdc.Record`, extracts filename from the metadata and upload the file to the remote server. The connector supports both password and private key authentication methods.
 
-2. Navigate to Settings -> Branches and add a branch protection rule.
+### Configuration Options
 
-   ![Add branch protection rule](https://github.com/ConduitIO/conduit-connector-template/assets/8320753/9f5a07bc-d141-42b9-9918-e8d9cc648482)
+| name           | description                                                                                           | required |
+| -------------- | ----------------------------------------------------------------------------------------------------- | -------- |
+| `address` | Address is the address of the sftp server to connect.| **true** |
+| `hostKey` | HostKey is the key used for host key callback validation.| **true** |
+| `username`| User is the username of the SFTP user. | **true** |
+| `password`| Password is the SFTP password (can be used as passphrase for private key). | false |
+| `privateKeyPath`| PrivateKeyPath is the private key for ssh login.| false |
+| `directoryPath` | DirectoryPath is the path to the directory to read/write data. | true |
 
-3. Create a rule for branch `main` that requires status checks `build` and `golangci-lint`.
-
-   ![Status checks](https://github.com/ConduitIO/conduit-connector-template/assets/8320753/96219185-c329-432a-8623-9b4462015f32)
-
-## Recommended repository settings
-
-- Allow squash merging only.
-- Always suggest updating pull request branches.
-- Automatically delete head branches.
-- Branch protection rules on branch `main` (only in public repositories):
-  - Require a pull request before merging.
-  - Require approvals.
-  - Require status checks `build` and `golangci-lint`.
-  - Require branches to be up to date before merging.
-  - Require conversation resolution before merging.
-  - Do not allow bypassing the above settings.
+![scarf pixel](https://static.scarf.sh/a.png?x-pxid=64b333ae-77ad-4895-a5cd-a73bb14362d9)

acceptance_test.go

@@ -0,0 +1,94 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sftp
+
+import (
+	"fmt"
+	"os/exec"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/conduitio-labs/conduit-connector-sftp/config"
+	"github.com/conduitio-labs/conduit-connector-sftp/destination"
+	"github.com/conduitio/conduit-commons/opencdc"
+	sdk "github.com/conduitio/conduit-connector-sdk"
+)
+
+type driver struct {
+	sdk.ConfigurableAcceptanceTestDriver
+	id int64
+}
+
+func (d *driver) GenerateRecord(_ *testing.T, _ opencdc.Operation) opencdc.Record {
+	atomic.AddInt64(&d.id, 1)
+
+	content := []byte("hello world")
+	filename := fmt.Sprintf("%d.txt", d.id)
+
+	return sdk.Util.Source.NewRecordCreate(
+		nil,
+		map[string]string{
+			opencdc.MetadataCollection: "upload",
+			opencdc.MetadataCreatedAt:  time.Now().UTC().Format(time.RFC3339),
+			"filename":                 filename,
+			"source_path":              "/upload",
+			"file_size":                fmt.Sprintf("%d", len(content)),
+			"mod_time":                 time.Now().UTC().Format(time.RFC3339),
+		},
+		opencdc.StructuredData{"filename": filename},
+		opencdc.RawData(content),
+	)
+}
+
+func (d *driver) ReadFromDestination(_ *testing.T, records []opencdc.Record) []opencdc.Record {
+	return records
+}
+
+func TestAcceptance(t *testing.T) {
+	hostKey, err := setupHostKey()
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	sdk.AcceptanceTest(t, &driver{
+		ConfigurableAcceptanceTestDriver: sdk.ConfigurableAcceptanceTestDriver{
+			Config: sdk.ConfigurableAcceptanceTestDriverConfig{
+				Connector: sdk.Connector{
+					NewSpecification: Specification,
+					NewDestination:   destination.NewDestination,
+					NewSource:        nil,
+				},
+				DestinationConfig: map[string]string{
+					config.ConfigAddress:       "localhost:2222",
+					config.ConfigHostKey:       hostKey,
+					config.ConfigUsername:      "user",
+					config.ConfigPassword:      "pass",
+					config.ConfigDirectoryPath: "/upload",
+				},
+			},
+		},
+	})
+}
+
+func setupHostKey() (string, error) {
+	cmd := exec.Command("ssh-keyscan", "-t", "rsa", "-p", "2222", "localhost")
+	output, err := cmd.Output()
+	if err != nil {
+		return "", fmt.Errorf("error setupHostKey: %w", err)
+	}
+	return string(output), nil
+}

cmd/connector/main.go

@@ -1,10 +1,24 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (
-	connectorname "github.com/conduitio/conduit-connector-connectorname"
+	sftp "github.com/conduitio-labs/conduit-connector-sftp"
 	sdk "github.com/conduitio/conduit-connector-sdk"
 )
 
 func main() {
-	sdk.Serve(connectorname.Connector)
+	sdk.Serve(sftp.Connector)
 }