feat: initial source implementation (#2)

* feat: initial source implementation

* seperated source logic into iterator

* added test cases

* added integration test

* remove go generate directive from source

* fix: test cases

* fix: teardown

* added go header

* added file chunk mechanism for files larger than 3 mb

* added configurable chunk

* fix: updated readme

* fix: large file processing

* fix: refactored iterator

* fix: handle file modification while read

* fix: source integration test

* modify README file

* fix: refactored iterator to provide record on demand

* added source and destination directories in docker compose

* fix: test workflow

---------

Co-authored-by: Gaurav Sahil <[email protected]>

Author: grvsahil

.github/pull_request_template.md

@@ -8,6 +8,6 @@ Fixes # (issue)
 
 ### Quick checks:
 
-- [ ] There is no other [pull request](https://github.com/conduitio/conduit-connector-connectorname/pulls) for the same update/change.
+- [ ] There is no other [pull request](https://github.com/conduitio/conduit-connector-sftp/pulls) for the same update/change.
 - [ ] I have written unit tests.
 - [ ] I have made sure that the PR is of reasonable size and can be easily reviewed.

.github/workflows/test.yml

@@ -17,4 +17,4 @@ jobs:
           go-version-file: 'go.mod'
 
       - name: Test
-        run: make test-integration GOTEST_FLAGS="-v -count=1"
+        run: make test GOTEST_FLAGS="-v -count=1"

.gitignore

@@ -20,7 +20,7 @@
 .vscode
 
 # Binary, built with `make build`
-/conduit-connector-connectorname
+/conduit-connector-sftp
 
 ### OS ###
 .DS_Store

.golangci.goheader.template

@@ -0,0 +1,13 @@
+Copyright © {{ copyright-year }} Meroxa, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

.golangci.yml

@@ -20,7 +20,12 @@ linters-settings:
       - .WithMessagef(
       - .WithStack(
       - (context.Context).Err()
-
+  goheader:
+    template-path: '.golangci.goheader.template'
+    values:
+      regexp:
+        copyright-year: 20[2-9]\d
+        
 issues:
   exclude-rules:
     - path: _test\.go

.goreleaser.yml

@@ -8,7 +8,7 @@ builds:
     env:
       - CGO_ENABLED=0
     ldflags:
-      - "-s -w -X 'github.com/conduitio/conduit-connector-connectorname.version={{ .Tag }}'"
+      - "-s -w -X 'github.com/conduitio/conduit-connector-sftp.version={{ .Tag }}'"
 checksum:
   name_template: checksums.txt
 archives:

Makefile

@@ -2,20 +2,28 @@ VERSION=$(shell git describe --tags --dirty --always)
 
 .PHONY: build
 build:
-	go build -ldflags "-X 'github.com/conduitio/conduit-connector-connectorname.version=${VERSION}'" -o conduit-connector-connectorname cmd/connector/main.go
+	go build -ldflags "-X 'github.com/conduitio/conduit-connector-sftp.version=${VERSION}'" -o conduit-connector-sftp cmd/connector/main.go
 
 .PHONY: test
 test:
-	go test $(GOTEST_FLAGS) -race ./...
-
-.PHONY: test-integration
-test-integration:
 	# run required docker containers, execute integration tests, stop containers after tests
 	docker compose -f test/docker-compose.yml up -d
 	go test $(GOTEST_FLAGS) -v -race ./...; ret=$$?; \
 		docker compose -f test/docker-compose.yml down; \
 		exit $$ret
 
+.PHONY: gofumpt
+gofumpt:
+	go install mvdan.cc/gofumpt@latest
+
+.PHONY: fmt
+fmt: gofumpt
+	gofumpt -l -w .
+
+.PHONY: lint
+lint:
+	golangci-lint run -v
+
 .PHONY: generate
 generate:
 	go generate ./...
@@ -25,11 +33,3 @@ install-tools:
 	@echo Installing tools from tools.go
 	@go list -e -f '{{ join .Imports "\n" }}' tools.go | xargs -I % go list -f "%@{{.Module.Version}}" % | xargs -tI % go install %
 	@go mod tidy
-
-.PHONY: fmt
-fmt:
-	gofumpt -l -w .
-
-.PHONY: lint
-lint:
-	golangci-lint run

README.md

@@ -1,65 +1,36 @@
-# Conduit Connector Template
+# Conduit Connector SFTP
 
-This is a template project for building [Conduit](https://conduit.io) connectors in Go. It makes it possible to
-start working on a Conduit connector in a matter of seconds.
+The SFTP connector is one of [Conduit](https://github.com/ConduitIO/conduit) plugins. It
+provides both, a source and a destination SFTP connector.
 
-## Quick start
+## How to build it
 
-1. Click [_Use this template_](https://github.com/new?template_name=conduit-connector-template&template_owner=ConduitIO) and clone your new repository.
-2. Initialize the repository using [`setup.sh`](https://github.com/ConduitIO/conduit-connector-template/blob/main/setup.sh) and commit your changes.
-   ```sh
-   ./setup.sh github.com/myusername/conduit-connector-myconnector
-   git add -A
-   git commit -m "initialize repository"
-   ```
-3. Set up [automatic Dependabot PR merges](#automatically-merging-dependabot-prs).
+Run `make build`.
 
-With that, you're all set up and ready to start working on your connector! As a next step, we recommend that you 
-check out the [Conduit Connector SDK](https://github.com/ConduitIO/conduit-connector-sdk).
+## Testing
 
-## What's included?
+Run `make test` to run all the unit and integration tests.
 
-* Skeleton code for the connector's configuration, source and destination.
-* Example unit tests.
-* A [Makefile](/Makefile) with commonly used targets.
-* A [GitHub workflow](/.github/workflows/test.yml) to build the code and run the tests.
-* A [GitHub workflow](/.github/workflows/lint.yml) to run a pre-configured set of linters.
-* A [GitHub workflow](/.github/workflows/release.yml) which automatically creates a release when a tag is pushed.
-* A [Dependabot setup](/.github/dependabot.yml) which checks your dependencies for available updates and 
-[merges minor version upgrades](/.github/workflows/dependabot-auto-merge-go.yml) automatically.
-* [Issue](/.github/ISSUE_TEMPLATE) and [PR templates](/.github/pull_request_template.md).
-* A [README template](/README_TEMPLATE.md).
+## Source
 
-## Automatically merging Dependabot PRs
+The source SFTP connector monitors a directory on an SFTP server for files matching a specified pattern. It reads these files and converts them into `opencdc.Record` that can be processed by Conduit. For handling large files, it splits them into smaller chunks, enabling smooth data handling through the Conduit pipeline.
+The connector supports both password and private key authentication methods.
 
-> [!NOTE]
-> This applies only to public connector repositories, as branch protection rules are not enforced in private repositories.
+### Configuration Options
 
-The template makes it simple to keep your connector up-to-date using automatic merging of
-[Dependabot](https://github.com/dependabot) PRs. To make use of this setup, you need to adjust
-some repository settings.
+| name           | description                                                                                           | required | default value |
+| -------------- | ----------------------------------------------------------------------------------------------------- | -------- | -------- |
+| `address` | Address is the address of the sftp server to connect.| **true** |  |
+| `hostKey` | HostKey is the key used for host key callback validation.| **true** |  |
+| `username`| User is the username of the SFTP user. | **true** |  |
+| `password`| Password is the SFTP password (can be used as passphrase for private key). | false |  |
+| `privateKeyPath`| PrivateKeyPath is the private key for ssh login.| false |  |
+| `directoryPath` | DirectoryPath is the path to the directory to read data. | **true** |  |
+| `filePattern` | Pattern to match files that should be read (e.g., "*.txt") | false | `*` |
+| `fileChunkSizeBytes` | Maximum size of a file chunk in bytes to split large files. | false | `3145728` |
 
-1. Navigate to Settings -> General and allow auto-merge of PRs.
+## Destination
 
-   ![Allow auto-merge](https://github.com/ConduitIO/conduit-connector-template/assets/8320753/695b15f0-85b4-49cb-966d-649e9bf03455)
+### Configuration Options
 
-2. Navigate to Settings -> Branches and add a branch protection rule.
-
-   ![Add branch protection rule](https://github.com/ConduitIO/conduit-connector-template/assets/8320753/9f5a07bc-d141-42b9-9918-e8d9cc648482)
-
-3. Create a rule for branch `main` that requires status checks `build` and `golangci-lint`.
-
-   ![Status checks](https://github.com/ConduitIO/conduit-connector-template/assets/8320753/96219185-c329-432a-8623-9b4462015f32)
-
-## Recommended repository settings
-
-- Allow squash merging only.
-- Always suggest updating pull request branches.
-- Automatically delete head branches.
-- Branch protection rules on branch `main` (only in public repositories):
-  - Require a pull request before merging.
-  - Require approvals.
-  - Require status checks `build` and `golangci-lint`.
-  - Require branches to be up to date before merging.
-  - Require conversation resolution before merging.
-  - Do not allow bypassing the above settings.
+![scarf pixel](https://static.scarf.sh/a.png?x-pxid=64b333ae-77ad-4895-a5cd-a73bb14362d9)

cmd/connector/main.go

@@ -1,10 +1,24 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (
-	connectorname "github.com/conduitio/conduit-connector-connectorname"
+	sftp "github.com/conduitio-labs/conduit-connector-sftp"
 	sdk "github.com/conduitio/conduit-connector-sdk"
 )
 
 func main() {
-	sdk.Serve(connectorname.Connector)
+	sdk.Serve(sftp.Connector)
 }

config.go

@@ -0,0 +1,45 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:generate paramgen -output=paramgen.go Config
+
+package config
+
+import "fmt"
+
+var ErrEmptyAuthFields = fmt.Errorf("both %q and %q can not be empty", ConfigPassword, ConfigPrivateKeyPath)
+
+// Config contains shared config parameters, common to the source and destination.
+type Config struct {
+	// Address is the address of the sftp server to connect.
+	Address string `json:"address" validate:"required"`
+	// HostKey is the key used for host key callback validation.
+	HostKey string `json:"hostKey" validate:"required"`
+	// User is the SFTP user.
+	Username string `json:"username" validate:"required"`
+	// Password is the SFTP password (can be used as passphrase for private key).
+	Password string `json:"password"`
+	// PrivateKeyPath is the private key for ssh login.
+	PrivateKeyPath string `json:"privateKeyPath"`
+	// DirectoryPath is the path to the directory to read/write data.
+	DirectoryPath string `json:"directoryPath" validate:"required"`
+}
+
+// Validate is used for custom validation for sftp authentication configuration.
+func (c *Config) Validate() error {
+	if c.Password == "" && c.PrivateKeyPath == "" {
+		return ErrEmptyAuthFields
+	}
+	return nil
+}

config/config.go

@@ -0,0 +1,76 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"testing"
+)
+
+func TestConfig_Validate(t *testing.T) {
+	type fields struct {
+		Address        string
+		Username       string
+		Password       string
+		PrivateKeyPath string
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		wantErr error
+	}{
+		{
+			name: "success: password authentication",
+			fields: fields{
+				Address:  "localhost:22",
+				Username: "user",
+				Password: "pass",
+			},
+			wantErr: nil,
+		},
+		{
+			name: "success: privatekey authentication",
+			fields: fields{
+				Address:        "localhost:22",
+				Username:       "user",
+				PrivateKeyPath: "path",
+			},
+			wantErr: nil,
+		},
+		{
+			name: "error: missing password and privateKeyPath",
+			fields: fields{
+				Address:  "localhost:22",
+				Username: "user",
+			},
+			wantErr: ErrEmptyAuthFields,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &Config{
+				Address:        tt.fields.Address,
+				Username:       tt.fields.Username,
+				Password:       tt.fields.Password,
+				PrivateKeyPath: tt.fields.PrivateKeyPath,
+			}
+			err := c.Validate()
+			if err != nil {
+				if tt.wantErr != nil && tt.wantErr.Error() != err.Error() {
+					t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr)
+				}
+			}
+		})
+	}
+}