Skip to content

Commit 3b40919

Browse files
sakamasakama
committed
first commit
0 parents  commit 3b40919

File tree

18 files changed

+1273
-0
lines changed

18 files changed

+1273
-0
lines changed

‎.gitignore

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
target/
2+
build/
3+
pkg/
4+
*.iml
5+
*~
6+
._*
7+
.idea
8+
tmp/
9+
vendor/
10+
/classpath/
11+
/.bundle
12+
.yardoc
13+
/embulk-*.jar
14+
/.gradle

‎README.md

+126
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
# SFTP file input plugin for Embulk
2+
3+
Reads files stored on remote server using SFTP
4+
5+
## Overview
6+
7+
* **Plugin type**: file input
8+
* **Resume supported**: yes
9+
* **Cleanup supported**: yes
10+
11+
## Configuration
12+
13+
- **host**: (string, required)
14+
- **port**: (string, default: `22`)
15+
- **user**: (string, required)
16+
- **password**: (string, default: `null`)
17+
- **secret_key_file**: (string, default: `null`)
18+
- **secret_key_passphrase**: (string, default: `""`)
19+
- **user_directory_is_root**: (boolean, default: `true`)
20+
- **timeout**: sftp connection timeout seconds (integer, default: `600`)
21+
- **path_prefix**: Prefix of output paths (string, required)
22+
- **file_ext**: Extension of output files (string, required)
23+
- **sequence_format**: Format for sequence part of output files (string, default: `".%03d.%02d"`)
24+
25+
### Proxy configuration
26+
27+
- **proxy**:
28+
- **type**: (string(http | socks | stream), required, default: `null`)
29+
- **http**: use HTTP Proxy
30+
- **socks**: use SOCKS Proxy
31+
- **stream**: Connects to the SFTP server through a remote host reached by SSH
32+
- **host**: (string, required)
33+
- **port**: (int, default: `22`)
34+
- **user**: (string, optional)
35+
- **password**: (string, optional, default: `null`)
36+
- **command**: (string, optional)
37+
38+
### Example
39+
40+
```yaml
41+
in:
42+
type: sftp
43+
host: 127.0.0.1
44+
port: 22
45+
user: embulk
46+
secret_key_file: /Users/embulk/.ssh/id_rsa
47+
secret_key_passphrase: secret_pass
48+
user_directory_is_root: false
49+
timeout: 600
50+
path_prefix: /data/sftp
51+
```
52+
53+
With proxy
54+
```yaml
55+
in:
56+
type: sftp
57+
host: 127.0.0.1
58+
port: 22
59+
user: embulk
60+
secret_key_file: /Users/embulk/.ssh/id_rsa
61+
secret_key_passphrase: secret_pass
62+
user_directory_is_root: false
63+
timeout: 600
64+
path_prefix: /data/sftp
65+
proxy:
66+
type: http
67+
host: proxy_host
68+
port: 8080
69+
user: proxy_user
70+
password: proxy_secret_pass
71+
command:
72+
```
73+
74+
## Proxy settings
75+
76+
### Example
77+
```yaml
78+
in:
79+
type: sftp
80+
host: 127.0.0.1
81+
port: 22
82+
user: embulk
83+
secret_key_file: /Users/embulk/.ssh/id_rsa
84+
secret_key_passphrase: secret_pass
85+
user_directory_is_root: false
86+
timeout: 600
87+
path_prefix: /data/sftp
88+
```
89+
90+
### Secret Keyfile configuration
91+
92+
Please set path of secret_key_file as follows.
93+
```yaml
94+
in:
95+
type: sftp
96+
...
97+
secret_key_file: /path/to/id_rsa
98+
...
99+
```
100+
101+
You can also embed contents of secret_key_file at config.yml.
102+
```yaml
103+
in:
104+
type: sftp
105+
...
106+
secret_key_file:
107+
content |
108+
-----BEGIN RSA PRIVATE KEY-----
109+
ABCDEFG...
110+
HIJKLMN...
111+
OPQRSTU...
112+
-----END RSA PRIVATE KEY-----
113+
...
114+
```
115+
116+
## Build
117+
118+
```
119+
$ ./gradlew gem # -t to watch change of files and rebuild continuously
120+
```
121+
122+
## Test
123+
124+
```
125+
$ ./gradlew test # -t to watch change of files and rebuild continuously
126+
```

‎build.gradle

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
plugins {
2+
id "com.jfrog.bintray" version "1.1"
3+
id "com.github.jruby-gradle.base" version "0.1.5"
4+
id "java"
5+
id "checkstyle"
6+
}
7+
import com.github.jrubygradle.JRubyExec
8+
repositories {
9+
mavenCentral()
10+
jcenter()
11+
}
12+
configurations {
13+
provided
14+
}
15+
16+
version = "0.1.0"
17+
18+
sourceCompatibility = 1.7
19+
targetCompatibility = 1.7
20+
21+
dependencies {
22+
compile "org.embulk:embulk-core:0.8.6"
23+
provided "org.embulk:embulk-core:0.8.6"
24+
compile "org.apache.commons:commons-vfs2:2.+"
25+
compile "org.apache.commons:commons-io:1.3.2"
26+
compile "com.jcraft:jsch:0.1.53"
27+
testCompile "junit:junit:4.+"
28+
testCompile "org.embulk:embulk-core:0.8.6:tests"
29+
testCompile "org.embulk:embulk-standards:0.8.6"
30+
testCompile "org.apache.sshd:apache-sshd:1.1.0"
31+
testCompile "org.littleshoot:littleproxy:1.1.0-beta1"
32+
testCompile "io.netty:netty-all:4.0.34.Final"
33+
}
34+
35+
task classpath(type: Copy, dependsOn: ["jar"]) {
36+
doFirst { file("classpath").deleteDir() }
37+
from (configurations.runtime - configurations.provided + files(jar.archivePath))
38+
into "classpath"
39+
}
40+
clean { delete "classpath" }
41+
42+
checkstyle {
43+
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
44+
toolVersion = '6.14.1'
45+
}
46+
checkstyleMain {
47+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
48+
ignoreFailures = true
49+
}
50+
checkstyleTest {
51+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
52+
ignoreFailures = true
53+
}
54+
task checkstyle(type: Checkstyle) {
55+
classpath = sourceSets.main.output + sourceSets.test.output
56+
source = sourceSets.main.allJava + sourceSets.test.allJava
57+
}
58+
59+
task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
60+
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
61+
script "build/gemspec"
62+
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
63+
}
64+
65+
task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
66+
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
67+
script "pkg/${project.name}-${project.version}.gem"
68+
}
69+
70+
task "package"(dependsOn: ["gemspec", "classpath"]) << {
71+
println "> Build succeeded."
72+
println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
73+
}
74+
75+
task gemspec << { file("build/gemspec").write($/
76+
Gem::Specification.new do |spec|
77+
spec.name = "${project.name}"
78+
spec.version = "${project.version}"
79+
spec.authors = ["Satoshi Akama"]
80+
spec.summary = %[SFTP file input plugin for Embulk]
81+
spec.description = %[Reads files stored on remote server using SFTP.]
82+
spec.email = ["[email protected]"]
83+
spec.licenses = ["Apache-2.0"]
84+
spec.homepage = "https://github.com/sakama/embulk-input-sftp"
85+
86+
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
87+
spec.test_files = spec.files.grep(%r"^(test|spec)/")
88+
spec.require_paths = ["lib"]
89+
90+
spec.add_development_dependency 'bundler', ['~> 1.0']
91+
spec.add_development_dependency 'rake', ['>= 10.0']
92+
end
93+
/$)
94+
}
95+
clean { delete "${project.name}.gemspec" }

‎config/checkstyle/checkstyle.xml

+128
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!DOCTYPE module PUBLIC
3+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5+
<module name="Checker">
6+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
7+
<module name="FileTabCharacter"/>
8+
<module name="NewlineAtEndOfFile">
9+
<property name="lineSeparator" value="lf"/>
10+
</module>
11+
<module name="RegexpMultiline">
12+
<property name="format" value="\r"/>
13+
<property name="message" value="Line contains carriage return"/>
14+
</module>
15+
<module name="RegexpMultiline">
16+
<property name="format" value=" \n"/>
17+
<property name="message" value="Line has trailing whitespace"/>
18+
</module>
19+
<module name="RegexpMultiline">
20+
<property name="format" value="\{\n\n"/>
21+
<property name="message" value="Blank line after opening brace"/>
22+
</module>
23+
<module name="RegexpMultiline">
24+
<property name="format" value="\n\n\s*\}"/>
25+
<property name="message" value="Blank line before closing brace"/>
26+
</module>
27+
<module name="RegexpMultiline">
28+
<property name="format" value="\n\n\n"/>
29+
<property name="message" value="Multiple consecutive blank lines"/>
30+
</module>
31+
<module name="RegexpMultiline">
32+
<property name="format" value="\n\n\Z"/>
33+
<property name="message" value="Blank line before end of file"/>
34+
</module>
35+
<module name="RegexpMultiline">
36+
<property name="format" value="Preconditions\.checkNotNull"/>
37+
<property name="message" value="Use of checkNotNull"/>
38+
</module>
39+
40+
<module name="TreeWalker">
41+
<module name="EmptyBlock">
42+
<property name="option" value="text"/>
43+
<property name="tokens" value="
44+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
45+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
46+
</module>
47+
<module name="EmptyStatement"/>
48+
<module name="EmptyForInitializerPad"/>
49+
<module name="EmptyForIteratorPad">
50+
<property name="option" value="space"/>
51+
</module>
52+
<module name="MethodParamPad">
53+
<property name="allowLineBreaks" value="true"/>
54+
<property name="option" value="nospace"/>
55+
</module>
56+
<module name="ParenPad"/>
57+
<module name="TypecastParenPad"/>
58+
<module name="NeedBraces"/>
59+
<module name="LeftCurly">
60+
<property name="option" value="nl"/>
61+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
62+
</module>
63+
<module name="LeftCurly">
64+
<property name="option" value="eol"/>
65+
<property name="tokens" value="
66+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
67+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
68+
</module>
69+
<module name="RightCurly">
70+
<property name="option" value="alone"/>
71+
</module>
72+
<module name="GenericWhitespace"/>
73+
<module name="WhitespaceAfter"/>
74+
<module name="NoWhitespaceBefore"/>
75+
76+
<module name="UpperEll"/>
77+
<module name="DefaultComesLast"/>
78+
<module name="ArrayTypeStyle"/>
79+
<module name="MultipleVariableDeclarations"/>
80+
<module name="ModifierOrder"/>
81+
<module name="OneStatementPerLine"/>
82+
<module name="StringLiteralEquality"/>
83+
<module name="MutableException"/>
84+
<module name="EqualsHashCode"/>
85+
<module name="InnerAssignment"/>
86+
<module name="InterfaceIsType"/>
87+
<module name="HideUtilityClassConstructor"/>
88+
89+
<module name="MemberName"/>
90+
<module name="LocalVariableName"/>
91+
<module name="LocalFinalVariableName"/>
92+
<module name="TypeName"/>
93+
<module name="PackageName"/>
94+
<module name="ParameterName"/>
95+
<module name="StaticVariableName"/>
96+
<module name="ClassTypeParameterName">
97+
<property name="format" value="^[A-Z][0-9]?$"/>
98+
</module>
99+
<module name="MethodTypeParameterName">
100+
<property name="format" value="^[A-Z][0-9]?$"/>
101+
</module>
102+
103+
<module name="AvoidStarImport"/>
104+
<module name="RedundantImport"/>
105+
<module name="UnusedImports"/>
106+
<module name="ImportOrder">
107+
<property name="groups" value="*,javax,java"/>
108+
<property name="separated" value="true"/>
109+
<property name="option" value="bottom"/>
110+
<property name="sortStaticImportsAlphabetically" value="true"/>
111+
</module>
112+
113+
<module name="WhitespaceAround">
114+
<property name="allowEmptyConstructors" value="true"/>
115+
<property name="allowEmptyMethods" value="true"/>
116+
<property name="ignoreEnhancedForColon" value="false"/>
117+
<property name="tokens" value="
118+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
119+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
120+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
121+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
122+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
123+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
124+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
125+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
126+
</module>
127+
</module>
128+
</module>

0 commit comments

Comments
 (0)