Skip to content

Commit e819829

Browse files
hindessmhindessm
committed
Add S3 watch node.
1 parent 08388ac commit e819829

File tree

3 files changed

+129
-1
lines changed

3 files changed

+129
-1
lines changed

‎aws/s3.html

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,53 @@
1414
limitations under the License.
1515
-->
1616

17+
<script type="text/x-red" data-template-name="amazon s3 in">
18+
<div class="form-row">
19+
<label for="node-input-aws"><i class="fa fa-user"></i> AWS</label>
20+
<input type="text" id="node-input-aws">
21+
</div>
22+
<div class="form-row">
23+
<label for="node-input-bucket"><i class="fa fa-folder"></i> Bucket</label>
24+
<input type="text" id="node-input-bucket" placeholder="bucket name">
25+
</div>
26+
<div class="form-row node-input-filepattern">
27+
<label for="node-input-filepattern"><i class="fa fa-file"></i> Filename Pattern</label>
28+
<input type="text" id="node-input-filepattern" placeholder="Filepattern">
29+
</div>
30+
<div class="form-row">
31+
<label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
32+
<input type="text" id="node-input-name" placeholder="Name">
33+
</div>
34+
</script>
35+
36+
<script type="text/x-red" data-help-name="amazon s3 in">
37+
<p>Amazon S3 watch node. Watches for file events on Box. By default all
38+
file events are reported, but the filename pattern can be supplied
39+
to limit the events to files which have full filenames that match
40+
the glob pattern. The event messages consist of the full filename
41+
in <b>msg.payload</b> property, the filename in <b>msg.file</b>,
42+
the event type in <b>msg.event</b>.</p>
43+
</script>
44+
45+
<script type="text/javascript">
46+
RED.nodes.registerType('amazon s3 in',{
47+
category: 'storage-input',
48+
color:"#C0DEED",
49+
defaults: {
50+
aws: {type:"aws-config",required:true},
51+
bucket: {required:true},
52+
filepattern: {value:""},
53+
name: {value:""},
54+
},
55+
inputs:0,
56+
outputs:1,
57+
icon: "amazon.png",
58+
label: function() {
59+
return this.bucket ? "s3 "+this.bucket : "s3";
60+
}
61+
});
62+
</script>
63+
1764
<script type="text/x-red" data-template-name="amazon s3">
1865
<div class="form-row">
1966
<label for="node-input-aws"><i class="fa fa-user"></i> AWS</label>

‎aws/s3.js

Lines changed: 81 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,86 @@
1717
module.exports = function(RED) {
1818
"use strict";
1919
var fs = require('fs');
20+
var minimatch = require("minimatch");
21+
22+
function AmazonS3InNode(n) {
23+
RED.nodes.createNode(this,n);
24+
this.awsConfig = RED.nodes.getNode(n.aws);
25+
this.region = n.region;
26+
this.bucket = n.bucket;
27+
this.filepattern = n.filepattern || "";
28+
var node = this;
29+
var AWS = this.awsConfig ? this.awsConfig.AWS : null;
30+
if (!AWS) {
31+
node.warn("Missing AWS credentials");
32+
return;
33+
}
34+
var s3 = new AWS.S3();
35+
node.status({fill:"blue",shape:"dot",text:"initializing"});
36+
s3.listObjects({ Bucket: node.bucket }, function(err, data) {
37+
if (err) {
38+
node.error("failed to fetch S3 state: " + err);
39+
node.status({fill:"red",shape:"ring",text:"error"});
40+
return;
41+
}
42+
node.state = data.Contents.filter(function (e) {
43+
return !node.filepattern || minimatch(e, node.filepattern);
44+
}).map(function (e) {
45+
return e.Key;
46+
});
47+
node.status({});
48+
node.on("input", function(msg) {
49+
node.status({fill:"blue",shape:"dot",text:"checking for changes"});
50+
s3.listObjects({ Bucket: node.bucket }, function(err, data) {
51+
if (err) {
52+
node.warn("failed to fetch S3 state: " + err);
53+
node.status({});
54+
return;
55+
}
56+
node.status({});
57+
var newState = data.Contents.filter(function (e) {
58+
return !node.filepattern ||
59+
minimatch(e, node.filepattern);
60+
}).map(function (e) {
61+
return e.Key;
62+
});
63+
var seen = {};
64+
var i;
65+
for (i = 0; i < node.state.length; i++) {
66+
seen[node.state[i]] = true;
67+
}
68+
for (i = 0; i < newState.length; i++) {
69+
if (seen[newState[i]]) {
70+
delete seen[newState[i]];
71+
} else {
72+
msg.payload = newState[i];
73+
msg.file = newState[i].substring(newState[i].lastIndexOf('/') + 1);
74+
msg.event = 'add';
75+
node.send(msg);
76+
}
77+
}
78+
for (var k in seen) {
79+
if (seen.hasOwnProperty(k)) {
80+
msg.payload = k;
81+
msg.file = k.substring(k.lastIndexOf('/') + 1);
82+
msg.event = 'delete';
83+
node.send(msg);
84+
}
85+
}
86+
node.state = newState;
87+
});
88+
});
89+
var interval = setInterval(function() {
90+
node.emit("input", {});
91+
}, 900000); // 15 minutes
92+
node.on("close", function() {
93+
if (interval !== null) {
94+
clearInterval(interval);
95+
}
96+
});
97+
});
98+
}
99+
RED.nodes.registerType("amazon s3 in", AmazonS3InNode);
20100

21101
function AmazonS3QueryNode(n) {
22102
RED.nodes.createNode(this,n);
@@ -32,7 +112,6 @@ module.exports = function(RED) {
32112
}
33113
var s3 = new AWS.S3();
34114
node.on("input", function(msg) {
35-
msg.filename = filename;
36115
var bucket = node.bucket || msg.bucket;
37116
if (bucket === "") {
38117
node.warn("No bucket specified");
@@ -43,6 +122,7 @@ module.exports = function(RED) {
43122
node.warn("No filename specified");
44123
return;
45124
}
125+
msg.bucket = bucket;
46126
msg.filename = filename;
47127
node.status({fill:"blue",shape:"dot",text:"downloading"});
48128
s3.getObject({

‎package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
"dependencies" : {
66
"aws-sdk": "^2.0.15",
77
"dropbox":"^0.10.3",
8+
"minimatch": "^1.0.0",
89
"oauth":"~0.9.11",
910
"request":"~2.40.0",
1011
"instagram-node":"0.5.1",

0 commit comments

Comments
 (0)