Optional Redis support

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## [1.9.0] - 01/08/2019
+Added optional **[Redis](https://redis.io) support** for big clusters (recommended for clusters with more than 50 workers/hosts).
+
 ## [1.8.9] - 01/08/2019
 Fixed CVE issue with docker-cli-js v2.5.2 ==> v2.5.3 & removed package-lock.json
 

README.md

@@ -251,6 +251,21 @@ Also, you can start DockerServerwith these parameters:
     <td>Shared folder between all docker-servers. (Used only in cluster mode)</td>
     <td>/tmp/docker-server</td>
   </tr>
+  <tr>
+    <td><code>--redis [hostname]</code></td>
+    <td>
+      Shared redis server hostname.<br>
+      If set, the `--folder` param. becomes the key prefix (default: "DSC:")
+    </td>
+    <td>null</td>
+  </tr>
+  <tr>
+    <td><code>--port [num]</code></td>
+    <td>
+      Shared server port number.<br>
+    </td>
+    <td>6379</td>
+  </tr>
   <tr>
     <td><code>--s3 [bucket-name]</code></td>
     <td>
@@ -396,11 +411,9 @@ And/or
 
 ## Changelog
 
-1.8.9 - Fixed CVE issue with docker-cli-js v2.5.2 ==> v2.5.3 & removed package-lock.json
-
-1.8.8 - Fixed volumes option (because of deprecated context var).
+1.9.0 - Added optional **[Redis](https://redis.io) support** for big clusters (recommended for clusters with more than 50 workers/hosts).
 
-1.8.5 - Added **docker pull** option as a HEAD request && **S3 support** for fully distributed clusters.
+1.8.9 - Fixed CVE issue with docker-cli-js v2.5.2 ==> v2.5.3 & removed package-lock.json
 
 [See full changelog](https://github.com/freaker2k7/dockerserver/blob/master/CHANGELOG.md)
 

lib/args.js

@@ -1,4 +1,7 @@
 // This file is dedicated for all the cli arguments.
+
+var default_tmp_folder = '/tmp/docker-server';
+
 var args = require('yargs')
 	.option('port', {describe: 'DockerServer port.', type: 'number', default: parseInt(process.env.DS_PORT) || 1717})
 	.option('token', {describe: 'Secret tocket (recommended between 1024-4096 chars).', type: 'string', default: process.env.DS_TOKEN || 'xxxxxx'})
@@ -9,17 +12,27 @@ var args = require('yargs')
 	.option('cluster', {describe: 'Flag to turn on the Cluster mode.', type: 'boolean', default: false})
 	.option('refresh_rate', {describe: 'Milliseconds between writes to the shared memory.', type: 'number', default: 30000})
 	.option('cache_interval', {describe: 'Milliseconds to cache reads of all the machines.', type: 'number', default: 3000})
-	.option('folder', {describe: 'Shared folder between all docker-servers.', type: 'string', default: '/tmp/docker-server'})
+	.option('folder', {describe: 'Shared folder between all docker-servers.', type: 'string', default: default_tmp_folder})
+	.option('redis', {describe: 'Shared redis server hostname.', type: 'string', default: null})
+	.option('db_port', {describe: 'Shared server port number.', type: 'number', default: 6379})
 	.option('s3', {describe: 'S3 bucket name, use with high refresh_rate.', type: 'string', default: null})
 	.option('log_level', {describe: 'Log level [trace|debug|info|warn|error|fatal]', type: 'string', default: 'info'})
 	.option('log_expiry', {describe: 'Time for a log to live in days.', type: 'number', default: 14})
 	.option('log_max_size', {describe: 'Max log size in MB.', type: 'number', default: 25})
 	.help('help', 'Show help.\nFor more documentation see https://github.com/freaker2k7/dockerserver')
 	.argv;
 
-
-// Add a trailing slash to the folder path.
-args.folder = args.folder.replace(/\/+$/g, '') + '/';
+if (args.redis) {
+	if (args.folder === default_tmp_folder) {
+		// See https://github.com/freaker2k7/dockerserver#parameters
+		// If --redis is set, the `--folder` parameter becomes the key prefix (default: "DSC:")
+		args.folder = 'DSC:';
+	}
+	args.expiry_seconds = parseInt(args.refresh_rate / 1000) * 2;
+} else {
+	// Add a trailing slash to the folder path.
+	args.folder = args.folder.replace(/\/+$/g, '') + '/';
+}
 
 // Small fix for "winston"
 args.log_level = args.log_level.toLowerCase().replace(/trace/, 'silly');

lib/docker.js

@@ -1,9 +1,10 @@
 var dockerCLI = require('docker-cli-js');
 var uuid = require('uuid/v4');
 
-var docker = new dockerCLI.Docker();
 var log = require('./logger.js').get();
 
+var docker = new dockerCLI.Docker();
+
 var handle_error = function(req, res) {
 	return function(err) {
 		res.status(500).json({
@@ -84,7 +85,7 @@ module.exports = {
 		}
 
 		return docker.command('run --name ' + (req.body.name || uuid()) + ports + volumes + remove + detach + ' ' + req.body.image + data).then(function(data) {
-			log.debug('PUT:', data);
+			log.debug('PUT:' + data);
 			res.json(data);
 		}).catch(handle_error(req, res));
 	},

lib/file.js

@@ -1,49 +1,66 @@
 var args = require('./args.js');
+var log = require('./logger.js').get();
 
 var S3 = null;
 var fs = null;
+var redis = null;
 
-if (args.s3) {
+if (args.redis) {
+	var REDIS = require('redis');
+	var redis = REDIS.createClient(args.db_port, args.redis);
+	redis.on('error', log.error);
+} else if (args.s3) {
 	var AWS = require('aws-sdk');
 	S3 = new AWS.S3();
 } else {
 	fs = require('fs');
 
-	// Otherwise, it'll created automatically
 	if (!fs.existsSync(args.folder)){
 		return fs.mkdirSync(args.folder, {recursive: true});
 	}
 }
 
 
 module.exports = {
-	'readFile': function(file, callback) {
-		if (args.s3) {
-			return S3.getObject({Bucket: args.s3, Key: args.folder + file}, function(err, data) {
+	'get': function(file, callback) {
+		if (args.redis) {
+			return redis.get(file, callback);
+		} else if (args.s3) {
+			return S3.getObject({Bucket: args.s3, Key: file}, function(err, data) {
 				return callback(err, !err && Buffer.from(data.Body).toString('utf8'));
 			});
 		}
 		return fs.readFile(file, callback);
 	},
-	'writeFile': function(file, data, callback) {
-		if (args.s3) {
-			return S3.putObject({Bucket: args.s3, Key: args.folder + file, Body: data}, function(err) {
+	'set': function(file, data, callback) {
+		if (args.redis) {
+			return redis.set(file, data, function(err, result) {
+				return redis.expire(file, args.expiry_seconds);
+			});
+		} else if (args.s3) {
+			return S3.putObject({Bucket: args.s3, Key: file, Body: data}, function(err) {
 				return callback(err);
 			});
 		}
 		return fs.writeFile(file, data, callback);
 	},
-	'unlink': function(file, callback) {
-		if (args.s3) {
-			return S3.deleteObject({Bucket: args.s3, Key: args.folder + file}, function(err) {
+	'del': function(file, callback) {
+		if (args.redis) {
+			return redis.expire(file, -1, callback);
+		} else if (args.s3) {
+			return S3.deleteObject({Bucket: args.s3, Key: file}, function(err) {
 				return callback(err);
 			});
 		}
 		return fs.unlink(file, callback);
 	},
-	'readdir': function(dir, callback) {
-		if (args.s3) {
-			return S3.listObjectsV2({Bucket: args.s3, Prefix: args.folder + dir}, function(err, data) {
+	'list': function(dir, callback) {
+		if (args.redis) {
+			return redis.keys(dir + '*', function(err, data) {
+				return callback(err, !err && data.map(function(v) { return v.replace(args.folder, ''); }));
+			});
+		} else if (args.s3) {
+			return S3.listObjectsV2({Bucket: args.s3, Prefix: dir}, function(err, data) {
 				return callback(err, !err && data.Contents.map(function(v) { return v.Key; }));
 			});
 		}

lib/machine.js

@@ -2,8 +2,8 @@ var ping = require('ping');
 var si = require('systeminformation');
 
 var args = require('./args.js');
-var file = require('./file.js');
 var log = require('./logger.js').get();
+var file = require('./file.js');
 
 var last_read = null;
 var cached_machines = [];
@@ -14,8 +14,11 @@ var localize = function(host) {
 
 var readfile = function(filename, hostname) {
 	return new Promise(function(resolve) {
-		file.readFile(args.folder + filename, function(err, data) {
-			var obj = {};
+		file.get(args.folder + filename, function(err, data) {
+			log.debug('file:' + args.folder);
+			log.debug('filename:' + filename);
+
+			var obj = {'filename': filename};
 			data = JSON.parse(data);
 			obj[data.load] = data;
 			if (obj[data.load].host && obj[data.load].host.startsWith(hostname)) {
@@ -32,7 +35,7 @@ var get_all = function(IP) {
 			return resolve(cached_machines);
 		}
 
-		return file.readdir(args.folder, function(err, files) {
+		return file.list(args.folder, function(err, files) {
 			if (err) {
 				log.error(err);
 				return resolve({});
@@ -70,7 +73,7 @@ var get_valid_host = function(ips, IP) {
 			if (is_alive) {
 				return resolve(ip.host);
 			}
-			file.unlink(args.folder + ip.host, function(err) {
+			file.del(args.folder + ip.filename, function(err) {
 				if (err) {
 					log.error('Probably someone else just removed it:' + err);
 				}
@@ -99,7 +102,7 @@ module.exports = {
 			return si.currentLoad(function(cpu_data) {
 				// (used_cpu / total_cpu) * (used_mem / total_mem) = load
 				var load = cpu_data.currentload / 100 * mem_data.used / mem_data.total;
-				return file.writeFile(args.folder + 'ds-' + IP.replace(/[^0-9]/g, '') + '.json', JSON.stringify({
+				return file.set(args.folder + 'ds-' + IP.replace(/[^0-9]/g, '') + '.json', JSON.stringify({
 					'host': IP,
 					'load': load
 				}), log.debug);

package.json

@@ -2,7 +2,7 @@
 	"name": "docker-server",
 	"author": "Evgeny Kolyakov <[email protected]>",
 	"description": "DockerServer - Super lightweight & simple RESTFul stateless server for running docker containers on a remote machine(s) in a secure way",
-	"version": "1.8.9",
+	"version": "1.9.0",
 	"homepage": "https://dockerserver.io",
 	"repository": "freaker2k7/dockerserver",
 	"license": "Apache-2.0",
@@ -32,6 +32,7 @@
 		"express-rate-limit": "^5.0.0",
 		"ip": "^1.1.5",
 		"ping": "^0.2.2",
+		"redis": "^2.8.0",
 		"request": "^2.88.0",
 		"systeminformation": "^4.14.4",
 		"uuid": "^3.3.2",

scripts/dockerfile-version-verify.js

@@ -3,15 +3,11 @@ const fs = require('fs');
 
 if (fs.existsSync('Dockerfile')) {
 	var package = fs.readFileSync('package.json');
-	var package_lock = fs.readFileSync('package-lock.json');
 	var data = fs.readFileSync('Dockerfile');
 
 
 	package = JSON.parse(package);
-	package_lock = JSON.parse(package_lock);
 	data = data.toString().replace(/@\d+\.\d+\.\d+/, '@' + package.version);
-	package_lock.version = package.version;
 
 	fs.writeFileSync('Dockerfile', data);
-	fs.writeFileSync('package-lock.json', JSON.stringify(package_lock, null, "\t") + "\n");
 }