Datasources (#26)

* implemented pre-commit prepareDvcFiles method

* implemented sync and push

* updated to use datasource-utils

* updated to use a published version of @hkube/datasource-utils

* splitted handlers to files, added error handling

* fix commands order

Author: reggev

.gitignore

@@ -69,4 +69,5 @@ hkubectl-*
 helpers/syncthing/syncthing
 output
 testers
-!tests/testData/**
+!tests/testData/**
+tests/repos

README.md

@@ -2,7 +2,7 @@
 
 ## Install  
 ```shell  
-curl -Lo hkubectl https://github.com/kube-HPC/hkubectl/releases/download/$(curl -s https://api.github.com/repos/kube-HPC/hkubectl/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')/hkubectl-linux \
+curl -Lo hkubectl https://github.com/kube-HPC/hkubectl/releases/latest/download/hkubectl-linux \
 && chmod +x hkubectl \
 && sudo mv hkubectl /usr/local/bin/  
 ```  
@@ -21,8 +21,9 @@ Options:
 |--version|Show version number|boolean|||  
 |--rejectUnauthorized|set to false to ignore certificate signing errors. Useful for self signed TLS certificate|boolean|||  
 |--endpoint|url of hkube api endpoint|string|||  
+|--pathPrefix|path prefix url of hkube api endpoint  |string||/hkube/api-server/|  
 |--verbose|verbose logging|boolean|||  
-|--json, -j|output json to stdout|boolean|||  
+|-j, --json|output json to stdout|boolean|||  
 |--help|Show help|boolean|||  
 ### exec  
 ---  
@@ -53,7 +54,7 @@ Options:
 
 |option|description|type|required|default|  
 |---|---|---|---|---|  
-|--file, -f|file path/name for running pipeline. use - to read from stdin|string|true||  
+|-f, --file|file path/name for running pipeline. use - to read from stdin|string|true||  
 |--noWait|if true, does not wait for the execution to finish  |boolean||false|  
 |--noResult|if true, does not show the result of the execution  |boolean||false|  
 #### stored  
@@ -68,7 +69,7 @@ Options:
 |option|description|type|required|default|  
 |---|---|---|---|---|  
 |name|The name of the algorithm|string|||  
-|--file, -f|file path/name for running pipeline|string|||  
+|-f, --file|file path/name for running pipeline|string|||  
 |--noWait|if true, does not wait for the execution to finish  |boolean||false|  
 |--noResult|if true, does not show the result of the execution  |boolean||false|  
 #### stop  
@@ -120,7 +121,7 @@ Options:
 |option|description|type|required|default|  
 |---|---|---|---|---|  
 |name|The name of the algorithm|string|||  
-|--file, -f|file path/name for extra data|string|||  
+|-f, --file|file path/name for extra data|string|||  
 |--noWait|if true, does not wait for the execution to finish  |boolean||false|  
 |--noResult|if true, does not show the result of the execution  |boolean||false|  
 ### algorithm  
@@ -141,16 +142,16 @@ Options:
 |option|description|type|required|default|  
 |---|---|---|---|---|  
 |name|The name of the algorithm|string|||  
-|--file, -f|the algorithm file|string|||  
+|-f, --file|the algorithm file|string|||  
 |--env|the algorithm env  [choices: "python", "nodejs", "java"]|string|||  
 |--codePath|the code path for the algorithm|string|||  
-|--codeEntryPoint, --entryPoint|the code entry point for the algorithm|string|||  
+|--codeEntryPoint, --entryPoint|the code entry point for the algorithm |string|||  
 |--image, --algorithmImage|set algorithm image|string|||  
 |--cpu|CPU requirements of the algorithm in cores |number|||  
 |--gpu|GPU requirements of the algorithm in cores |number|||  
-|--mem|memory requirements of the algorithm. Possible units are ['Ki', 'M', 'Mi', 'Gi', 'm', 'K', 'G', 'T', 'Ti', 'P', 'Pi', 'E', 'Ei']. Minimum is 4Mi|string|||  
-|--noWait|if true, does not wait for the build to finish  |boolean||false|  
-|--setCurrent|if true, sets the new version as the current version|boolean||false|  
+|--mem|memory requirements of the algorithm. Possible units are ['Mi', 'Gi']. Minimum is 4Mi|string|||  
+|--noWait|if true, does not wait for the build to finish|boolean||false|  
+|--setCurrent|if true, sets the new version as the current version  |boolean||false|  
 #### list  
 
 ```shell  
@@ -213,7 +214,7 @@ Options:
 |--overwrite|overwrite an existing folder|boolean|||  
 |--cpu|CPU requirements of the algorithm in cores  |number||0.1|  
 |--gpu|GPU requirements of the algorithm in cores  |number||0|  
-|--mem|memory requirements of the algorithm. Possible units are ['Ki', 'M', 'Mi', 'Gi', 'm', 'K', 'G', 'T', 'Ti', 'P', 'Pi', 'E', 'Ei']. Minimum is 4Mi|string||512Mi|  
+|--mem|memory requirements of the algorithm. Possible units are ['Ki', 'M', 'Mi', 'Gi', 'm', 'K', 'G', 'T', 'Ti', 'P', 'Pi', 'E', 'Ei']. Minimum is 4Mi  |string||512Mi|  
 ### pipeline  
 ---  
 ```shell  
@@ -243,8 +244,8 @@ Options:
 
 |option|description|type|required|default|  
 |---|---|---|---|---|  
-|--file, -f|path for descriptor file|string|true||  
-|--readmeFile|path for readme file. example: --readmeFile="./readme.md |string|||  
+|-f, --file|path for descriptor file|string|true||  
+|--readmeFile|path for readme file. example: --readmeFile="./readme.md|string|||  
 ### sync  
 ---  
 ```shell  
@@ -262,9 +263,9 @@ Options:
 
 |option|description|type|required|default|  
 |---|---|---|---|---|  
-|--algorithmName, -a|The name of the algorithm to sync data into  [required]|string|true||  
-|--folder, -f|local folder to sync.|string||./|  
-|--bidirectional, --bidi|Sync files in both ways|boolean||false|  
+|-a, --algorithmName|The name of the algorithm to sync data into  [required]|string|true||  
+|-f, --folder|local folder to sync.|string||./|  
+|--bidirectional, --bidi|Sync files in both ways |boolean||false|  
 #### create  
 
 ```shell  
@@ -276,10 +277,10 @@ Options:
 
 |option|description|type|required|default|  
 |---|---|---|---|---|  
-|--algorithmName, -a|The name of the algorithm|string|true||  
-|--folder, -f|local folder to build from.|string||./|  
+|-a, --algorithmName|The name of the algorithm|string|true||  
+|-f, --folder|local folder to build from.  |string||./|  
 |--env|algorithm runtime environment  [choices: "python", "nodejs"]|string|||  
-|--entryPoint, -e|the main file of the algorithm|string|||  
+|-e, --entryPoint|the main file of the algorithm|string|||  
 |--baseImage|base image for the algorithm|string|||  
 ### config  
 ---  
@@ -298,4 +299,30 @@ Sets configuration options.
 ```shell  
 $ hkubectl config get   
 ```  
-Gets the current configuration.  
+Gets the current configuration.    
+### datasource  
+---  
+```shell  
+$ hkubectl datasource < command >  
+```  
+Execution pipelines as raw or stored    
+#### sync  
+  
+```shell  
+$ hkubectl datasource sync   
+```  
+should be called after push updates to git, creates a new version entry on the
+datasource service    
+#### prepare  
+  
+```shell  
+$ hkubectl datasource prepare   
+```  
+should be called before commiting to git, scans the directory and updates all
+the.dvc files    
+#### push  
+  
+```shell  
+$ hkubectl datasource push   
+```  
+calls dvc, git push and hkube datasource hkube datasource prepare and sync  

builders/dataSource.js

@@ -0,0 +1,15 @@
+const commands = require('../commands/datasource');
+
+const dataSource = {
+    command: 'datasource <command>',
+    description: 'Execution pipelines as raw or stored',
+    builder: (yargs) => {
+        Object.values(commands).forEach((cmd) => {
+            yargs.command(cmd);
+        });
+        return yargs;
+    },
+    handler: () => { }
+};
+
+module.exports = { dataSource };

commands/datasource/index.js

@@ -0,0 +1,5 @@
+const { sync } = require('./sync');
+const { push } = require('./push');
+const { prepare } = require('./prepare');
+
+module.exports = { sync, prepare, push };

commands/datasource/prepare.js

@@ -0,0 +1,19 @@
+const { parse } = require('path');
+const Repository = require('../../helpers/dataSource/Repository');
+
+const prepare = {
+    command: 'prepare',
+    description: 'should be called before commiting to git, scans the directory and updates all the.dvc files',
+    options: {},
+    builder: {},
+    handler: async () => {
+        const cwd = process.cwd();
+        const { dir, name } = parse(cwd);
+        const hkubeFile = await Repository.readHkubeFile(cwd);
+        const repo = new Repository(hkubeFile.repositoryName, dir, name);
+        await repo.prepareDvcFiles();
+        return null;
+    }
+};
+
+module.exports = { prepare };

commands/datasource/push.js

@@ -0,0 +1,20 @@
+const { parse } = require('path');
+const Repository = require('../../helpers/dataSource/Repository');
+const { handleSync } = require('./sync');
+
+const push = {
+    command: 'push',
+    description: 'calls dvc, git push and hkube datasource hkube datasource prepare and sync',
+    options: {},
+    builder: {},
+    handler: async () => {
+        const cwd = process.cwd();
+        const hkubeFile = await Repository.readHkubeFile(cwd);
+        const { dir } = parse(cwd);
+        const repo = new Repository(hkubeFile.repositoryName, dir);
+        await repo.push();
+        return handleSync();
+    }
+};
+
+module.exports = { push };

commands/datasource/sync.js

@@ -0,0 +1,40 @@
+const formatSize = require('pretty-bytes');
+const get = require('lodash.get');
+const chalk = require('chalk');
+const Repository = require('../../helpers/dataSource/Repository');
+const { api: client } = require('../../helpers/clients');
+const { log } = require('../../helpers/output');
+
+const handleSync = async () => {
+    const cwd = process.cwd();
+    const hkubeFile = await Repository.readHkubeFile(cwd);
+    let response = null;
+    try {
+        response = await client.post(
+            `datasource/${hkubeFile.repositoryName}/sync`
+        );
+    }
+    catch (e) {
+        e.code && log(chalk.red`sync failed! ${e.code}`);
+        const message = get(e, 'response.data.error.message');
+        message && log(chalk.red(message));
+        return null;
+    }
+    const { avgFileSize, totalSize } = response.data;
+    log({
+        ...response.data,
+        avgFileSize: formatSize(avgFileSize),
+        totalSize: formatSize(totalSize)
+    });
+    return null;
+};
+
+const sync = {
+    command: 'sync',
+    description: 'should be called after push updates to git, creates a new version entry on the datasource service',
+    options: {},
+    builder: {},
+    handler: handleSync
+};
+
+module.exports = { handleSync, sync };

helpers/clients.js

@@ -0,0 +1,19 @@
+const { default: axios } = require('axios');
+const pathLib = require('path');
+const https = require('https');
+
+const api = axios.create();
+
+const setupClient = (argv) => {
+    const apiPrefix = 'api/v1';
+    const { endpoint, pathPrefix, rejectUnauthorized } = argv || {};
+    const tmp = pathLib.join(pathPrefix, apiPrefix);
+    const baseURL = new URL(tmp, endpoint);
+    api.defaults.baseURL = baseURL.toString();
+    if (baseURL.protocol === 'https:') {
+        const agent = new https.Agent({ rejectUnauthorized });
+        api.defaults.httpAgent = agent;
+    }
+};
+
+module.exports = { api, setupClient };

helpers/dataSource/Repository.js

@@ -0,0 +1,50 @@
+const mimeTypes = require('mime-types');
+const { Repository: _Repository, createFileMeta, glob } = require('@hkube/datasource-utils');
+const fse = require('fs-extra');
+const get = require('lodash.get');
+const { default: simpleGit } = require('simple-git');
+/**
+ * @typedef {import('@hkube/datasource-utils').LocalFileMeta} LocalFileMeta
+ */
+class Repository extends _Repository {
+    /**
+     * @param {string} repositoryName
+     * @param {string} rootDir
+     */
+    constructor(repositoryName, rootDir, dirName) {
+        super(repositoryName, rootDir, dirName);
+        this.dirName = dirName || repositoryName;
+        this.gitClient = simpleGit({ baseDir: this.cwd });
+    }
+
+    _createFileMeta(currentMeta) {
+        const { size, path: name } = currentMeta.outs[0];
+        const mimetype = mimeTypes.lookup(name) || '';
+        return createFileMeta({
+            originalname: name,
+            size,
+            mimetype
+        }, null, 'cli');
+    }
+
+    async prepareDvcFiles() {
+        const dvcFiles = await glob('**/*.dvc', this.cwd);
+        for await (const dvcPath of dvcFiles) {
+            const dvcContent = await this.dvc.loadDvcContent(dvcPath, true);
+            /** @type {LocalFileMeta} */
+            let fileMeta = get(dvcContent, 'meta.hkube', {});
+            if (dvcContent.outs[0].md5 !== get(dvcContent, 'meta.hkube.hash', null)) {
+                const { path, ...generatedMeta } = this._createFileMeta(dvcContent);
+                fileMeta = generatedMeta;
+            }
+            const metaFilePath = dvcPath.replace('.dvc', '.meta');
+            if (await fse.pathExists(metaFilePath)) {
+                const content = await fse.readFile(`${this.cwd}/${metaFilePath}`);
+                fileMeta.meta = content.toString('utf8');
+            }
+            await this.dvc.enrichMeta(dvcPath, dvcContent, 'hkube', fileMeta);
+        }
+    }
+}
+
+module.exports = Repository;

hkubectl.js

@@ -7,6 +7,8 @@ const algorithms = require('./builders/algorithm');
 const pipelines = require('./builders/pipeline');
 const dryRun = require('./builders/dry-run');
 const sync = require('./builders/sync');
+const { dataSource } = require('./builders/dataSource');
+const { setupClient } = require('./helpers/clients');
 const syncthing = require('./helpers/syncthing/syncthing.js');
 
 global.args = {};
@@ -28,6 +30,7 @@ const main = async () => {
     yargs.command(dryRun);
     yargs.command(sync);
     yargs.command(config);
+    yargs.command(dataSource);
     yargs.options('rejectUnauthorized', {
         description: 'set to false to ignore certificate signing errors. Useful for self signed TLS certificate',
         type: 'boolean'
@@ -55,8 +58,10 @@ const main = async () => {
         .help()
         .epilog(chalk.bold('for more information visit http://hkube.io'))
         .completion();
+
     yargs.middleware((args) => {
         global.args = args;
+        setupClient(global.args);
     });
     const args = yargs.argv;
     if (!args._[0]) {