Skip to content

Commit 800235f

Browse files
yehiyamyehiyam
committed
add sync create
add exec algorithm refactor for axios add sync watch events
1 parent db16c7c commit 800235f

File tree

16 files changed

+811
-180
lines changed

16 files changed

+811
-180
lines changed

‎commands/exec/algorithm.js

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
const yaml = require('js-yaml');
2+
const prettyjson = require('prettyjson');
3+
const fse = require('fs-extra');
4+
const { post } = require('../../helpers/request-helper');
5+
const path = `exec/algorithm/`;
6+
7+
const executeHandler = async ({ endpoint, rejectUnauthorized, name, file }) => {
8+
let result;
9+
10+
if (file) {
11+
result = yaml.safeLoad(fse.readFileSync(file, 'utf8'));
12+
}
13+
const body = {
14+
name, ...result
15+
}
16+
return post({
17+
endpoint,
18+
rejectUnauthorized,
19+
path,
20+
body
21+
});
22+
}
23+
24+
module.exports = {
25+
command: 'algorithm [name]',
26+
alias: ['e'],
27+
description: 'execute algorithm',
28+
options: {
29+
},
30+
builder: {
31+
'file': {
32+
demandOption: false,
33+
describe: 'file path/name for extra data',
34+
type: 'string',
35+
alias: ['f']
36+
}
37+
},
38+
handler: async (argv) => {
39+
const ret = await executeHandler(argv);
40+
console.log(prettyjson.render(ret));
41+
}
42+
}

‎commands/exec/index.js

+3-1
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@ const stored = require('./stored');
44
const stop = require('./stop');
55
const status = require('./status');
66
const result = require('./result');
7+
const algorithm = require('./algorithm');
78

89
module.exports = {
910
get,
1011
raw,
1112
stored,
1213
stop,
1314
status,
14-
result
15+
result,
16+
algorithm
1517
};

‎commands/readme/algorithms/add.js

+7-7
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
11
const prettyjson = require("prettyjson");
22
const fse = require("fs-extra");
3+
var FormData = require('form-data');
34
const { post, postFile } = require("../../helpers/request-helper");
45

56
const handleAdd = async ({ endpoint, rejectUnauthorized, name, readmeFile }) => {
67
const path = `readme/algorithms/${name}`;
78
let stream = fse.createReadStream(readmeFile);
8-
const formData = {
9-
"README.md": {
10-
value: stream,
11-
options: {
12-
filename: "README.md"
13-
}
9+
const formData = new FormData();
10+
formData.append('README.md', {
11+
value: stream,
12+
options: {
13+
filename: "README.md"
1414
}
15-
};
15+
})
1616
const result = await postFile({
1717
endpoint,
1818
rejectUnauthorized,

‎commands/store/algorithms/apply.js

+1-76
Original file line numberDiff line numberDiff line change
@@ -1,80 +1,5 @@
1-
const os = require('os');
2-
const fse = require('fs-extra');
3-
const yaml = require('js-yaml');
4-
const merge = require('lodash.merge');
51
const prettyjson = require('prettyjson');
6-
const { postFile } = require('../../../helpers/request-helper');
7-
const applyPath = 'store/algorithms/apply';
8-
9-
const handleApply = async ({ endpoint, rejectUnauthorized, name, file, ...cli }) => {
10-
let result, error;
11-
try {
12-
let stream, fileData;
13-
if (file) {
14-
const fileContent = readFile(file);
15-
if (fileContent.error) {
16-
throw new Error(fileContent.error);
17-
}
18-
fileData = adaptFileData(fileContent.result);
19-
}
20-
21-
const cliData = adaptCliData(cli);
22-
const algorithmData = merge(fileData, cliData, { name });
23-
24-
const { code, ...algorithm } = algorithmData;
25-
26-
const body = {
27-
...algorithm,
28-
entryPoint: code.entryPoint,
29-
userInfo: {
30-
platform: os.platform(),
31-
hostname: os.hostname(),
32-
username: os.userInfo().username
33-
}
34-
};
35-
36-
if (code.path) {
37-
stream = fse.createReadStream(code.path);
38-
}
39-
40-
const formData = {
41-
payload: JSON.stringify(body),
42-
file: stream || ''
43-
};
44-
result = await postFile({
45-
endpoint,
46-
rejectUnauthorized,
47-
formData,
48-
path: applyPath
49-
});
50-
}
51-
catch (e) {
52-
error = e.message;
53-
}
54-
return { error, result };
55-
};
56-
57-
const readFile = (file) => {
58-
let result, error;
59-
try {
60-
result = yaml.safeLoad(fse.readFileSync(file, 'utf8'));
61-
}
62-
catch (e) {
63-
error = e.message;
64-
}
65-
return { error, result };
66-
};
67-
68-
const adaptFileData = (fileData) => {
69-
const { name, env, image, version, code, resources, algorithmEnv, workerEnv, minHotWorkers, nodeSelector, baseImage, options } = fileData || {};
70-
const { cpu, gpu, mem } = resources || {};
71-
return { name, env, code: code || {}, version, algorithmImage: image, baseImage, options, cpu, gpu, mem, algorithmEnv, workerEnv, minHotWorkers, nodeSelector };
72-
};
73-
74-
const adaptCliData = (cliData) => {
75-
const { env, image, ver, cpu, gpu, mem, algorithmEnv, workerEnv, codePath, codeEntryPoint, baseImage, options } = cliData || {};
76-
return { env, algorithmImage: image, baseImage, version: ver, cpu, gpu, mem, algorithmEnv, workerEnv, options, code: { path: codePath, entryPoint: codeEntryPoint } };
77-
};
2+
const {handleApply} = require('./applyImpl');
783

794
module.exports = {
805
command: 'apply [name]',

‎commands/store/algorithms/applyImpl.js

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
const os = require('os');
2+
const path = require('path');
3+
const fse = require('fs-extra');
4+
const yaml = require('js-yaml');
5+
const merge = require('lodash.merge');
6+
var FormData = require('form-data');
7+
const { postFile } = require('../../../helpers/request-helper');
8+
const applyPath = 'store/algorithms/apply';
9+
10+
11+
const handleApply = async ({ endpoint, rejectUnauthorized, name, file, ...cli }) => {
12+
let result, error;
13+
try {
14+
let stream, fileData;
15+
if (file) {
16+
const fileContent = readFile(file);
17+
if (fileContent.error) {
18+
throw new Error(fileContent.error);
19+
}
20+
fileData = adaptFileData(fileContent.result);
21+
}
22+
23+
const cliData = adaptCliData(cli);
24+
const algorithmData = merge(fileData, cliData, { name });
25+
26+
const { code, ...algorithm } = algorithmData;
27+
28+
const body = {
29+
...algorithm,
30+
entryPoint: code.entryPoint,
31+
userInfo: {
32+
platform: os.platform(),
33+
hostname: os.hostname(),
34+
username: os.userInfo().username
35+
}
36+
};
37+
38+
if (code.path) {
39+
const codePath = path.resolve(
40+
file?path.dirname(file):process.cwd(),
41+
code.path);
42+
stream = fse.createReadStream(codePath);
43+
}
44+
45+
const formData = new FormData()
46+
formData.append('payload',JSON.stringify(body))
47+
formData.append('file', stream || '')
48+
49+
result = await postFile({
50+
endpoint,
51+
rejectUnauthorized,
52+
formData,
53+
path: applyPath
54+
});
55+
}
56+
catch (e) {
57+
error = e.message;
58+
}
59+
return { error, result };
60+
};
61+
62+
const readFile = (file) => {
63+
let result, error;
64+
try {
65+
result = yaml.safeLoad(fse.readFileSync(file, 'utf8'));
66+
}
67+
catch (e) {
68+
error = e.message;
69+
}
70+
return { error, result };
71+
};
72+
73+
const adaptFileData = (fileData) => {
74+
const { name, env, image, version, code, resources, algorithmEnv, workerEnv, minHotWorkers, nodeSelector, baseImage, options } = fileData || {};
75+
const { cpu, gpu, mem } = resources || {};
76+
return { name, env, code: code || {}, version, algorithmImage: image, baseImage, options, cpu, gpu, mem, algorithmEnv, workerEnv, minHotWorkers, nodeSelector };
77+
};
78+
79+
const adaptCliData = (cliData) => {
80+
const { env, image, ver, cpu, gpu, mem, algorithmEnv, workerEnv, codePath, codeEntryPoint, baseImage, options } = cliData || {};
81+
return { env, algorithmImage: image, baseImage, version: ver, cpu, gpu, mem, algorithmEnv, workerEnv, options, code: { path: codePath, entryPoint: codeEntryPoint } };
82+
};
83+
84+
module.exports = {
85+
handleApply
86+
}

‎commands/store/pipelines/store.js

+7-7
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
const prettyjson = require('prettyjson');
22
const fse = require('fs-extra');
33
const yaml = require('js-yaml');
4+
var FormData = require('form-data');
45
const { post, postFile } = require('../../../helpers/request-helper');
56
const path = 'store/pipelines';
67

@@ -24,14 +25,13 @@ const handleAdd = async ({ endpoint, rejectUnauthorized, file, readmeFile }) =>
2425
const readmeAdd = async (readmeFile, endpoint, rejectUnauthorized, name) => {
2526
const path = `readme/pipelines/${name}`;
2627
let stream = fse.createReadStream(readmeFile);
27-
const formData = {
28-
"README.md": {
29-
value: stream,
30-
options: {
31-
filename: "README.md"
32-
}
28+
const formData = new FormData();
29+
formData.append('README.md', {
30+
value: stream,
31+
options: {
32+
filename: "README.md"
3333
}
34-
};
34+
})
3535
await postFile({
3636
endpoint,
3737
rejectUnauthorized,

‎commands/sync/create.js

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
const path = require('path');
2+
const prettyjson = require('prettyjson');
3+
const socketTunnelClient = require('../../helpers/tcpTunnel/client');
4+
const agentSyncIngressPath = '/hkube/sync/sync'
5+
const agentRestIngressPath = '/hkube/sync/ui'
6+
const syncthing = require('../../helpers/syncthing/syncthing.js');
7+
const { events } = require('../../helpers/syncthing/consts');
8+
const { zipDirectory } = require('../../helpers/zipper');
9+
const { handleApply } = require('../store/algorithms/applyImpl');
10+
const { getUntil, post, get } = require('../../helpers/request-helper');
11+
12+
const createHandler = async ({ endpoint, rejectUnauthorized, algorithmName, folder, entryPoint, env, baseImage }) => {
13+
try {
14+
const fullFolderPath = path.resolve(folder);
15+
console.log(`Creating zip of ${fullFolderPath}`)
16+
const zipFileName = await zipDirectory(fullFolderPath, { tmpFile: true });
17+
console.log(`Requesting build for algorithm ${algorithmName}`);
18+
const algorithmData = {
19+
codePath: zipFileName,
20+
options: {
21+
devMode: true
22+
},
23+
codeEntryPoint: entryPoint,
24+
env,
25+
baseImage
26+
}
27+
const applyRes = await handleApply({
28+
endpoint, rejectUnauthorized, name: algorithmName, zipFileName, ...algorithmData
29+
})
30+
const error = applyRes.error || applyRes.result.error;
31+
if (error) {
32+
console.error(error.message || error);
33+
return;
34+
}
35+
console.log(applyRes.result.result.messages.join('\n'))
36+
const buildId = applyRes.result.result.buildId;
37+
if (buildId) {
38+
// wait for build
39+
console.log(`build ${buildId} in progress.`);
40+
let lastStatus = '';
41+
const buildResult = await getUntil({ endpoint, rejectUnauthorized, path: `builds/status/${buildId}` }, (res) => {
42+
if (lastStatus !== res.result.status) {
43+
console.log(res.result.status)
44+
}
45+
lastStatus = res.result.status;
46+
return (res.result.status === 'completed')
47+
}, 1000 * 60 * 10)
48+
const { algorithmImage, version } = buildResult.result
49+
console.log(`Setting version ${version} as current`);
50+
const setVersionRes = await post({
51+
endpoint, rejectUnauthorized, path: `versions/algorithms/apply`, body: {
52+
name: algorithmName,
53+
image: algorithmImage,
54+
force: true
55+
}
56+
})
57+
}
58+
console.log(`algorithm ${algorithmName} is ready`)
59+
console.log('to sync the folder to the algorithm run')
60+
console.log(`hkubectl sync watch -a ${algorithmName} -f ${folder}`)
61+
62+
} catch (error) {
63+
console.error(`error connecting to cluster. Error: ${error.message}`)
64+
}
65+
}
66+
module.exports = {
67+
command: 'create',
68+
description: 'creates the algorithm for development.',
69+
options: {
70+
},
71+
builder: {
72+
algorithmName: {
73+
demandOption: true,
74+
describe: 'The name of the algorithm',
75+
type: 'string',
76+
alias: ['a']
77+
},
78+
folder: {
79+
demandOption: false,
80+
describe: 'local folder to build from.',
81+
default: './',
82+
type: 'string',
83+
alias: ['f']
84+
},
85+
env: {
86+
describe: 'algorithm runtime environment',
87+
type: 'string',
88+
choices: ['python', 'nodejs']
89+
90+
},
91+
entryPoint: {
92+
describe: 'the main file of the algorithm',
93+
type: 'string',
94+
demandOption: true,
95+
alias: ['e']
96+
},
97+
baseImage: {
98+
describe: 'base image for the algorithm',
99+
type: 'string'
100+
}
101+
},
102+
handler: async (argv) => {
103+
const ret = await createHandler(argv)
104+
}
105+
}

0 commit comments

Comments
 (0)