Skip to content

Commit 803cb9b

Browse files
Merge pull request #17 from modzy/feat/open_jobs
Supporting big files
2 parents 7c54020 + f2f88af commit 803cb9b

10 files changed

+568
-174
lines changed

package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@modzy/modzy-sdk",
3-
"version": "1.0.6",
3+
"version": "1.0.7",
44
"description": "Javascript SDK for Modzy",
55
"license": "Booz Allen Public License v1.0",
66
"main": "dist/main.js",
@@ -27,6 +27,7 @@
2727
"dependencies": {
2828
"axios": "^0.21.1",
2929
"dotenv": "^8.2.0",
30+
"form-data": "^4.0.0",
3031
"log4js": "^6.1.2",
3132
"string-similarity": "^4.0.4"
3233
},

samples/job_with_aws_input_sample.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ async function createJobWithAWSInput() {
7575
// process and you can identify and refer to a specific input by the key assigned. For example we can add:
7676
sources["second-key"] = { "image": { 'bucket': BUCKET_NAME, 'key': FILE_KEY } };
7777
sources["another-key"] = { "image": { 'bucket': BUCKET_NAME, 'key': FILE_KEY } };
78-
// If you send a wrong input key, the model fails to process the input.
78+
// If you send an incorrect input key, the model fails to process the input.
7979
sources["wrong-key"] = { "a.wrong.key": { 'bucket': BUCKET_NAME, 'key': FILE_KEY } };
8080
// If you send a correct input key, but a wrong AWS S3 value key, the model fails to process the input.
8181
sources["wrong-value"] = { "image": { 'bucket': BUCKET_NAME, 'key': "wrong-aws-file-key.png" } };

samples/job_with_embedded_input_sample.js

+9-9
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ logger.level = "info";
1111
// The MODZY_BASE_URL should point to the API services route which may be different from the Modzy page URL.
1212
// (ie: https://modzy.example.com/api).
1313
const BASE_URL = process.env.MODZY_BASE_URL;
14-
// The MODZY_API_KEY is your own personal API key. It is composed by a public part, a dot character and a private part
14+
// The MODZY_API_KEY is your own personal API key. It is composed by a public part, a dot character, and a private part
1515
// (ie: AzQBJ3h4B1z60xNmhAJF.uQyQh8putLIRDi1nOldh).
1616
const API_KEY = process.env.MODZY_API_KEY;
1717

@@ -26,7 +26,7 @@ async function createJobWithEmbeddedInput(){
2626
try {
2727
// Get the model object:
2828
// If you already know the model identifier (i.e.: you got from the URL of the model details page or the input sample),
29-
// you can skip this step. If you don't you can find the model identifier by using its name as follows:
29+
// you can skip this step. If you don't, you can find the model identifier by using its name as follows:
3030
let model = await modzyClient.getModelByName("Multi-Language OCR");
3131
// Or if you already know the model id and want to know more about the model, you can use this instead:
3232
//let model = await modzyClient.getModel("c60c8dbd79");
@@ -37,7 +37,7 @@ async function createJobWithEmbeddedInput(){
3737
logger.info(`The model identifier is ${model.modelId} and the latest version is ${model.latestVersion}`);
3838
// Get the model version object:
3939
// If you already know the model version and the input key(s) of the model version you can skip this step. Also, you can
40-
// use the following code block to know about the inputs keys and skip the call on future job submissions.
40+
// use the following code block to know about the input keys and skip the call on future job submissions.
4141
let modelVersion = await modzyClient.getModelVersion(model.modelId, model.latestVersion);
4242
// The info stored in modelVersion provides insights about the amount of time that the model can spend processing, the inputs, and
4343
// output keys of the model.
@@ -56,7 +56,7 @@ async function createJobWithEmbeddedInput(){
5656

5757
// Send the job:
5858
// An embedded input is a byte array encoded as a string in Base64, that's very handy for small to middle size files, for
59-
// bigger files can be a memory issue because you need to load the file in memory (load + encode).
59+
// bigger files can cause memory issues because you need to load the file in the memory (load + encode).
6060
const imageBytes = fs.readFileSync('samples/image.png');
6161
let configBytes = fs.readFileSync('samples/config.json');
6262
// With the info about the model (identifier), the model version (version string, input/output keys), you are ready to
@@ -65,10 +65,10 @@ async function createJobWithEmbeddedInput(){
6565
// An inference job groups input data that you send to a model. You can send any amount of inputs to
6666
// process and you can identify and refer to a specific input by the key that you assign, for example we can add:
6767
sources["second-key"] = {"input": imageBytes, "config.json":configBytes}
68-
// You don't need to load all the inputs from files, just convert to bytes as follows:
68+
// You dont need to load all the inputs from the files, just convert to bytes as follows:
6969
configBytes = Buffer.from(JSON.stringify({"languages":["spa"]}));
7070
sources["another-key"] = {"input": imageBytes, "config.json":configBytes}
71-
// If you send a wrong input key, the model fails to process the input.
71+
// If you send an incorrect input key, the model fails to process the input.
7272
sources["wrong-key"] = {"a.wrong.key": imageBytes, "config.json":configBytes}
7373
// If you send a correct input key, but some wrong values, the model fails to process the input.
7474
sources["wrong-value"] = {"input": configBytes, "config.json":imageBytes}
@@ -78,20 +78,20 @@ async function createJobWithEmbeddedInput(){
7878
// of the process, the most important being the job identifier and the job status.
7979
logger.info("job: "+job.jobIdentifier+" "+job.status);
8080
// The job moves to SUBMITTED, meaning that Modzy acknowledged the job and sent it to the queue to be processed.
81-
// We provide a helper method to listen until the job finishes processing. it will listen until the job finishes
81+
// We provide a helper method to listen until the job finishes processing. It will listen until the job finishes
8282
// and moves to COMPLETED, CANCELED, or TIMEOUT.
8383
job = await modzyClient.blockUntilComplete(job);
8484
// Get the results:
8585
// Check the status of the job. Jobs may be canceled or may reach a timeout.
8686
if( job.status === "COMPLETED" ){
8787
// A completed job means that all the inputs were processed by the model. Check the results for each
88-
// input keys provided in the source object to see the model output.
88+
// input key provided in the source object to see the model output.
8989
let result = await modzyClient.getResult(job.jobIdentifier);
9090
// The result object has some useful info:
9191
logger.info(`Result: finished: ${result.finished}, total: ${result.total}, completed: ${result.completed}, failed: ${result.failed}`);
9292
// Notice that we are iterating through the same input sources keys
9393
for( key in sources ){
94-
// The result object has the individual results of each job input. In this case the output key is called
94+
// The result object has the individual results of each job input. In this case, the output key is called
9595
// results.json, so we can get the results as follows:
9696
if( result.results[key] ){
9797
let model_res = result.results[key]["results.json"];

samples/job_with_file_input_sample.js

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
const logger = require('log4js').getLogger("modzy");
2+
const modzy = require('modzy-sdk');
3+
const fs = require('fs');
4+
5+
// Always configure the logger level (ie: all, trace, debug, info, warn, error, fatal)
6+
logger.level = "info";
7+
8+
// The system admin can provide the right base API URL, the API key can be downloaded from your profile page on Modzy.
9+
// You can config those params as is described in the readme file (as environment variables, or by using the .env file), or you
10+
// or you can just update the BASE_URL and API_KEY vars and use this sample code (not recommended for production environments).
11+
// The MODZY_BASE_URL should point to the API services route which may be different from the Modzy page URL.
12+
// (ie: https://modzy.example.com/api).
13+
const BASE_URL = process.env.MODZY_BASE_URL;
14+
// The MODZY_API_KEY is your own personal API key. It is composed by a public part, a dot character, and a private part
15+
// (ie: AzQBJ3h4B1z60xNmhAJF.uQyQh8putLIRDi1nOldh).
16+
const API_KEY = process.env.MODZY_API_KEY;
17+
18+
// Client initialization
19+
// Initialize the ApiClient instance with the BASE_URL and the API_KEY to store those arguments
20+
// for the following API calls.
21+
const modzyClient = new modzy.ModzyClient(BASE_URL, API_KEY);
22+
23+
// Create a Job with an embedded input, wait, and retrieve results:
24+
25+
async function createJobWithFileInput(){
26+
try {
27+
// Get the model object:
28+
// If you already know the model identifier (i.e.: you got from the URL of the model details page or the input sample),
29+
// you can skip this step. If you don't, you can find the model identifier by using its name as follows:
30+
let model = await modzyClient.getModelByName("Multi-Language OCR");
31+
// Or if you already know the model id and want to know more about the model, you can use this instead:
32+
//let model = await modzyClient.getModel("c60c8dbd79");
33+
34+
// The model identifier is under the modelId key. You can take a look at the other keys by uncommenting the following line
35+
logger.info(Object.keys(model).toString().replace('\n', ' '));
36+
// Or just log the model identifier and the latest version
37+
logger.info(`The model identifier is ${model.modelId} and the latest version is ${model.latestVersion}`);
38+
// Get the model version object:
39+
// If you already know the model version and the input key(s) of the model version you can skip this step. Also, you can
40+
// use the following code block to know about the input keys and skip the call on future job submissions.
41+
let modelVersion = await modzyClient.getModelVersion(model.modelId, model.latestVersion);
42+
// The info stored in modelVersion provides insights about the amount of time that the model can spend processing, the inputs, and
43+
// output keys of the model.
44+
logger.info(`Ths model version is ${modelVersion.version}`);
45+
logger.info(` timeouts: status ${modelVersion.timeout.status}ms, run ${modelVersion.timeout.run}ms `);
46+
logger.info(" inputs: ");
47+
for(key in modelVersion.inputs){
48+
let input = modelVersion.inputs[key];
49+
logger.info(` key ${input.name}, type ${input.acceptedMediaTypes}, description: ${input.description}`);
50+
}
51+
logger.info(" outputs: ")
52+
for(key in modelVersion.outputs){
53+
let output = modelVersion.outputs[key];
54+
logger.info(` key ${output.name}, type ${output.mediaType}, description: ${output.description}`);
55+
}
56+
// Send the job:
57+
// A file input can be a byte array or any file path. This input type fits files of any size.
58+
const imagePath = 'samples/image.png';
59+
const configPath = 'samples/config.json';
60+
// With the info about the model (identifier), the model version (version string, input/output keys), you are ready to
61+
// submit the job. Just prepare the source object:
62+
let sources = {"source-key": {"input": imagePath, "config.json": configPath}};
63+
// An inference job groups input data that you send to a model. You can send any amount of inputs to
64+
// process and you can identify and refer to a specific input by the key that you assign, for example we can add:
65+
sources["second-key"] = {"input": imagePath, "config.json": configPath}
66+
// You don’t need to load all the inputs from the files, just convert to bytes as follows:
67+
const configBytes = Buffer.from(JSON.stringify({"languages":["spa"]}));
68+
sources["another-key"] = {"input": imagePath, "config.json":configBytes}
69+
// If you send an incorrect input key, the model fails to process the input.
70+
sources["wrong-key"] = {"a.wrong.key": imagePath, "config.json":configPath}
71+
// If you send a correct input key, but some wrong values, the model fails to process the input.
72+
sources["wrong-value"] = {"input": configPath, "config.json":imagePath}
73+
// When you have all your inputs ready, you can use our helper method to submit the job as follows:
74+
let job = await modzyClient.submitJobFiles(model.modelId,modelVersion.version, sources);
75+
// Modzy creates the job and queue for processing. The job object contains all the info that you need to keep track
76+
// of the process, the most important being the job identifier and the job status.
77+
logger.info("job: "+job.jobIdentifier+" "+job.status);
78+
// The job moves to SUBMITTED, meaning that Modzy acknowledged the job and sent it to the queue to be processed.
79+
// We provide a helper method to listen until the job finishes processing. It will listen until the job finishes
80+
// and moves to COMPLETED, CANCELED, or TIMEOUT.
81+
job = await modzyClient.blockUntilComplete(job);
82+
// Get the results:
83+
// Check the status of the job. Jobs may be canceled or may reach a timeout.
84+
if( job.status === "COMPLETED" ){
85+
// A completed job means that all the inputs were processed by the model. Check the results for each
86+
// input key provided in the source object to see the model output.
87+
let result = await modzyClient.getResult(job.jobIdentifier);
88+
// The result object has some useful info:
89+
logger.info(`Result: finished: ${result.finished}, total: ${result.total}, completed: ${result.completed}, failed: ${result.failed}`);
90+
// Notice that we are iterating through the same input sources keys
91+
for( key in sources ){
92+
// The result object has the individual results of each job input. In this case, the output key is called
93+
// results.json, so we can get the results as follows:
94+
if( result.results[key] ){
95+
let model_res = result.results[key]["results.json"];
96+
// The output for this model comes in a JSON format, so we can directly log the model results:
97+
logger.info(` ${key}: ${JSON.stringify(model_res)}`);
98+
}
99+
else{
100+
logger.warn(` ${key}: failure ${result.failures[key]['error']}`);
101+
}
102+
}
103+
}
104+
else{
105+
log.warn(`The job ends with status ${job.status}`);
106+
}
107+
}
108+
catch(error){
109+
logger.warn(error);
110+
}
111+
}
112+
113+
114+
createJobWithFileInput();

samples/job_with_text_input_sample.js

+7-7
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ logger.level = "info";
1010
// The MODZY_BASE_URL should point to the API services route which may be different from the Modzy page URL.
1111
// (ie: https://modzy.example.com/api).
1212
const BASE_URL = process.env.MODZY_BASE_URL;
13-
// The MODZY_API_KEY is your own personal API key. It is composed by a public part, a dot character and a private part
13+
// The MODZY_API_KEY is your own personal API key. It is composed by a public part, a dot character, and a private part
1414
// (ie: AzQBJ3h4B1z60xNmhAJF.uQyQh8putLIRDi1nOldh).
1515
const API_KEY = process.env.MODZY_API_KEY;
1616

@@ -25,7 +25,7 @@ async function createJobWithTextInput(){
2525
try{
2626
// Get the model object:
2727
// If you already know the model identifier (i.e.: you got from the URL of the model details page or the input sample),
28-
// you can skip this step. If you don't you can find the model identifier by using its name as follows:
28+
// you can skip this step. If you don't, you can find the model identifier by using its name as follows:
2929
let model = await modzyClient.getModelByName("Sentiment Analysis");
3030
// Or if you already know the model id and want to know more about the model, you can use this instead:
3131
//let model = await modzyClient.getModel("ed542963de");
@@ -36,7 +36,7 @@ async function createJobWithTextInput(){
3636
logger.info(`The model identifier is ${model.modelId} and the latest version is ${model.latestVersion}`);
3737
// Get the model version object:
3838
// If you already know the model version and the input key(s) of the model version you can skip this step. Also, you can
39-
// use the following code block to know about the inputs keys and skip the call on future job submissions.
39+
// use the following code block to know about the input keys and skip the call on future job submissions.
4040
let modelVersion = await modzyClient.getModelVersion(model.modelId, model.latestVersion);
4141
// The info stored in modelVersion provides insights about the amount of time that the model can spend processing, the inputs, and
4242
// output keys of the model.
@@ -61,28 +61,28 @@ async function createJobWithTextInput(){
6161
// process and you can identify and refer to a specific input by the key that you assign, for example we can add:
6262
sources["second-key"] = {"input.txt": "Sometimes I really hate ribs"}
6363
sources["another-key"] = {"input.txt": "Born and raised in Pennsylvania, Swift moved to Nashville, Tennessee, at the age of 14 to pursue a career in country music"}
64-
// If you send a wrong input key, the model fails to process the input.
64+
// If you send an incorrect input key, the model fails to process the input.
6565
sources["wrong-key"] = {"a.wrong.key": "This input is wrong!"}
6666
// When you have all your inputs ready, you can use our helper method to submit the job as follows:
6767
let job = await modzyClient.submitJobText(model.modelId,modelVersion.version, sources);
6868
// Modzy creates the job and queue for processing. The job object contains all the info that you need to keep track
6969
// of the process, the most important being the job identifier and the job status.
7070
logger.info("job: "+job.jobIdentifier+" "+job.status);
7171
// The job moves to SUBMITTED, meaning that Modzy acknowledged the job and sent it to the queue to be processed.
72-
// We provide a helper method to listen until the job finishes processing. it will listen until the job finishes
72+
// We provide a helper method to listen until the job finishes processing. It will listen until the job finishes
7373
// and moves to COMPLETED, CANCELED, or TIMEOUT.
7474
job = await modzyClient.blockUntilComplete(job);
7575
// Get the results:
7676
// Check the status of the job. Jobs may be canceled or may reach a timeout.
7777
if( job.status === "COMPLETED" ){
7878
// A completed job means that all the inputs were processed by the model. Check the results for each
79-
// input keys provided in the source object to see the model output.
79+
// input key provided in the source object to see the model output.
8080
let result = await modzyClient.getResult(job.jobIdentifier);
8181
// The result object has some useful info:
8282
logger.info(`Result: finished: ${result.finished}, total: ${result.total}, completed: ${result.completed}, failed: ${result.failed}`);
8383
// Notice that we are iterating through the same input sources keys
8484
for( key in sources ){
85-
// The result object has the individual results of each job input. In this case the output key is called
85+
// The result object has the individual results of each job input. In this case, the output key is called
8686
// results.json, so we can get the results as follows:
8787
if( result.results[key] ){
8888
let model_res = result.results[key]["results.json"];

0 commit comments

Comments
 (0)