Skip to content

Commit a5b2d91

Browse files
luancazarinecoderabbitai[bot]
luancazarine
and
coderabbitai[bot]
authored
New Components - vectorshift (#16023)
* vectorshift init * [Components] vectorshift #16018 Sources - New Pipeline - New Knowledge Base - New Chatbot Actions - Create Pipeline - Run Pipeline - Add Data To Knowledge Base * pnpm update * pnpm update * Update components/vectorshift/common/constants.mjs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * Update components/vectorshift/sources/common/base.mjs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * fix import * [Components] vectorshift #16018 Sources - New Pipeline - New Knowledge Base - New Chatbot Actions - Create Pipeline - Run Pipeline - Add Data To Knowledge Base * pnpm update * some adjusts --------- Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
1 parent 9738e9f commit a5b2d91

File tree

21 files changed

+630
-15
lines changed

21 files changed

+630
-15
lines changed

‎components/botsonic/botsonic.app.mjs

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ export default {
88
console.log(Object.keys(this.$auth));
99
},
1010
},
11-
};
11+
};

‎components/piped/piped.app.mjs

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ export default {
88
console.log(Object.keys(this.$auth));
99
},
1010
},
11-
};
11+
};

‎components/rocketadmin/rocketadmin.app.mjs

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ export default {
88
console.log(Object.keys(this.$auth));
99
},
1010
},
11-
};
11+
};

‎components/supadata/supadata.app.mjs

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ export default {
88
console.log(Object.keys(this.$auth));
99
},
1010
},
11-
};
11+
};

‎components/timelink/timelink.app.mjs

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ export default {
88
console.log(Object.keys(this.$auth));
99
},
1010
},
11-
};
11+
};

‎components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import { RESCRAPE_FREQUENCY_OPTIONS } from "../../common/constants.mjs";
2+
import vectorshift from "../../vectorshift.app.mjs";
3+
4+
export default {
5+
key: "vectorshift-add-data-to-knowledge-base",
6+
name: "Add Data to Knowledge Base",
7+
description: "Adds data to a knowledge base in VectorShift. [See the documentation](https://docs.vectorshift.ai/api-reference/knowledge-bases/index).",
8+
version: "0.0.1",
9+
type: "action",
10+
props: {
11+
vectorshift,
12+
knowledgeBaseId: {
13+
propDefinition: [
14+
vectorshift,
15+
"knowledgeBaseId",
16+
],
17+
},
18+
url: {
19+
type: "string",
20+
label: "URL",
21+
description: "URL to add to the knowledge base",
22+
},
23+
recursive: {
24+
type: "boolean",
25+
label: "Recursive",
26+
description: "Whether the scrape is recursive or not",
27+
default: false,
28+
},
29+
rescrapeFrequency: {
30+
type: "string",
31+
label: "Rescrape Frequency",
32+
description: "The frequency to rescrape the URL",
33+
options: RESCRAPE_FREQUENCY_OPTIONS,
34+
default: RESCRAPE_FREQUENCY_OPTIONS[0],
35+
},
36+
},
37+
async run({ $ }) {
38+
const response = await this.vectorshift.addDataToKnowledgeBase({
39+
$,
40+
knowledgeBaseId: this.knowledgeBaseId,
41+
data: {
42+
url_data: {
43+
request: {
44+
url: this.url,
45+
recursive: this.recursive,
46+
return_type: "CONTENT",
47+
},
48+
rescrape_frequency: this.rescrapeFrequency,
49+
},
50+
},
51+
});
52+
53+
$.export("$summary", `Added ${response.document_ids.length} document(s) to knowledge base ${this.knowledgeBaseId}. Document IDs: ${response.document_ids.join(", ")}`);
54+
return response;
55+
},
56+
};

‎components/vectorshift/actions/create-pipeline/create-pipeline.mjs

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import { ConfigurationError } from "@pipedream/platform";
2+
import { parseObject } from "../../common/utils.mjs";
3+
import vectorshift from "../../vectorshift.app.mjs";
4+
5+
export default {
6+
key: "vectorshift-create-pipeline",
7+
name: "Create Pipeline",
8+
description: "Creates a new pipeline in VectorShift. [See the documentation](https://docs.vectorshift.ai)",
9+
version: "0.0.1",
10+
type: "action",
11+
props: {
12+
vectorshift,
13+
name: {
14+
type: "string",
15+
label: "Pipeline Name",
16+
description: "Name of the new pipeline",
17+
},
18+
config: {
19+
type: "object",
20+
label: "Pipeline Config",
21+
description: "Configuration for the new pipeline",
22+
},
23+
description: {
24+
type: "string",
25+
label: "Description",
26+
description: "Optional description of the new pipeline",
27+
optional: true,
28+
},
29+
},
30+
async run({ $ }) {
31+
try {
32+
const response = await this.vectorshift.createPipeline({
33+
$,
34+
data: {
35+
name: this.name,
36+
config: parseObject(this.config),
37+
description: this.description,
38+
},
39+
});
40+
41+
$.export("$summary", `Created pipeline with ID ${response.id}`);
42+
return response;
43+
} catch ({ message }) {
44+
const parsedError = JSON.parse(message).error;
45+
throw new ConfigurationError(parsedError);
46+
}
47+
},
48+
};

‎components/vectorshift/actions/run-pipeline/run-pipeline.mjs

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import { ConfigurationError } from "@pipedream/platform";
2+
import { parseObject } from "../../common/utils.mjs";
3+
import vectorshift from "../../vectorshift.app.mjs";
4+
5+
export default {
6+
key: "vectorshift-run-pipeline",
7+
name: "Run Pipeline",
8+
description: "Executes a VectorShift pipeline with specified inputs. [See the documentation](https://docs.vectorshift.ai/api-reference/pipelines/run)",
9+
version: "0.0.1",
10+
type: "action",
11+
props: {
12+
vectorshift,
13+
pipelineId: {
14+
propDefinition: [
15+
vectorshift,
16+
"pipelineId",
17+
],
18+
},
19+
inputs: {
20+
type: "object",
21+
label: "Pipeline Inputs",
22+
description: "Inputs for the pipeline execution. [See the documentation](https://docs.vectorshift.ai/platform/pipelines/general/input) for further details",
23+
optional: true,
24+
},
25+
},
26+
async run({ $ }) {
27+
try {
28+
const response = await this.vectorshift.executePipeline({
29+
$,
30+
pipelineId: this.pipelineId,
31+
data: {
32+
inputs: parseObject(this.inputs),
33+
},
34+
});
35+
$.export("$summary", `Pipeline executed successfully. Run ID: ${response.run_id}`);
36+
return response;
37+
} catch ({ message }) {
38+
const parsedError = JSON.parse(message).error;
39+
throw new ConfigurationError(parsedError);
40+
}
41+
},
42+
};

‎components/vectorshift/common/constants.mjs

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
export const DATA_TYPE_OPTIONS = [
2+
"url",
3+
"wikipedia",
4+
"youtube",
5+
"arxiv",
6+
"git",
7+
];
8+
9+
export const RESCRAPE_FREQUENCY_OPTIONS = [
10+
"Never",
11+
"Hourly",
12+
"Daily",
13+
"Weekly",
14+
"Monthly",
15+
];

‎components/vectorshift/common/utils.mjs

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
export const checkTmp = (filename) => {
2+
if (!filename.startsWith("/tmp")) {
3+
return `/tmp/${filename}`;
4+
}
5+
return filename;
6+
};
7+
8+
export const parseObject = (obj) => {
9+
if (!obj) return undefined;
10+
11+
if (Array.isArray(obj)) {
12+
return obj.map((item) => {
13+
if (typeof item === "string") {
14+
try {
15+
return JSON.parse(item);
16+
} catch (e) {
17+
return item;
18+
}
19+
}
20+
return item;
21+
});
22+
}
23+
if (typeof obj === "string") {
24+
try {
25+
return JSON.parse(obj);
26+
} catch (e) {
27+
return obj;
28+
}
29+
}
30+
return obj;
31+
};

‎components/vectorshift/package.json

+6-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@pipedream/vectorshift",
3-
"version": "0.0.1",
3+
"version": "0.1.0",
44
"description": "Pipedream VectorShift Components",
55
"main": "vectorshift.app.mjs",
66
"keywords": [
@@ -11,5 +11,9 @@
1111
"author": "Pipedream <[email protected]> (https://pipedream.com/)",
1212
"publishConfig": {
1313
"access": "public"
14+
},
15+
"dependencies": {
16+
"@pipedream/platform": "^3.0.3",
17+
"fs": "^0.0.1-security"
1418
}
15-
}
19+
}

‎components/vectorshift/sources/common/base.mjs

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
2+
import app from "../../vectorshift.app.mjs";
3+
4+
export default {
5+
props: {
6+
app,
7+
timer: {
8+
type: "$.interface.timer",
9+
default: {
10+
intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
11+
},
12+
},
13+
},
14+
methods: {
15+
async emitEvent(maxResults = false) {
16+
const fn = this.getFunction();
17+
const { objects: response = [] } = await fn();
18+
19+
if (response.length) {
20+
if (maxResults && (response.length > maxResults)) {
21+
response.length = maxResults;
22+
}
23+
}
24+
25+
for (const item of response) {
26+
this.$emit(item, {
27+
id: item._id,
28+
summary: this.getSummary(item),
29+
ts: Date.parse(item.createdDate || new Date()),
30+
});
31+
}
32+
},
33+
},
34+
hooks: {
35+
async deploy() {
36+
await this.emitEvent(25);
37+
},
38+
},
39+
async run() {
40+
await this.emitEvent();
41+
},
42+
};

‎components/vectorshift/sources/new-chatbot/new-chatbot.mjs

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import common from "../common/base.mjs";
2+
import sampleEmit from "./test-event.mjs";
3+
4+
export default {
5+
...common,
6+
key: "vectorshift-new-chatbot",
7+
name: "New Chatbot Created",
8+
description: "Emit new event when a chatbot is created.",
9+
version: "0.0.1",
10+
type: "source",
11+
dedupe: "unique",
12+
methods: {
13+
...common.methods,
14+
getFunction() {
15+
return this.app.listChatbots;
16+
},
17+
getSummary(item) {
18+
return `New Chatbot: ${item.name || item._id}`;
19+
},
20+
},
21+
sampleEmit,
22+
};

0 commit comments

Comments
 (0)