From 41b655813cf0a77e2d3d9bfa6c5def01f1e1a06f Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Tue, 25 Mar 2025 10:43:00 -0300 Subject: [PATCH 01/10] vectorshift init --- .../add-data-to-knowledge-base.mjs | 100 ++++++++ .../create-pipeline/create-pipeline.mjs | 47 ++++ .../actions/run-pipeline/run-pipeline.mjs | 34 +++ components/vectorshift/package.json | 2 +- .../sources/new-chatbot/new-chatbot.mjs | 79 ++++++ .../new-knowledge-base/new-knowledge-base.mjs | 132 ++++++++++ .../sources/new-pipeline/new-pipeline.mjs | 118 +++++++++ components/vectorshift/vectorshift.app.mjs | 235 +++++++++++++++++- 8 files changed, 743 insertions(+), 4 deletions(-) create mode 100644 components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs create mode 100644 components/vectorshift/actions/create-pipeline/create-pipeline.mjs create mode 100644 components/vectorshift/actions/run-pipeline/run-pipeline.mjs create mode 100644 components/vectorshift/sources/new-chatbot/new-chatbot.mjs create mode 100644 components/vectorshift/sources/new-knowledge-base/new-knowledge-base.mjs create mode 100644 components/vectorshift/sources/new-pipeline/new-pipeline.mjs diff --git a/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs b/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs new file mode 100644 index 0000000000000..3c75bbb6af5d2 --- /dev/null +++ b/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs @@ -0,0 +1,100 @@ +import vectorshift from "../../vectorshift.app.mjs"; +import { axios } from "@pipedream/platform"; + +export default { + key: "vectorshift-add-data-to-knowledge-base", + name: "Add Data to Knowledge Base", + description: "Adds data to a knowledge base in VectorShift. [See the documentation](https://docs.vectorshift.ai/api-reference/knowledge-bases/index).", + version: "0.0.{{ts}}", + type: "action", + props: { + vectorshift, + knowledgeBaseId: { + propDefinition: [ + vectorshift, + "knowledgeBaseId", + ], + }, + fileData: { + propDefinition: [ + vectorshift, + "fileData", + ], + optional: true, + }, + urlData: { + propDefinition: [ + vectorshift, + "urlData", + ], + optional: true, + }, + wikipedia: { + propDefinition: [ + vectorshift, + "wikipedia", + ], + optional: true, + }, + youtube: { + propDefinition: [ + vectorshift, + "youtube", + ], + optional: true, + }, + arxiv: { + propDefinition: [ + vectorshift, + "arxiv", + ], + optional: true, + }, + git: { + propDefinition: [ + vectorshift, + "git", + ], + optional: true, + }, + addDataConfig: { + propDefinition: [ + vectorshift, + "addDataConfig", + ], + optional: true, + }, + }, + async run({ $ }) { + if ( + !this.fileData && + !this.urlData && + !this.wikipedia && + !this.youtube && + !this.arxiv && + !this.git + ) { + throw new Error("At least one of fileData, urlData, wikipedia, youtube, arxiv, or git must be provided."); + } + + const documentIds = await this.vectorshift.addDataToKnowledgeBase({ + knowledgeBaseId: this.knowledgeBaseId, + fileData: this.fileData, + urlData: this.urlData, + wikipedia: this.wikipedia, + youtube: this.youtube, + arxiv: this.arxiv, + git: this.git, + addDataConfig: this.addDataConfig, + }); + + $.export( + "$summary", + `Added ${documentIds.length} document(s) to knowledge base ${this.knowledgeBaseId}. Document IDs: ${documentIds.join(", ")}`, + ); + + return { + document_ids: documentIds, + }; + }, +}; diff --git a/components/vectorshift/actions/create-pipeline/create-pipeline.mjs b/components/vectorshift/actions/create-pipeline/create-pipeline.mjs new file mode 100644 index 0000000000000..73780da3f0ce3 --- /dev/null +++ b/components/vectorshift/actions/create-pipeline/create-pipeline.mjs @@ -0,0 +1,47 @@ +import vectorshift from "../../vectorshift.app.mjs"; +import { axios } from "@pipedream/platform"; + +export default { + key: "vectorshift-create-pipeline", + name: "Create Pipeline", + description: "Creates a new pipeline in VectorShift. [See the documentation](https://docs.vectorshift.ai)", + version: "0.0.{{ts}}", + type: "action", + props: { + vectorshift: { + type: "app", + app: "vectorshift", + }, + name: { + propDefinition: [ + "vectorshift", + "name", + ], + }, + config: { + propDefinition: [ + "vectorshift", + "config", + ], + }, + description: { + propDefinition: [ + "vectorshift", + "description", + ], + optional: true, + }, + }, + async run({ $ }) { + const pipelineId = await this.vectorshift.createPipeline({ + name: this.name, + config: this.config, + description: this.description, + }); + + $.export("$summary", `Created pipeline with ID ${pipelineId}`); + return { + id: pipelineId, + }; + }, +}; diff --git a/components/vectorshift/actions/run-pipeline/run-pipeline.mjs b/components/vectorshift/actions/run-pipeline/run-pipeline.mjs new file mode 100644 index 0000000000000..bba025418f82b --- /dev/null +++ b/components/vectorshift/actions/run-pipeline/run-pipeline.mjs @@ -0,0 +1,34 @@ +import vectorshift from "../../vectorshift.app.mjs"; +import { axios } from "@pipedream/platform"; + +export default { + key: "vectorshift-run-pipeline", + name: "Run Pipeline", + description: "Executes a VectorShift pipeline with specified inputs. [See the documentation](https://docs.vectorshift.ai/api-reference/pipelines/run)", + version: "0.0.{{ts}}", + type: "action", + props: { + vectorshift, + pipelineId: { + propDefinition: [ + vectorshift, + "pipelineId", + ], + }, + inputs: { + type: "string", + label: "Pipeline Inputs", + description: "Inputs for the pipeline execution as a JSON string", + }, + }, + async run({ $ }) { + const runId = await this.vectorshift.executePipeline({ + pipelineId: this.pipelineId, + inputs: this.inputs, + }); + $.export("$summary", `Pipeline executed successfully. Run ID: ${runId}`); + return { + run_id: runId, + }; + }, +}; diff --git a/components/vectorshift/package.json b/components/vectorshift/package.json index 1fca8e920ff45..13ca7eb0569f5 100644 --- a/components/vectorshift/package.json +++ b/components/vectorshift/package.json @@ -12,4 +12,4 @@ "publishConfig": { "access": "public" } -} \ No newline at end of file +} diff --git a/components/vectorshift/sources/new-chatbot/new-chatbot.mjs b/components/vectorshift/sources/new-chatbot/new-chatbot.mjs new file mode 100644 index 0000000000000..7c6b63768b545 --- /dev/null +++ b/components/vectorshift/sources/new-chatbot/new-chatbot.mjs @@ -0,0 +1,79 @@ +import vectorshift from "../../vectorshift.app.mjs"; +import { + axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; + +export default { + key: "vectorshift-new-chatbot", + name: "New Chatbot Created", + description: "Emit new event when a chatbot is created. [See the documentation]()", + version: "0.0.{{ts}}", + type: "source", + dedupe: "unique", + props: { + vectorshift, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + hooks: { + async deploy() { + try { + const chatbots = await this.vectorshift.listChatbots(); + if (!Array.isArray(chatbots)) { + throw new Error("Chatbots response is not an array"); + } + const sortedChatbots = chatbots.sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); + const recentChatbots = sortedChatbots.slice(0, 50).reverse(); + for (const chatbot of recentChatbots) { + this.$emit(chatbot, { + id: chatbot.id, + summary: `New Chatbot: ${chatbot.name || "Unnamed"}`, + ts: chatbot.created_at + ? Date.parse(chatbot.created_at) + : Date.now(), + }); + } + const chatbotIds = chatbots.map((cb) => cb.id); + await this.db.set("chatbotIds", chatbotIds); + } catch (error) { + console.error(`Error in deploy hook: ${error.message}`); + } + }, + async activate() { + // No webhook setup required + }, + async deactivate() { + // No webhook teardown required + }, + }, + async run() { + try { + const currentChatbots = await this.vectorshift.listChatbots(); + if (!Array.isArray(currentChatbots)) { + throw new Error("Chatbots response is not an array"); + } + const storedChatbotIds = await this.db.get("chatbotIds") || []; + const newChatbots = currentChatbots.filter((chatbot) => !storedChatbotIds.includes(chatbot.id)); + + for (const chatbot of newChatbots) { + this.$emit(chatbot, { + id: chatbot.id, + summary: `New Chatbot: ${chatbot.name || "Unnamed"}`, + ts: chatbot.created_at + ? Date.parse(chatbot.created_at) + : Date.now(), + }); + } + + const updatedChatbotIds = currentChatbots.map((cb) => cb.id); + await this.db.set("chatbotIds", updatedChatbotIds); + } catch (error) { + console.error(`Error in run method: ${error.message}`); + } + }, +}; diff --git a/components/vectorshift/sources/new-knowledge-base/new-knowledge-base.mjs b/components/vectorshift/sources/new-knowledge-base/new-knowledge-base.mjs new file mode 100644 index 0000000000000..896860b4bfbbb --- /dev/null +++ b/components/vectorshift/sources/new-knowledge-base/new-knowledge-base.mjs @@ -0,0 +1,132 @@ +import { + axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; +import vectorshift from "../../vectorshift.app.mjs"; + +export default { + key: "vectorshift-new-knowledge-base", + name: "New Knowledge Base Created", + description: "Emit new event when a knowledge base is created in Vectorshift. [See the documentation]()", + version: "0.0.{{ts}}", + type: "source", + dedupe: "unique", + props: { + vectorshift: { + type: "app", + app: "vectorshift", + }, + db: { + type: "$.service.db", + }, + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + hooks: { + async deploy() { + try { + const knowledgeBases = await this.vectorshift.listKnowledgeBases(); + if (!knowledgeBases || knowledgeBases.length === 0) return; + + const sortedKnowledgeBases = knowledgeBases + .sort((a, b) => { + const aTs = a.created_at + ? Date.parse(a.created_at) + : 0; + const bTs = b.created_at + ? Date.parse(b.created_at) + : 0; + return bTs - aTs; + }) + .slice(0, 50); + + for (const kb of sortedKnowledgeBases.reverse()) { + this.$emit( + kb, + { + id: kb.id || Date.parse(kb.created_at) || Date.now(), + summary: `New Knowledge Base: ${kb.name}`, + ts: kb.created_at + ? Date.parse(kb.created_at) + : Date.now(), + }, + ); + } + + const latestTimestamp = sortedKnowledgeBases.reduce((max, kb) => { + const kbTs = kb.created_at + ? Date.parse(kb.created_at) + : 0; + return kbTs > max + ? kbTs + : max; + }, 0); + + await this.db.set("lastEmitTimestamp", latestTimestamp); + } catch (error) { + console.error("Error during deploy hook:", error); + } + }, + async activate() { + // No webhook subscription necessary for polling source + }, + async deactivate() { + // No webhook subscription to remove for polling source + }, + }, + async run() { + try { + const lastTimestamp = (await this.db.get("lastEmitTimestamp")) || 0; + const knowledgeBases = await this.vectorshift.listKnowledgeBases(); + if (!knowledgeBases || knowledgeBases.length === 0) return; + + const newKnowledgeBases = knowledgeBases + .filter((kb) => { + const kbTs = kb.created_at + ? Date.parse(kb.created_at) + : 0; + return kbTs > lastTimestamp; + }) + .sort((a, b) => { + const aTs = a.created_at + ? Date.parse(a.created_at) + : 0; + const bTs = b.created_at + ? Date.parse(b.created_at) + : 0; + return aTs - bTs; + }); + + for (const kb of newKnowledgeBases) { + const kbTimestamp = kb.created_at + ? Date.parse(kb.created_at) + : Date.now(); + this.$emit( + kb, + { + id: kb.id || kbTimestamp, + summary: `New Knowledge Base: ${kb.name}`, + ts: kbTimestamp, + }, + ); + } + + if (newKnowledgeBases.length > 0) { + const newLastTimestamp = newKnowledgeBases.reduce((max, kb) => { + const kbTs = kb.created_at + ? Date.parse(kb.created_at) + : 0; + return kbTs > max + ? kbTs + : max; + }, lastTimestamp); + await this.db.set("lastEmitTimestamp", newLastTimestamp); + } + } catch (error) { + console.error("Error during run method:", error); + } + }, +}; diff --git a/components/vectorshift/sources/new-pipeline/new-pipeline.mjs b/components/vectorshift/sources/new-pipeline/new-pipeline.mjs new file mode 100644 index 0000000000000..55e35f6fcc427 --- /dev/null +++ b/components/vectorshift/sources/new-pipeline/new-pipeline.mjs @@ -0,0 +1,118 @@ +import vectorshift from "../../vectorshift.app.mjs"; +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; + +export default { + key: "vectorshift-new-pipeline", + name: "New Pipeline Created", + description: "Emit new event when a new pipeline is created in VectorShift. [See the documentation]()", + version: "0.0.{{ts}}", + type: "source", + dedupe: "unique", + props: { + vectorshift: { + type: "app", + app: "vectorshift", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + hooks: { + async deploy() { + try { + const pipelines = await this.vectorshift.listPipelines(); + if (!pipelines || pipelines.length === 0) { + return; + } + + // Sort pipelines by creation time descending + pipelines.sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); + + const recentPipelines = pipelines.slice(0, 50); + + let latestTimestamp = 0; + + for (const pipeline of recentPipelines) { + const timestamp = pipeline.created_at + ? Date.parse(pipeline.created_at) + : Date.now(); + latestTimestamp = Math.max(latestTimestamp, timestamp); + + this.$emit(pipeline, { + id: pipeline.id, + summary: `New Pipeline Created: ${pipeline.name}`, + ts: timestamp, + }); + } + + await this.db.set("last_timestamp", latestTimestamp); + } catch (error) { + this.$emit( + { + error: error.message, + }, + { + summary: "Error during deploy", + ts: Date.now(), + }, + ); + } + }, + async activate() { + // No activation steps needed for polling source + }, + async deactivate() { + // No deactivation steps needed for polling source + }, + }, + async run() { + try { + const pipelines = await this.vectorshift.listPipelines(); + if (!pipelines || pipelines.length === 0) { + return; + } + + const lastTimestamp = (await this.db.get("last_timestamp")) || 0; + + // Sort pipelines by creation time ascending + pipelines.sort((a, b) => new Date(a.created_at) - new Date(b.created_at)); + + let newLastTimestamp = lastTimestamp; + + for (const pipeline of pipelines) { + const pipelineTimestamp = pipeline.created_at + ? Date.parse(pipeline.created_at) + : Date.now(); + if (pipelineTimestamp > lastTimestamp) { + this.$emit(pipeline, { + id: pipeline.id, + summary: `New Pipeline Created: ${pipeline.name}`, + ts: pipelineTimestamp, + }); + + if (pipelineTimestamp > newLastTimestamp) { + newLastTimestamp = pipelineTimestamp; + } + } + } + + if (newLastTimestamp > lastTimestamp) { + await this.db.set("last_timestamp", newLastTimestamp); + } + } catch (error) { + this.$emit( + { + error: error.message, + }, + { + summary: "Error during run", + ts: Date.now(), + }, + ); + } + }, +}; diff --git a/components/vectorshift/vectorshift.app.mjs b/components/vectorshift/vectorshift.app.mjs index 6a8e0a6aeaf84..ed69117e47d46 100644 --- a/components/vectorshift/vectorshift.app.mjs +++ b/components/vectorshift/vectorshift.app.mjs @@ -1,11 +1,240 @@ +import { axios } from "@pipedream/platform"; + export default { type: "app", app: "vectorshift", - propDefinitions: {}, + version: "0.0.{{ts}}", + propDefinitions: { + // Create Pipeline Props + name: { + type: "string", + label: "Pipeline Name", + description: "Name of the new pipeline", + }, + config: { + type: "string", + label: "Pipeline Config", + description: "Configuration for the new pipeline as a JSON string", + }, + description: { + type: "string", + label: "Description", + description: "Optional description of the new pipeline", + optional: true, + }, + // Execute Pipeline Props + pipelineId: { + type: "string", + label: "Pipeline ID", + description: "The ID of the pipeline to execute", + async options() { + const pipelines = await this.listPipelines(); + if (!pipelines) return []; + return pipelines.map((pipeline) => ({ + label: pipeline.name, + value: pipeline.id, + })); + }, + }, + inputs: { + type: "string", + label: "Pipeline Inputs", + description: "Inputs for the pipeline execution as a JSON string", + }, + // Add Data to Knowledge Base Props + knowledgeBaseId: { + type: "string", + label: "Knowledge Base ID", + description: "The ID of the knowledge base", + async options() { + const knowledgeBases = await this.listKnowledgeBases(); + if (!knowledgeBases) return []; + return knowledgeBases.map((kb) => ({ + label: kb.name, + value: kb.id, + })); + }, + }, + fileData: { + type: "object", + label: "File Data", + description: "File data to add to the knowledge base", + optional: true, + }, + urlData: { + type: "object", + label: "URL Data", + description: "URL data to add to the knowledge base", + optional: true, + }, + wikipedia: { + type: "string", + label: "Wikipedia", + description: "Wikipedia data to add to the knowledge base", + optional: true, + }, + youtube: { + type: "string", + label: "YouTube", + description: "YouTube data to add to the knowledge base", + optional: true, + }, + arxiv: { + type: "string", + label: "ArXiv", + description: "ArXiv data to add to the knowledge base", + optional: true, + }, + git: { + type: "string", + label: "Git", + description: "Git data to add to the knowledge base", + optional: true, + }, + addDataConfig: { + type: "string", + label: "Add Data Config", + description: "Configuration for adding data to the knowledge base as a JSON string", + optional: true, + }, + }, methods: { - // this.$auth contains connected account data + // Existing Method authKeys() { console.log(Object.keys(this.$auth)); }, + // Base URL Method + _baseUrl() { + return "https://api.vectorshift.ai/v1"; + }, + // Make Request Method + async _makeRequest(opts = {}) { + const { + $ = this, method = "GET", path = "/", headers = {}, ...otherOpts + } = opts; + return axios($, { + method, + url: this._baseUrl() + path, + headers: { + ...headers, + Authorization: `Bearer ${this.$auth.token}`, + }, + ...otherOpts, + }); + }, + // List Pipelines Method + async listPipelines(opts = {}) { + const response = await this._makeRequest({ + path: "/pipelines", + method: "GET", + ...opts, + }); + if (response.status !== "success") { + throw new Error(`Failed to list pipelines: ${response.status}`); + } + return response.objects; + }, + // List Knowledge Bases Method + async listKnowledgeBases(opts = {}) { + const response = await this._makeRequest({ + path: "/knowledge-bases", + method: "GET", + ...opts, + }); + if (response.status !== "success") { + throw new Error(`Failed to list knowledge bases: ${response.status}`); + } + return response.objects; + }, + // Create Pipeline Method + async createPipeline({ + name, config, description, + }) { + const data = { + name, + config: JSON.parse(config), + }; + if (description) { + data.description = description; + } + const response = await this._makeRequest({ + path: "/pipeline", + method: "POST", + data, + }); + if (response.status !== "success") { + throw new Error(`Failed to create pipeline: ${response.status}`); + } + return response.id; + }, + // Execute Pipeline Method + async executePipeline({ + pipelineId, inputs, + }) { + const data = { + inputs: JSON.parse(inputs), + }; + const response = await this._makeRequest({ + path: `/pipeline/${pipelineId}/run`, + method: "POST", + data, + }); + if (response.status !== "success") { + throw new Error(`Failed to execute pipeline: ${response.status}`); + } + return response.run_id; + }, + // Add Data to Knowledge Base Method + async addDataToKnowledgeBase({ + knowledgeBaseId, fileData, urlData, wikipedia, youtube, arxiv, git, addDataConfig, + }) { + if (!fileData && !urlData && !wikipedia && !youtube && !arxiv && !git) { + throw new Error("At least one of fileData, urlData, wikipedia, youtube, arxiv, or git must be provided."); + } + const data = {}; + if (fileData) data.file_data = fileData; + if (urlData) data.url_data = urlData; + if (wikipedia) data.wikipedia = wikipedia; + if (youtube) data.youtube = youtube; + if (arxiv) data.arxiv = arxiv; + if (git) data.git = git; + if (addDataConfig) data.config = JSON.parse(addDataConfig); + const response = await this._makeRequest({ + path: `/knowledge-base/${knowledgeBaseId}/index`, + method: "POST", + data, + }); + if (response.status !== "success") { + throw new Error(`Failed to add data to knowledge base: ${response.status}`); + } + return response.document_ids; + }, + // Create Knowledge Base Method + async createKnowledgeBase({ name }) { + const data = { + name, + }; + const response = await this._makeRequest({ + path: "/knowledge-bases/create", + method: "POST", + data, + }); + if (response.status !== "success") { + throw new Error(`Failed to create knowledge base: ${response.status}`); + } + return response.id; + }, + // Emit New Pipeline Created Event + async emitNewPipelineCreated(pipeline) { + // Implementation to emit event + }, + // Emit New Knowledge Base Created Event + async emitNewKnowledgeBaseCreated(knowledgeBase) { + // Implementation to emit event + }, + // Emit New Chatbot Created Event + async emitNewChatbotCreated(chatbot) { + // Implementation to emit event + }, }, -}; \ No newline at end of file +}; From 3fe71d53d890fe9f0d2e61473142f96ca409e6b4 Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Wed, 26 Mar 2025 19:52:30 -0300 Subject: [PATCH 02/10] [Components] vectorshift #16018 Sources - New Pipeline - New Knowledge Base - New Chatbot Actions - Create Pipeline - Run Pipeline - Add Data To Knowledge Base --- .../add-data-to-knowledge-base.mjs | 148 ++++++----- .../create-pipeline/create-pipeline.mjs | 55 ++-- .../actions/run-pipeline/run-pipeline.mjs | 32 ++- components/vectorshift/common/constants.mjs | 15 ++ components/vectorshift/common/utils.mjs | 31 +++ components/vectorshift/package.json | 6 +- .../vectorshift/sources/common/base.mjs | 42 +++ .../sources/new-chatbot/new-chatbot.mjs | 81 +----- .../sources/new-chatbot/test-event.mjs | 46 ++++ .../new-knowledge-base/new-knowledge-base.mjs | 134 +--------- .../sources/new-knowledge-base/test-event.mjs | 46 ++++ .../sources/new-pipeline/new-pipeline.mjs | 120 +-------- .../sources/new-pipeline/test-event.mjs | 46 ++++ components/vectorshift/vectorshift.app.mjs | 250 ++++-------------- 14 files changed, 453 insertions(+), 599 deletions(-) create mode 100644 components/vectorshift/common/constants.mjs create mode 100644 components/vectorshift/common/utils.mjs create mode 100644 components/vectorshift/sources/common/base.mjs create mode 100644 components/vectorshift/sources/new-chatbot/test-event.mjs create mode 100644 components/vectorshift/sources/new-knowledge-base/test-event.mjs create mode 100644 components/vectorshift/sources/new-pipeline/test-event.mjs diff --git a/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs b/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs index 3c75bbb6af5d2..3bc088f5008f4 100644 --- a/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs +++ b/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs @@ -1,11 +1,13 @@ +import { + DATA_TYPE_OPTIONS, RESCRAPE_FRANQUENCY_OPTIONS, +} from "../../common/constants.mjs"; import vectorshift from "../../vectorshift.app.mjs"; -import { axios } from "@pipedream/platform"; export default { key: "vectorshift-add-data-to-knowledge-base", name: "Add Data to Knowledge Base", description: "Adds data to a knowledge base in VectorShift. [See the documentation](https://docs.vectorshift.ai/api-reference/knowledge-bases/index).", - version: "0.0.{{ts}}", + version: "0.0.1", type: "action", props: { vectorshift, @@ -15,86 +17,100 @@ export default { "knowledgeBaseId", ], }, - fileData: { - propDefinition: [ - vectorshift, - "fileData", - ], - optional: true, + dataType: { + type: "string", + label: "Data Type", + description: "The type of the data to be added.", + options: DATA_TYPE_OPTIONS, + reloadProps: true, }, - urlData: { - propDefinition: [ - vectorshift, - "urlData", - ], + file: { + type: "string", + label: "File Data", + description: "The file to be uploaded, please provide a file from `/tmp`. To upload a file to `/tmp` folder, [See the documentation](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + hidden: true, + }, + url: { + type: "string", + label: "URL", + description: "URL to add to the knowledge base", + hidden: true, + }, + recursive: { + type: "boolean", + label: "Recursive", + description: "Whether the scrape is recursive or not", + default: false, + }, + rescrapeFrequency: { + type: "string", + label: "Rescrape Frequency", + description: "The frequency to rescrape the URL", + options: RESCRAPE_FRANQUENCY_OPTIONS, optional: true, + hidden: true, }, wikipedia: { - propDefinition: [ - vectorshift, - "wikipedia", - ], - optional: true, + type: "string", + label: "Wikipedia", + description: "Wikipedia data to add to the knowledge base", + hidden: true, }, youtube: { - propDefinition: [ - vectorshift, - "youtube", - ], - optional: true, + type: "string", + label: "YouTube", + description: "YouTube data to add to the knowledge base", + hidden: true, }, arxiv: { - propDefinition: [ - vectorshift, - "arxiv", - ], - optional: true, + type: "string", + label: "ArXiv", + description: "ArXiv data to add to the knowledge base", + hidden: true, }, git: { - propDefinition: [ - vectorshift, - "git", - ], - optional: true, - }, - addDataConfig: { - propDefinition: [ - vectorshift, - "addDataConfig", - ], - optional: true, + type: "string", + label: "Git", + description: "Git data to add to the knowledge base", + hidden: true, }, }, - async run({ $ }) { - if ( - !this.fileData && - !this.urlData && - !this.wikipedia && - !this.youtube && - !this.arxiv && - !this.git - ) { - throw new Error("At least one of fileData, urlData, wikipedia, youtube, arxiv, or git must be provided."); + async additionalProps(props) { + if (this.dataType) { + props.url.hidden = true; + props.wikipedia.hidden = true; + props.youtube.hidden = true; + props.arxiv.hidden = true; + props.git.hidden = true; + props[this.dataType].hidden = false; + + const isUrl = this.dataType === "url"; + props.rescrapeFrequency.hidden = !isUrl; + props.recursive.hidden = !isUrl; } + return {}; + }, + async run({ $ }) { + let data = (this.dataType === "url") + ? { + url_data: { + request: { + url: this.url, + recursive: this.recursive, + return_type: "CONTENT", + }, + rescrape_frequency: this.rescrapeFrequency, + }, + } + : this[this.dataType]; - const documentIds = await this.vectorshift.addDataToKnowledgeBase({ + const response = await this.vectorshift.addDataToKnowledgeBase({ + $, knowledgeBaseId: this.knowledgeBaseId, - fileData: this.fileData, - urlData: this.urlData, - wikipedia: this.wikipedia, - youtube: this.youtube, - arxiv: this.arxiv, - git: this.git, - addDataConfig: this.addDataConfig, + data, }); - $.export( - "$summary", - `Added ${documentIds.length} document(s) to knowledge base ${this.knowledgeBaseId}. Document IDs: ${documentIds.join(", ")}`, - ); - - return { - document_ids: documentIds, - }; + $.export("$summary", `Added ${response.document_ids.length} document(s) to knowledge base ${this.knowledgeBaseId}. Document IDs: ${response.document_ids.join(", ")}`); + return response; }, }; diff --git a/components/vectorshift/actions/create-pipeline/create-pipeline.mjs b/components/vectorshift/actions/create-pipeline/create-pipeline.mjs index 73780da3f0ce3..df6e2e149b830 100644 --- a/components/vectorshift/actions/create-pipeline/create-pipeline.mjs +++ b/components/vectorshift/actions/create-pipeline/create-pipeline.mjs @@ -1,47 +1,48 @@ +import { ConfigurationError } from "@pipedream/platform"; +import { parseObject } from "../../common/utils.mjs"; import vectorshift from "../../vectorshift.app.mjs"; -import { axios } from "@pipedream/platform"; export default { key: "vectorshift-create-pipeline", name: "Create Pipeline", description: "Creates a new pipeline in VectorShift. [See the documentation](https://docs.vectorshift.ai)", - version: "0.0.{{ts}}", + version: "0.0.1", type: "action", props: { - vectorshift: { - type: "app", - app: "vectorshift", - }, + vectorshift, name: { - propDefinition: [ - "vectorshift", - "name", - ], + type: "string", + label: "Pipeline Name", + description: "Name of the new pipeline", }, config: { - propDefinition: [ - "vectorshift", - "config", - ], + type: "object", + label: "Pipeline Config", + description: "Configuration for the new pipeline", }, description: { - propDefinition: [ - "vectorshift", - "description", - ], + type: "string", + label: "Description", + description: "Optional description of the new pipeline", optional: true, }, }, async run({ $ }) { - const pipelineId = await this.vectorshift.createPipeline({ - name: this.name, - config: this.config, - description: this.description, - }); + try { + const response = await this.vectorshift.createPipeline({ + $, + data: { + name: this.name, + config: parseObject(this.config), + description: this.description, + }, + }); - $.export("$summary", `Created pipeline with ID ${pipelineId}`); - return { - id: pipelineId, - }; + $.export("$summary", `Created pipeline with ID ${response.id}`); + return response; + } catch ({ message }) { + const parsedError = JSON.parse(message).error; + throw new ConfigurationError(parsedError); + } }, }; diff --git a/components/vectorshift/actions/run-pipeline/run-pipeline.mjs b/components/vectorshift/actions/run-pipeline/run-pipeline.mjs index bba025418f82b..2d33468c9539b 100644 --- a/components/vectorshift/actions/run-pipeline/run-pipeline.mjs +++ b/components/vectorshift/actions/run-pipeline/run-pipeline.mjs @@ -1,11 +1,12 @@ +import { ConfigurationError } from "@pipedream/platform"; +import { parseObject } from "../../common/utils.mjs"; import vectorshift from "../../vectorshift.app.mjs"; -import { axios } from "@pipedream/platform"; export default { key: "vectorshift-run-pipeline", name: "Run Pipeline", description: "Executes a VectorShift pipeline with specified inputs. [See the documentation](https://docs.vectorshift.ai/api-reference/pipelines/run)", - version: "0.0.{{ts}}", + version: "0.0.1", type: "action", props: { vectorshift, @@ -16,19 +17,26 @@ export default { ], }, inputs: { - type: "string", + type: "object", label: "Pipeline Inputs", - description: "Inputs for the pipeline execution as a JSON string", + description: "Inputs for the pipeline execution. [See the documentation](https://docs.vectorshift.ai/platform/pipelines/general/input) for further details", + optional: true, }, }, async run({ $ }) { - const runId = await this.vectorshift.executePipeline({ - pipelineId: this.pipelineId, - inputs: this.inputs, - }); - $.export("$summary", `Pipeline executed successfully. Run ID: ${runId}`); - return { - run_id: runId, - }; + try { + const response = await this.vectorshift.executePipeline({ + $, + pipelineId: this.pipelineId, + data: { + inputs: parseObject(this.inputs), + }, + }); + $.export("$summary", `Pipeline executed successfully. Run ID: ${response.run_id}`); + return response; + } catch ({ message }) { + const parsedError = JSON.parse(message).error; + throw new ConfigurationError(parsedError); + } }, }; diff --git a/components/vectorshift/common/constants.mjs b/components/vectorshift/common/constants.mjs new file mode 100644 index 0000000000000..f2031e56e060b --- /dev/null +++ b/components/vectorshift/common/constants.mjs @@ -0,0 +1,15 @@ +export const DATA_TYPE_OPTIONS = [ + "url", + "wikipedia", + "youtube", + "arxiv", + "git", +]; + +export const RESCRAPE_FRANQUENCY_OPTIONS = [ + "Never", + "Hourly", + "Daily", + "Weekly", + "Monthly", +]; diff --git a/components/vectorshift/common/utils.mjs b/components/vectorshift/common/utils.mjs new file mode 100644 index 0000000000000..0cd1a12b6a4ba --- /dev/null +++ b/components/vectorshift/common/utils.mjs @@ -0,0 +1,31 @@ +export const checkTmp = (filename) => { + if (!filename.startsWith("/tmp")) { + return `/tmp/${filename}`; + } + return filename; +}; + +export const parseObject = (obj) => { + if (!obj) return undefined; + + if (Array.isArray(obj)) { + return obj.map((item) => { + if (typeof item === "string") { + try { + return JSON.parse(item); + } catch (e) { + return item; + } + } + return item; + }); + } + if (typeof obj === "string") { + try { + return JSON.parse(obj); + } catch (e) { + return obj; + } + } + return obj; +}; diff --git a/components/vectorshift/package.json b/components/vectorshift/package.json index 13ca7eb0569f5..dcb3e1a05fb62 100644 --- a/components/vectorshift/package.json +++ b/components/vectorshift/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/vectorshift", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream VectorShift Components", "main": "vectorshift.app.mjs", "keywords": [ @@ -11,5 +11,9 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3", + "fs": "^0.0.1-security" } } diff --git a/components/vectorshift/sources/common/base.mjs b/components/vectorshift/sources/common/base.mjs new file mode 100644 index 0000000000000..b1334d2299af3 --- /dev/null +++ b/components/vectorshift/sources/common/base.mjs @@ -0,0 +1,42 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import app from "../../verctorshift.app.mjs"; + +export default { + props: { + app, + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + methods: { + async emitEvent(maxResults = false) { + const fn = this.getFunction(); + const { objects: response } = await fn(); + + if (response.length) { + if (maxResults && (response.length > maxResults)) { + response.length = maxResults; + } + } + + for (const item of response) { + this.$emit(item, { + id: item.id, + summary: this.getSummary(item), + ts: Date.parse(item.created || new Date()), + }); + } + }, + }, + hooks: { + async deploy() { + await this.emitEvent(25); + }, + }, + async run() { + await this.emitEvent(); + }, +}; diff --git a/components/vectorshift/sources/new-chatbot/new-chatbot.mjs b/components/vectorshift/sources/new-chatbot/new-chatbot.mjs index 7c6b63768b545..f1ed5dfa132d8 100644 --- a/components/vectorshift/sources/new-chatbot/new-chatbot.mjs +++ b/components/vectorshift/sources/new-chatbot/new-chatbot.mjs @@ -1,79 +1,22 @@ -import vectorshift from "../../vectorshift.app.mjs"; -import { - axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, -} from "@pipedream/platform"; +import common from "../common/base.mjs"; +import sampleEmit from "./test-event.mjs"; export default { + ...common, key: "vectorshift-new-chatbot", name: "New Chatbot Created", - description: "Emit new event when a chatbot is created. [See the documentation]()", - version: "0.0.{{ts}}", + description: "Emit new event when a chatbot is created.", + version: "0.0.1", type: "source", dedupe: "unique", - props: { - vectorshift, - db: "$.service.db", - timer: { - type: "$.interface.timer", - default: { - intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, - }, + methods: { + ...common.methods, + getFunction() { + return this.app.listChatbots; }, - }, - hooks: { - async deploy() { - try { - const chatbots = await this.vectorshift.listChatbots(); - if (!Array.isArray(chatbots)) { - throw new Error("Chatbots response is not an array"); - } - const sortedChatbots = chatbots.sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); - const recentChatbots = sortedChatbots.slice(0, 50).reverse(); - for (const chatbot of recentChatbots) { - this.$emit(chatbot, { - id: chatbot.id, - summary: `New Chatbot: ${chatbot.name || "Unnamed"}`, - ts: chatbot.created_at - ? Date.parse(chatbot.created_at) - : Date.now(), - }); - } - const chatbotIds = chatbots.map((cb) => cb.id); - await this.db.set("chatbotIds", chatbotIds); - } catch (error) { - console.error(`Error in deploy hook: ${error.message}`); - } - }, - async activate() { - // No webhook setup required - }, - async deactivate() { - // No webhook teardown required + getSummary(item) { + return `New Chatbot: ${item.name || "Unnamed"}`; }, }, - async run() { - try { - const currentChatbots = await this.vectorshift.listChatbots(); - if (!Array.isArray(currentChatbots)) { - throw new Error("Chatbots response is not an array"); - } - const storedChatbotIds = await this.db.get("chatbotIds") || []; - const newChatbots = currentChatbots.filter((chatbot) => !storedChatbotIds.includes(chatbot.id)); - - for (const chatbot of newChatbots) { - this.$emit(chatbot, { - id: chatbot.id, - summary: `New Chatbot: ${chatbot.name || "Unnamed"}`, - ts: chatbot.created_at - ? Date.parse(chatbot.created_at) - : Date.now(), - }); - } - - const updatedChatbotIds = currentChatbots.map((cb) => cb.id); - await this.db.set("chatbotIds", updatedChatbotIds); - } catch (error) { - console.error(`Error in run method: ${error.message}`); - } - }, + sampleEmit, }; diff --git a/components/vectorshift/sources/new-chatbot/test-event.mjs b/components/vectorshift/sources/new-chatbot/test-event.mjs new file mode 100644 index 0000000000000..6c4f7457f87bf --- /dev/null +++ b/components/vectorshift/sources/new-chatbot/test-event.mjs @@ -0,0 +1,46 @@ +export default { + "course_title": "Class title", + "course_subject": "Course subject", + "course_level": "Course level", + "school_id": 16385, + "start_date": "2024-10-21", + "end_date": "2024-10-28", + "recurrence": "weekly", + "payment_frequency": "free", + "id": 153319, + "photo": null, + "payment_fee": "0", + "course_description": "", + "course_full_title": "Class title [Online Lesson]", + "created": "2024-10-17 18:20:13", + "modified": "2024-10-17 18:20:15", + "color": "color1", + "course_started": false, + "course_ended": false, + "course_status": 1, + "num_enrolled_students": 0, + "teachers": "66792", + "classrooms": "39627", + "billing_month_start_date": "2024-10-01", + "billing_month_end_date": "2024-10-01", + "custom_payments": null, + "archived": false, + "awarding_body": "", + "course_code": "", + "book_code": "", + "total_lessons": 2, + "total_lessons_hrs": "02:00", + "skype_meeting_link": "", + "year": null, + "credit_hours": "", + "class_type": "", + "is_ended": null, + "teacher_hourly_fees": null, + "is_booking_class": false, + "subscription_plan_id": null, + "is_stripe_sub_allow": 0, + "created_by": 66114, + "modified_by": 66114, + "exception_dates": null, + "removed_exception_dates": null +} \ No newline at end of file diff --git a/components/vectorshift/sources/new-knowledge-base/new-knowledge-base.mjs b/components/vectorshift/sources/new-knowledge-base/new-knowledge-base.mjs index 896860b4bfbbb..de0e4b4e0da05 100644 --- a/components/vectorshift/sources/new-knowledge-base/new-knowledge-base.mjs +++ b/components/vectorshift/sources/new-knowledge-base/new-knowledge-base.mjs @@ -1,132 +1,22 @@ -import { - axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, -} from "@pipedream/platform"; -import vectorshift from "../../vectorshift.app.mjs"; +import common from "../common/base.mjs"; +import sampleEmit from "./test-event.mjs"; export default { + ...common, key: "vectorshift-new-knowledge-base", name: "New Knowledge Base Created", - description: "Emit new event when a knowledge base is created in Vectorshift. [See the documentation]()", - version: "0.0.{{ts}}", + description: "Emit new event when a knowledge base is created in Vectorshift.", + version: "0.0.1", type: "source", dedupe: "unique", - props: { - vectorshift: { - type: "app", - app: "vectorshift", + methods: { + ...common.methods, + getFunction() { + return this.app.listKnowledgeBases; }, - db: { - type: "$.service.db", + getSummary(item) { + return `New Knowledge Base: ${item.name}`; }, - timer: { - type: "$.interface.timer", - default: { - intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, - }, - }, - }, - hooks: { - async deploy() { - try { - const knowledgeBases = await this.vectorshift.listKnowledgeBases(); - if (!knowledgeBases || knowledgeBases.length === 0) return; - - const sortedKnowledgeBases = knowledgeBases - .sort((a, b) => { - const aTs = a.created_at - ? Date.parse(a.created_at) - : 0; - const bTs = b.created_at - ? Date.parse(b.created_at) - : 0; - return bTs - aTs; - }) - .slice(0, 50); - - for (const kb of sortedKnowledgeBases.reverse()) { - this.$emit( - kb, - { - id: kb.id || Date.parse(kb.created_at) || Date.now(), - summary: `New Knowledge Base: ${kb.name}`, - ts: kb.created_at - ? Date.parse(kb.created_at) - : Date.now(), - }, - ); - } - - const latestTimestamp = sortedKnowledgeBases.reduce((max, kb) => { - const kbTs = kb.created_at - ? Date.parse(kb.created_at) - : 0; - return kbTs > max - ? kbTs - : max; - }, 0); - - await this.db.set("lastEmitTimestamp", latestTimestamp); - } catch (error) { - console.error("Error during deploy hook:", error); - } - }, - async activate() { - // No webhook subscription necessary for polling source - }, - async deactivate() { - // No webhook subscription to remove for polling source - }, - }, - async run() { - try { - const lastTimestamp = (await this.db.get("lastEmitTimestamp")) || 0; - const knowledgeBases = await this.vectorshift.listKnowledgeBases(); - if (!knowledgeBases || knowledgeBases.length === 0) return; - - const newKnowledgeBases = knowledgeBases - .filter((kb) => { - const kbTs = kb.created_at - ? Date.parse(kb.created_at) - : 0; - return kbTs > lastTimestamp; - }) - .sort((a, b) => { - const aTs = a.created_at - ? Date.parse(a.created_at) - : 0; - const bTs = b.created_at - ? Date.parse(b.created_at) - : 0; - return aTs - bTs; - }); - - for (const kb of newKnowledgeBases) { - const kbTimestamp = kb.created_at - ? Date.parse(kb.created_at) - : Date.now(); - this.$emit( - kb, - { - id: kb.id || kbTimestamp, - summary: `New Knowledge Base: ${kb.name}`, - ts: kbTimestamp, - }, - ); - } - - if (newKnowledgeBases.length > 0) { - const newLastTimestamp = newKnowledgeBases.reduce((max, kb) => { - const kbTs = kb.created_at - ? Date.parse(kb.created_at) - : 0; - return kbTs > max - ? kbTs - : max; - }, lastTimestamp); - await this.db.set("lastEmitTimestamp", newLastTimestamp); - } - } catch (error) { - console.error("Error during run method:", error); - } }, + sampleEmit, }; diff --git a/components/vectorshift/sources/new-knowledge-base/test-event.mjs b/components/vectorshift/sources/new-knowledge-base/test-event.mjs new file mode 100644 index 0000000000000..6c4f7457f87bf --- /dev/null +++ b/components/vectorshift/sources/new-knowledge-base/test-event.mjs @@ -0,0 +1,46 @@ +export default { + "course_title": "Class title", + "course_subject": "Course subject", + "course_level": "Course level", + "school_id": 16385, + "start_date": "2024-10-21", + "end_date": "2024-10-28", + "recurrence": "weekly", + "payment_frequency": "free", + "id": 153319, + "photo": null, + "payment_fee": "0", + "course_description": "", + "course_full_title": "Class title [Online Lesson]", + "created": "2024-10-17 18:20:13", + "modified": "2024-10-17 18:20:15", + "color": "color1", + "course_started": false, + "course_ended": false, + "course_status": 1, + "num_enrolled_students": 0, + "teachers": "66792", + "classrooms": "39627", + "billing_month_start_date": "2024-10-01", + "billing_month_end_date": "2024-10-01", + "custom_payments": null, + "archived": false, + "awarding_body": "", + "course_code": "", + "book_code": "", + "total_lessons": 2, + "total_lessons_hrs": "02:00", + "skype_meeting_link": "", + "year": null, + "credit_hours": "", + "class_type": "", + "is_ended": null, + "teacher_hourly_fees": null, + "is_booking_class": false, + "subscription_plan_id": null, + "is_stripe_sub_allow": 0, + "created_by": 66114, + "modified_by": 66114, + "exception_dates": null, + "removed_exception_dates": null +} \ No newline at end of file diff --git a/components/vectorshift/sources/new-pipeline/new-pipeline.mjs b/components/vectorshift/sources/new-pipeline/new-pipeline.mjs index 55e35f6fcc427..4075eaa66ede4 100644 --- a/components/vectorshift/sources/new-pipeline/new-pipeline.mjs +++ b/components/vectorshift/sources/new-pipeline/new-pipeline.mjs @@ -1,118 +1,22 @@ -import vectorshift from "../../vectorshift.app.mjs"; -import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import common from "../common/base.mjs"; +import sampleEmit from "./test-event.mjs"; export default { + ...common, key: "vectorshift-new-pipeline", name: "New Pipeline Created", - description: "Emit new event when a new pipeline is created in VectorShift. [See the documentation]()", - version: "0.0.{{ts}}", + description: "Emit new event when a new pipeline is created in VectorShift.", + version: "0.0.1", type: "source", dedupe: "unique", - props: { - vectorshift: { - type: "app", - app: "vectorshift", + methods: { + ...common.methods, + getFunction() { + return this.app.listPipelines; }, - db: "$.service.db", - timer: { - type: "$.interface.timer", - default: { - intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, - }, + getSummary(item) { + return `New Pipeline Created: ${item.name}`; }, }, - hooks: { - async deploy() { - try { - const pipelines = await this.vectorshift.listPipelines(); - if (!pipelines || pipelines.length === 0) { - return; - } - - // Sort pipelines by creation time descending - pipelines.sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); - - const recentPipelines = pipelines.slice(0, 50); - - let latestTimestamp = 0; - - for (const pipeline of recentPipelines) { - const timestamp = pipeline.created_at - ? Date.parse(pipeline.created_at) - : Date.now(); - latestTimestamp = Math.max(latestTimestamp, timestamp); - - this.$emit(pipeline, { - id: pipeline.id, - summary: `New Pipeline Created: ${pipeline.name}`, - ts: timestamp, - }); - } - - await this.db.set("last_timestamp", latestTimestamp); - } catch (error) { - this.$emit( - { - error: error.message, - }, - { - summary: "Error during deploy", - ts: Date.now(), - }, - ); - } - }, - async activate() { - // No activation steps needed for polling source - }, - async deactivate() { - // No deactivation steps needed for polling source - }, - }, - async run() { - try { - const pipelines = await this.vectorshift.listPipelines(); - if (!pipelines || pipelines.length === 0) { - return; - } - - const lastTimestamp = (await this.db.get("last_timestamp")) || 0; - - // Sort pipelines by creation time ascending - pipelines.sort((a, b) => new Date(a.created_at) - new Date(b.created_at)); - - let newLastTimestamp = lastTimestamp; - - for (const pipeline of pipelines) { - const pipelineTimestamp = pipeline.created_at - ? Date.parse(pipeline.created_at) - : Date.now(); - if (pipelineTimestamp > lastTimestamp) { - this.$emit(pipeline, { - id: pipeline.id, - summary: `New Pipeline Created: ${pipeline.name}`, - ts: pipelineTimestamp, - }); - - if (pipelineTimestamp > newLastTimestamp) { - newLastTimestamp = pipelineTimestamp; - } - } - } - - if (newLastTimestamp > lastTimestamp) { - await this.db.set("last_timestamp", newLastTimestamp); - } - } catch (error) { - this.$emit( - { - error: error.message, - }, - { - summary: "Error during run", - ts: Date.now(), - }, - ); - } - }, + sampleEmit, }; diff --git a/components/vectorshift/sources/new-pipeline/test-event.mjs b/components/vectorshift/sources/new-pipeline/test-event.mjs new file mode 100644 index 0000000000000..6c4f7457f87bf --- /dev/null +++ b/components/vectorshift/sources/new-pipeline/test-event.mjs @@ -0,0 +1,46 @@ +export default { + "course_title": "Class title", + "course_subject": "Course subject", + "course_level": "Course level", + "school_id": 16385, + "start_date": "2024-10-21", + "end_date": "2024-10-28", + "recurrence": "weekly", + "payment_frequency": "free", + "id": 153319, + "photo": null, + "payment_fee": "0", + "course_description": "", + "course_full_title": "Class title [Online Lesson]", + "created": "2024-10-17 18:20:13", + "modified": "2024-10-17 18:20:15", + "color": "color1", + "course_started": false, + "course_ended": false, + "course_status": 1, + "num_enrolled_students": 0, + "teachers": "66792", + "classrooms": "39627", + "billing_month_start_date": "2024-10-01", + "billing_month_end_date": "2024-10-01", + "custom_payments": null, + "archived": false, + "awarding_body": "", + "course_code": "", + "book_code": "", + "total_lessons": 2, + "total_lessons_hrs": "02:00", + "skype_meeting_link": "", + "year": null, + "credit_hours": "", + "class_type": "", + "is_ended": null, + "teacher_hourly_fees": null, + "is_booking_class": false, + "subscription_plan_id": null, + "is_stripe_sub_allow": 0, + "created_by": 66114, + "modified_by": 66114, + "exception_dates": null, + "removed_exception_dates": null +} \ No newline at end of file diff --git a/components/vectorshift/vectorshift.app.mjs b/components/vectorshift/vectorshift.app.mjs index ed69117e47d46..373230f87adad 100644 --- a/components/vectorshift/vectorshift.app.mjs +++ b/components/vectorshift/vectorshift.app.mjs @@ -3,238 +3,100 @@ import { axios } from "@pipedream/platform"; export default { type: "app", app: "vectorshift", - version: "0.0.{{ts}}", propDefinitions: { - // Create Pipeline Props - name: { - type: "string", - label: "Pipeline Name", - description: "Name of the new pipeline", - }, - config: { - type: "string", - label: "Pipeline Config", - description: "Configuration for the new pipeline as a JSON string", - }, - description: { + knowledgeBaseId: { type: "string", - label: "Description", - description: "Optional description of the new pipeline", - optional: true, + label: "Knowledge Base ID", + description: "The ID of the knowledge base", + async options() { + const { objects } = await this.listKnowledgeBases({ + params: { + verbose: true, + }, + }); + + return objects?.map(({ + _id: value, name: label, + }) => ({ + label, + value, + })) || []; + }, }, - // Execute Pipeline Props pipelineId: { type: "string", label: "Pipeline ID", description: "The ID of the pipeline to execute", async options() { - const pipelines = await this.listPipelines(); - if (!pipelines) return []; - return pipelines.map((pipeline) => ({ - label: pipeline.name, - value: pipeline.id, - })); - }, - }, - inputs: { - type: "string", - label: "Pipeline Inputs", - description: "Inputs for the pipeline execution as a JSON string", - }, - // Add Data to Knowledge Base Props - knowledgeBaseId: { - type: "string", - label: "Knowledge Base ID", - description: "The ID of the knowledge base", - async options() { - const knowledgeBases = await this.listKnowledgeBases(); - if (!knowledgeBases) return []; - return knowledgeBases.map((kb) => ({ - label: kb.name, - value: kb.id, + const { objects } = await this.listPipelines({ + params: { + verbose: true, + }, + }); + + return objects.map(({ + _id: value, name: label, + }) => ({ + label, + value, })); }, }, - fileData: { - type: "object", - label: "File Data", - description: "File data to add to the knowledge base", - optional: true, - }, - urlData: { - type: "object", - label: "URL Data", - description: "URL data to add to the knowledge base", - optional: true, - }, - wikipedia: { - type: "string", - label: "Wikipedia", - description: "Wikipedia data to add to the knowledge base", - optional: true, - }, - youtube: { - type: "string", - label: "YouTube", - description: "YouTube data to add to the knowledge base", - optional: true, - }, - arxiv: { - type: "string", - label: "ArXiv", - description: "ArXiv data to add to the knowledge base", - optional: true, - }, - git: { - type: "string", - label: "Git", - description: "Git data to add to the knowledge base", - optional: true, - }, - addDataConfig: { - type: "string", - label: "Add Data Config", - description: "Configuration for adding data to the knowledge base as a JSON string", - optional: true, - }, }, methods: { - // Existing Method - authKeys() { - console.log(Object.keys(this.$auth)); - }, - // Base URL Method _baseUrl() { return "https://api.vectorshift.ai/v1"; }, - // Make Request Method - async _makeRequest(opts = {}) { - const { - $ = this, method = "GET", path = "/", headers = {}, ...otherOpts - } = opts; + _headers() { + return { + Authorization: `Bearer ${this.$auth.api_key}`, + }; + }, + _makeRequest({ + $ = this, path, ...opts + }) { return axios($, { - method, url: this._baseUrl() + path, - headers: { - ...headers, - Authorization: `Bearer ${this.$auth.token}`, - }, - ...otherOpts, + headers: this._headers(), + ...opts, }); }, - // List Pipelines Method - async listPipelines(opts = {}) { - const response = await this._makeRequest({ + listPipelines(opts = {}) { + return this._makeRequest({ path: "/pipelines", - method: "GET", ...opts, }); - if (response.status !== "success") { - throw new Error(`Failed to list pipelines: ${response.status}`); - } - return response.objects; }, - // List Knowledge Bases Method - async listKnowledgeBases(opts = {}) { - const response = await this._makeRequest({ + listKnowledgeBases(opts = {}) { + return this._makeRequest({ path: "/knowledge-bases", - method: "GET", ...opts, }); - if (response.status !== "success") { - throw new Error(`Failed to list knowledge bases: ${response.status}`); - } - return response.objects; }, - // Create Pipeline Method - async createPipeline({ - name, config, description, - }) { - const data = { - name, - config: JSON.parse(config), - }; - if (description) { - data.description = description; - } - const response = await this._makeRequest({ - path: "/pipeline", + createPipeline(opts = {}) { + return this._makeRequest({ method: "POST", - data, + path: "/pipeline", + ...opts, }); - if (response.status !== "success") { - throw new Error(`Failed to create pipeline: ${response.status}`); - } - return response.id; }, - // Execute Pipeline Method - async executePipeline({ - pipelineId, inputs, + executePipeline({ + pipelineId, ...opts }) { - const data = { - inputs: JSON.parse(inputs), - }; - const response = await this._makeRequest({ - path: `/pipeline/${pipelineId}/run`, + return this._makeRequest({ method: "POST", - data, + path: `/pipeline/${pipelineId}/run`, + ...opts, }); - if (response.status !== "success") { - throw new Error(`Failed to execute pipeline: ${response.status}`); - } - return response.run_id; }, - // Add Data to Knowledge Base Method - async addDataToKnowledgeBase({ - knowledgeBaseId, fileData, urlData, wikipedia, youtube, arxiv, git, addDataConfig, + addDataToKnowledgeBase({ + knowledgeBaseId, ...opts }) { - if (!fileData && !urlData && !wikipedia && !youtube && !arxiv && !git) { - throw new Error("At least one of fileData, urlData, wikipedia, youtube, arxiv, or git must be provided."); - } - const data = {}; - if (fileData) data.file_data = fileData; - if (urlData) data.url_data = urlData; - if (wikipedia) data.wikipedia = wikipedia; - if (youtube) data.youtube = youtube; - if (arxiv) data.arxiv = arxiv; - if (git) data.git = git; - if (addDataConfig) data.config = JSON.parse(addDataConfig); - const response = await this._makeRequest({ - path: `/knowledge-base/${knowledgeBaseId}/index`, - method: "POST", - data, - }); - if (response.status !== "success") { - throw new Error(`Failed to add data to knowledge base: ${response.status}`); - } - return response.document_ids; - }, - // Create Knowledge Base Method - async createKnowledgeBase({ name }) { - const data = { - name, - }; - const response = await this._makeRequest({ - path: "/knowledge-bases/create", + return this._makeRequest({ method: "POST", - data, + path: `/knowledge-base/${knowledgeBaseId}/index`, + ...opts, }); - if (response.status !== "success") { - throw new Error(`Failed to create knowledge base: ${response.status}`); - } - return response.id; - }, - // Emit New Pipeline Created Event - async emitNewPipelineCreated(pipeline) { - // Implementation to emit event - }, - // Emit New Knowledge Base Created Event - async emitNewKnowledgeBaseCreated(knowledgeBase) { - // Implementation to emit event - }, - // Emit New Chatbot Created Event - async emitNewChatbotCreated(chatbot) { - // Implementation to emit event }, }, }; From bdce38d445fc90a162c99e0bbeb1922fd1b34e79 Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Wed, 26 Mar 2025 20:02:32 -0300 Subject: [PATCH 03/10] pnpm update --- pnpm-lock.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 54cf65bf430ae..42dbc4a763069 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5404,8 +5404,7 @@ importers: specifier: ^3.0.3 version: 3.0.3 - components/google_identity: - specifiers: {} + components/google_identity: {} components/google_maps_platform: dependencies: @@ -5852,8 +5851,7 @@ importers: specifier: ^1.5.1 version: 1.6.6 - components/helpdesk: - specifiers: {} + components/helpdesk: {} components/helper_functions: dependencies: @@ -13572,7 +13570,13 @@ importers: components/vectera: {} components/vectorshift: - specifiers: {} + dependencies: + '@pipedream/platform': + specifier: ^3.0.3 + version: 3.0.3 + fs: + specifier: ^0.0.1-security + version: 0.0.1-security components/vend: dependencies: @@ -34359,6 +34363,8 @@ snapshots: '@putout/operator-filesystem': 5.0.0(putout@36.13.1(eslint@8.57.1)(typescript@5.6.3)) '@putout/operator-json': 2.2.0 putout: 36.13.1(eslint@8.57.1)(typescript@5.6.3) + transitivePeerDependencies: + - supports-color '@putout/operator-regexp@1.0.0(putout@36.13.1(eslint@8.57.1)(typescript@5.6.3))': dependencies: From 0f0010577f81b68692238ffb2aa63edab81bd8a9 Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Wed, 26 Mar 2025 20:19:16 -0300 Subject: [PATCH 04/10] pnpm update --- pnpm-lock.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 45d0c43f7eadf..fbaef68ea71a2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41929,7 +41929,7 @@ snapshots: jest-util: 29.7.0 natural-compare: 1.4.0 pretty-format: 29.7.0 - semver: 7.6.3 + semver: 7.7.1 transitivePeerDependencies: - supports-color From 4cb8730fdc9fc3dd611d058e9b6789a52893d6c8 Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Wed, 26 Mar 2025 20:21:16 -0300 Subject: [PATCH 05/10] Update components/vectorshift/common/constants.mjs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- components/vectorshift/common/constants.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/vectorshift/common/constants.mjs b/components/vectorshift/common/constants.mjs index f2031e56e060b..0e174478f1f98 100644 --- a/components/vectorshift/common/constants.mjs +++ b/components/vectorshift/common/constants.mjs @@ -6,7 +6,7 @@ export const DATA_TYPE_OPTIONS = [ "git", ]; -export const RESCRAPE_FRANQUENCY_OPTIONS = [ +export const RESCRAPE_FREQUENCY_OPTIONS = [ "Never", "Hourly", "Daily", From f8454766a54d1656cdca7f93301e15f4b2dc6cdd Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Wed, 26 Mar 2025 20:21:34 -0300 Subject: [PATCH 06/10] Update components/vectorshift/sources/common/base.mjs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- components/vectorshift/sources/common/base.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/vectorshift/sources/common/base.mjs b/components/vectorshift/sources/common/base.mjs index b1334d2299af3..b8e1b0509c010 100644 --- a/components/vectorshift/sources/common/base.mjs +++ b/components/vectorshift/sources/common/base.mjs @@ -1,5 +1,5 @@ import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; -import app from "../../verctorshift.app.mjs"; +import app from "../../vectorshift.app.mjs"; export default { props: { From f56e9647b2401912b6ff0fa74accbd3f7f290db7 Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Wed, 26 Mar 2025 20:27:10 -0300 Subject: [PATCH 07/10] fix import --- .../add-data-to-knowledge-base/add-data-to-knowledge-base.mjs | 4 ++-- pnpm-lock.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs b/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs index 3bc088f5008f4..506ccb5a0ec55 100644 --- a/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs +++ b/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs @@ -1,5 +1,5 @@ import { - DATA_TYPE_OPTIONS, RESCRAPE_FRANQUENCY_OPTIONS, + DATA_TYPE_OPTIONS, RESCRAPE_FREQUENCY_OPTIONS, } from "../../common/constants.mjs"; import vectorshift from "../../vectorshift.app.mjs"; @@ -46,7 +46,7 @@ export default { type: "string", label: "Rescrape Frequency", description: "The frequency to rescrape the URL", - options: RESCRAPE_FRANQUENCY_OPTIONS, + options: RESCRAPE_FREQUENCY_OPTIONS, optional: true, hidden: true, }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fbaef68ea71a2..514ea7c6e61b8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44875,7 +44875,7 @@ snapshots: jsdoc: 4.0.4 minimist: 1.2.8 protobufjs: 7.2.4 - semver: 7.6.3 + semver: 7.7.1 tmp: 0.2.3 uglify-js: 3.19.3 From c77e15646f8a861f96948b180b65ce7dd68e75fe Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Mon, 31 Mar 2025 14:28:39 -0300 Subject: [PATCH 08/10] [Components] vectorshift #16018 Sources - New Pipeline - New Knowledge Base - New Chatbot Actions - Create Pipeline - Run Pipeline - Add Data To Knowledge Base --- .../add-data-to-knowledge-base.mjs | 72 +------ .../vectorshift/sources/common/base.mjs | 6 +- .../sources/new-chatbot/new-chatbot.mjs | 2 +- .../sources/new-chatbot/test-event.mjs | 187 +++++++++++++----- .../new-knowledge-base/new-knowledge-base.mjs | 2 +- .../sources/new-knowledge-base/test-event.mjs | 79 ++++---- .../sources/new-pipeline/new-pipeline.mjs | 2 +- .../sources/new-pipeline/test-event.mjs | 52 +---- components/vectorshift/vectorshift.app.mjs | 37 +++- 9 files changed, 227 insertions(+), 212 deletions(-) diff --git a/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs b/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs index 506ccb5a0ec55..ef60968f6eec0 100644 --- a/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs +++ b/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs @@ -1,6 +1,4 @@ -import { - DATA_TYPE_OPTIONS, RESCRAPE_FREQUENCY_OPTIONS, -} from "../../common/constants.mjs"; +import { RESCRAPE_FREQUENCY_OPTIONS } from "../../common/constants.mjs"; import vectorshift from "../../vectorshift.app.mjs"; export default { @@ -17,24 +15,10 @@ export default { "knowledgeBaseId", ], }, - dataType: { - type: "string", - label: "Data Type", - description: "The type of the data to be added.", - options: DATA_TYPE_OPTIONS, - reloadProps: true, - }, - file: { - type: "string", - label: "File Data", - description: "The file to be uploaded, please provide a file from `/tmp`. To upload a file to `/tmp` folder, [See the documentation](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", - hidden: true, - }, url: { type: "string", label: "URL", description: "URL to add to the knowledge base", - hidden: true, }, recursive: { type: "boolean", @@ -48,51 +32,13 @@ export default { description: "The frequency to rescrape the URL", options: RESCRAPE_FREQUENCY_OPTIONS, optional: true, - hidden: true, - }, - wikipedia: { - type: "string", - label: "Wikipedia", - description: "Wikipedia data to add to the knowledge base", - hidden: true, }, - youtube: { - type: "string", - label: "YouTube", - description: "YouTube data to add to the knowledge base", - hidden: true, - }, - arxiv: { - type: "string", - label: "ArXiv", - description: "ArXiv data to add to the knowledge base", - hidden: true, - }, - git: { - type: "string", - label: "Git", - description: "Git data to add to the knowledge base", - hidden: true, - }, - }, - async additionalProps(props) { - if (this.dataType) { - props.url.hidden = true; - props.wikipedia.hidden = true; - props.youtube.hidden = true; - props.arxiv.hidden = true; - props.git.hidden = true; - props[this.dataType].hidden = false; - - const isUrl = this.dataType === "url"; - props.rescrapeFrequency.hidden = !isUrl; - props.recursive.hidden = !isUrl; - } - return {}; }, async run({ $ }) { - let data = (this.dataType === "url") - ? { + const response = await this.vectorshift.addDataToKnowledgeBase({ + $, + knowledgeBaseId: this.knowledgeBaseId, + data: { url_data: { request: { url: this.url, @@ -101,13 +47,7 @@ export default { }, rescrape_frequency: this.rescrapeFrequency, }, - } - : this[this.dataType]; - - const response = await this.vectorshift.addDataToKnowledgeBase({ - $, - knowledgeBaseId: this.knowledgeBaseId, - data, + }, }); $.export("$summary", `Added ${response.document_ids.length} document(s) to knowledge base ${this.knowledgeBaseId}. Document IDs: ${response.document_ids.join(", ")}`); diff --git a/components/vectorshift/sources/common/base.mjs b/components/vectorshift/sources/common/base.mjs index b8e1b0509c010..2460b1ad3ad3b 100644 --- a/components/vectorshift/sources/common/base.mjs +++ b/components/vectorshift/sources/common/base.mjs @@ -14,7 +14,7 @@ export default { methods: { async emitEvent(maxResults = false) { const fn = this.getFunction(); - const { objects: response } = await fn(); + const { objects: response = [] } = await fn(); if (response.length) { if (maxResults && (response.length > maxResults)) { @@ -24,9 +24,9 @@ export default { for (const item of response) { this.$emit(item, { - id: item.id, + id: item._id, summary: this.getSummary(item), - ts: Date.parse(item.created || new Date()), + ts: Date.parse(item.createdDate || new Date()), }); } }, diff --git a/components/vectorshift/sources/new-chatbot/new-chatbot.mjs b/components/vectorshift/sources/new-chatbot/new-chatbot.mjs index f1ed5dfa132d8..120365303340a 100644 --- a/components/vectorshift/sources/new-chatbot/new-chatbot.mjs +++ b/components/vectorshift/sources/new-chatbot/new-chatbot.mjs @@ -15,7 +15,7 @@ export default { return this.app.listChatbots; }, getSummary(item) { - return `New Chatbot: ${item.name || "Unnamed"}`; + return `New Chatbot: ${item.name || item._id}`; }, }, sampleEmit, diff --git a/components/vectorshift/sources/new-chatbot/test-event.mjs b/components/vectorshift/sources/new-chatbot/test-event.mjs index 6c4f7457f87bf..0ec37a46370d0 100644 --- a/components/vectorshift/sources/new-chatbot/test-event.mjs +++ b/components/vectorshift/sources/new-chatbot/test-event.mjs @@ -1,46 +1,145 @@ export default { - "course_title": "Class title", - "course_subject": "Course subject", - "course_level": "Course level", - "school_id": 16385, - "start_date": "2024-10-21", - "end_date": "2024-10-28", - "recurrence": "weekly", - "payment_frequency": "free", - "id": 153319, - "photo": null, - "payment_fee": "0", - "course_description": "", - "course_full_title": "Class title [Online Lesson]", - "created": "2024-10-17 18:20:13", - "modified": "2024-10-17 18:20:15", - "color": "color1", - "course_started": false, - "course_ended": false, - "course_status": 1, - "num_enrolled_students": 0, - "teachers": "66792", - "classrooms": "39627", - "billing_month_start_date": "2024-10-01", - "billing_month_end_date": "2024-10-01", - "custom_payments": null, - "archived": false, - "awarding_body": "", - "course_code": "", - "book_code": "", - "total_lessons": 2, - "total_lessons_hrs": "02:00", - "skype_meeting_link": "", - "year": null, - "credit_hours": "", - "class_type": "", - "is_ended": null, - "teacher_hourly_fees": null, - "is_booking_class": false, - "subscription_plan_id": null, - "is_stripe_sub_allow": 0, - "created_by": 66114, - "modified_by": 66114, - "exception_dates": null, - "removed_exception_dates": null + "_id": "67ea096700fcfd7de5313ffa", + "access_config": { + "accessContactText": "Please contact the owner to request access", + "accessPermissionText": "You do not have permission to access this chatbot.", + "enablePassword": false, + "enableSSO": false, + "loginButtonText": "Submit", + "loginHeader": "Authenticate to Chatbot", + "loginPasswordText": "Password" + }, + "createdDate": "2025-03-31T17:24:39.768Z", + "deployed": true, + "deployment_options": { + "aIDisclaimerText": "Generated by AI. Double check for accuracy", + "aIDisclaimerVariant": "every_message", + "accentColor": "#6366F1", + "accessColor": "#6366F1", + "accountAuthToken": null, + "accountSid": null, + "assistantImageUrl": "https://vectorshift-public.s3.amazonaws.com/android-chrome-512x512.png", + "backgroundColor": "#FFFFFF", + "borderColor": "#1C2536", + "borderRadius": "8px", + "botName": "Assistant", + "botNameColor": "#6366F1", + "bottomBarIconUrl": "", + "bottomBarRedirectUrl": null, + "bottomBarText": null, + "bottomSpacing": "5px", + "chatBubbleUrl": "", + "chatOpen": true, + "chatbotPassword": "", + "chatbotWelcomeTitle": "Vectorshift", + "clearChat": false, + "copilotMaxWidth": "1000px", + "customDomains": [ + { + "id": "7c539459-ebf9-4d12-9c62-980bafc493ce", + "value": "" + } + ], + "deployed": false, + "disclaimerText": null, + "displayDescription": "Hi, how can I assist you today?", + "displayDescriptionColor": "#656d75", + "displayDescriptionFontSize": "14px", + "displayDescriptionFontWeight": 400, + "displayDescriptionTextAlignment": "center", + "displayImageSize": "80px", + "displayNameColor": "black", + "displayNameFontSize": "20px", + "displayNameFontWeight": 600, + "embedBorderRadius": "large", + "embedPosition": "right", + "errorMessage": null, + "followUpPrompt": "", + "fontSize": "16px", + "fontStyle": "inter", + "headerFont": "inter", + "height": "100%", + "iconUrl": "https://vectorshift-public.s3.amazonaws.com/android-chrome-512x512.png", + "inputFieldPlaceholder": "Write a message", + "inputIconHeight": "28px", + "inputMessageColor": "#EBEEFE", + "inputTextColor": "#000000", + "inputVariant": "single", + "inputboxHeight": "35px", + "integrationID": null, + "launcherBottomSpacing": 20, + "launcherMessage": "", + "launcherSideSpacing": 20, + "launcherSize": 48, + "launcherText": "Chat Launcher", + "launcherVariant": "icon", + "limitDomains": false, + "messageLimitPerConversation": null, + "numberOfRelatedQuestions": 3, + "oAuthToken": null, + "outputMessageColor": "#f1f2f2", + "outputTextColor": "#000000", + "persistenceTimeDuration": "week", + "persistenceTimeQuantity": 1, + "poweredByVectorshift": true, + "promptA": "", + "promptB": "", + "promptC": "", + "promptD": "", + "rbac": false, + "relatedText": "Related", + "responseLoaderIconUrl": "https://vectorshift-public.s3.amazonaws.com/chat-response-loader.svg", + "responseLoaderLabel": "Processing Request", + "saveConversations": false, + "secondaryColor": "#F5F7FF", + "sendBorderRadius": "100px", + "showAIDisclaimer": false, + "showDocumentUpload": false, + "showHeaderLogo": true, + "showLauncherBackgroundColor": true, + "showRemainingMessageInfo": true, + "showSelectedDocuments": true, + "showVoiceinChat": true, + "showWelcomeImage": true, + "slackDeployed": false, + "stopGeneration": true, + "suggestRelatedQuestions": true, + "suggestedMessages": false, + "title": "VectorShift", + "titleColor": "#000000", + "topBarColor": "#EBEEFE", + "topBarPadding": "12px", + "turnOnUserFeedback": true, + "userName": "User", + "userNameColor": "#6366F1", + "variant": "bubbles", + "welcomeDisplayImage": "https://vectorshift-public.s3.amazonaws.com/android-chrome-512x512.png", + "welcomeMessage": null, + "width": "100%" + }, + "description": "", + "input": "input_0", + "mainBranch": "67ea096700fcfd7de5313ffa", + "modifiedDate": "2025-03-31T17:24:39.768Z", + "name": "Chatbot Named", + "orgID": "Personal", + "output": "output_0", + "pipeline": { + "branch_id": null, + "object_id": "67ea096700fcfd7de5313ffa", + "object_type": 0, + "state_id": null, + "version": null + }, + "show_document_upload": false, + "slack_config": { + "integrationID": null, + "oAuthToken": null, + "slackDeployed": false + }, + "twilio_config": { + "accountAuthToken": null, + "accountSid": null + }, + "userID": "auth0|67ea096700fcfd7de5313ffa" } \ No newline at end of file diff --git a/components/vectorshift/sources/new-knowledge-base/new-knowledge-base.mjs b/components/vectorshift/sources/new-knowledge-base/new-knowledge-base.mjs index de0e4b4e0da05..6bc6e95a46ee5 100644 --- a/components/vectorshift/sources/new-knowledge-base/new-knowledge-base.mjs +++ b/components/vectorshift/sources/new-knowledge-base/new-knowledge-base.mjs @@ -15,7 +15,7 @@ export default { return this.app.listKnowledgeBases; }, getSummary(item) { - return `New Knowledge Base: ${item.name}`; + return `New Knowledge Base: ${item.name || item._id}`; }, }, sampleEmit, diff --git a/components/vectorshift/sources/new-knowledge-base/test-event.mjs b/components/vectorshift/sources/new-knowledge-base/test-event.mjs index 6c4f7457f87bf..395913b6d0f0c 100644 --- a/components/vectorshift/sources/new-knowledge-base/test-event.mjs +++ b/components/vectorshift/sources/new-knowledge-base/test-event.mjs @@ -1,46 +1,37 @@ export default { - "course_title": "Class title", - "course_subject": "Course subject", - "course_level": "Course level", - "school_id": 16385, - "start_date": "2024-10-21", - "end_date": "2024-10-28", - "recurrence": "weekly", - "payment_frequency": "free", - "id": 153319, - "photo": null, - "payment_fee": "0", - "course_description": "", - "course_full_title": "Class title [Online Lesson]", - "created": "2024-10-17 18:20:13", - "modified": "2024-10-17 18:20:15", - "color": "color1", - "course_started": false, - "course_ended": false, - "course_status": 1, - "num_enrolled_students": 0, - "teachers": "66792", - "classrooms": "39627", - "billing_month_start_date": "2024-10-01", - "billing_month_end_date": "2024-10-01", - "custom_payments": null, - "archived": false, - "awarding_body": "", - "course_code": "", - "book_code": "", - "total_lessons": 2, - "total_lessons_hrs": "02:00", - "skype_meeting_link": "", - "year": null, - "credit_hours": "", - "class_type": "", - "is_ended": null, - "teacher_hourly_fees": null, - "is_booking_class": false, - "subscription_plan_id": null, - "is_stripe_sub_allow": 0, - "created_by": 66114, - "modified_by": 66114, - "exception_dates": null, - "removed_exception_dates": null + "_id": "7667e40a6c748705592c629b", + "analyzeDocuments": false, + "apifyKey": "", + "chunkOverlap": 0, + "chunkSize": 400, + "createdDate": "2025-03-26T21:41:27.705Z", + "description": "", + "fileProcessingImplementation": "default", + "isHybrid": false, + "modifiedDate": "2025-03-31T16:22:01.405Z", + "name": "Knowledge Base Name", + "orgID": "Personal", + "segmentationMethod": "words", + "sharedWith": [ + { + "id": "auth0|7667e40a6c748705592c629b", + "item_type": "USER", + "org_id": "Personal", + "role": "Owner" + } + ], + "splitterMethod": "markdown", + "userID": "auth0|7667e40a6c748705592c629b", + "vector_db_details": { + "collection_name": "text-embedding-3-small", + "dimension": null, + "embedding_model": "text-embedding-3-small", + "embedding_provider": "openai", + "precision": "Float16", + "query_embedding_model": null, + "sharded": true, + "sparse_embedding_model": null, + "sparse_query_embedding_model": null, + "vector_db_provider": "qdrant" + } } \ No newline at end of file diff --git a/components/vectorshift/sources/new-pipeline/new-pipeline.mjs b/components/vectorshift/sources/new-pipeline/new-pipeline.mjs index 4075eaa66ede4..3f791fc000a27 100644 --- a/components/vectorshift/sources/new-pipeline/new-pipeline.mjs +++ b/components/vectorshift/sources/new-pipeline/new-pipeline.mjs @@ -15,7 +15,7 @@ export default { return this.app.listPipelines; }, getSummary(item) { - return `New Pipeline Created: ${item.name}`; + return `New Pipeline Created: ${item.name || item._id}`; }, }, sampleEmit, diff --git a/components/vectorshift/sources/new-pipeline/test-event.mjs b/components/vectorshift/sources/new-pipeline/test-event.mjs index 6c4f7457f87bf..798152185773b 100644 --- a/components/vectorshift/sources/new-pipeline/test-event.mjs +++ b/components/vectorshift/sources/new-pipeline/test-event.mjs @@ -1,46 +1,10 @@ export default { - "course_title": "Class title", - "course_subject": "Course subject", - "course_level": "Course level", - "school_id": 16385, - "start_date": "2024-10-21", - "end_date": "2024-10-28", - "recurrence": "weekly", - "payment_frequency": "free", - "id": 153319, - "photo": null, - "payment_fee": "0", - "course_description": "", - "course_full_title": "Class title [Online Lesson]", - "created": "2024-10-17 18:20:13", - "modified": "2024-10-17 18:20:15", - "color": "color1", - "course_started": false, - "course_ended": false, - "course_status": 1, - "num_enrolled_students": 0, - "teachers": "66792", - "classrooms": "39627", - "billing_month_start_date": "2024-10-01", - "billing_month_end_date": "2024-10-01", - "custom_payments": null, - "archived": false, - "awarding_body": "", - "course_code": "", - "book_code": "", - "total_lessons": 2, - "total_lessons_hrs": "02:00", - "skype_meeting_link": "", - "year": null, - "credit_hours": "", - "class_type": "", - "is_ended": null, - "teacher_hourly_fees": null, - "is_booking_class": false, - "subscription_plan_id": null, - "is_stripe_sub_allow": 0, - "created_by": 66114, - "modified_by": 66114, - "exception_dates": null, - "removed_exception_dates": null + "_id": "6709e0f530eacae6703df110", + "createdDate": "2025-03-31T17:03:31.277Z", + "mainBranch": "6709e0f530eacae6703df110", + "modifiedDate": "2025-03-31T17:03:31.277Z", + "name": "Pipeline Name", + "nodes": {}, + "orgID": "Personal", + "userID": "auth0|6709e0f530eacae6703df110" } \ No newline at end of file diff --git a/components/vectorshift/vectorshift.app.mjs b/components/vectorshift/vectorshift.app.mjs index 373230f87adad..af4ee7bdbcabd 100644 --- a/components/vectorshift/vectorshift.app.mjs +++ b/components/vectorshift/vectorshift.app.mjs @@ -28,11 +28,7 @@ export default { label: "Pipeline ID", description: "The ID of the pipeline to execute", async options() { - const { objects } = await this.listPipelines({ - params: { - verbose: true, - }, - }); + const { objects } = await this.listPipelines(); return objects.map(({ _id: value, name: label, @@ -49,7 +45,8 @@ export default { }, _headers() { return { - Authorization: `Bearer ${this.$auth.api_key}`, + "Authorization": `Bearer ${this.$auth.api_key}`, + "Content-Type": "application/json", }; }, _makeRequest({ @@ -61,15 +58,39 @@ export default { ...opts, }); }, - listPipelines(opts = {}) { + listPipelines({ + params = {}, ...opts + } = {}) { return this._makeRequest({ path: "/pipelines", + params: { + verbose: true, + ...params, + }, ...opts, }); }, - listKnowledgeBases(opts = {}) { + listKnowledgeBases({ + params = {}, ...opts + } = {}) { return this._makeRequest({ path: "/knowledge-bases", + params: { + verbose: true, + ...params, + }, + ...opts, + }); + }, + listChatbots({ + params = {}, ...opts + } = {}) { + return this._makeRequest({ + path: "/chatbots", + params: { + verbose: true, + ...params, + }, ...opts, }); }, From 6a564ffa0fa1fa9641a475d14d0ed653c88094a6 Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Mon, 31 Mar 2025 14:49:22 -0300 Subject: [PATCH 09/10] pnpm update --- pnpm-lock.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 514ea7c6e61b8..378470341c641 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16401,12 +16401,12 @@ packages: '@dabh/diagnostics@2.0.3': resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==} - '@definitelytyped/header-parser@0.2.17': - resolution: {integrity: sha512-U0juKFkTOcbkSfO83WSzMEJHYDwoBFiq0tf/JszulL3+7UoSiqunpGmxXS54bm3eGqy7GWjV8AqPQHdeoEaWBQ==} + '@definitelytyped/header-parser@0.2.18': + resolution: {integrity: sha512-3JWGzhieGOx+zhy+qaPDoiby2TPA1PZGpEJHt0VwR1aK0R9dER5BoBvnT5zSafg9kHQTw4aBRFbt3o41FNkaLw==} engines: {node: '>=18.18.0'} - '@definitelytyped/typescript-versions@0.1.7': - resolution: {integrity: sha512-sBzBi1SBn79OkSr8V0H+FzR7QumHk23syPyRxod/VRBrSkgN9rCliIe+nqLoWRAKN8EeKbp00ketnJNLZhucdA==} + '@definitelytyped/typescript-versions@0.1.8': + resolution: {integrity: sha512-iz6q9aTwWW7CzN2g8jFQfZ955D63LA+wdIAKz4+2pCc/7kokmEHie1/jVWSczqLFOlmH+69bWQxIurryBP/sig==} engines: {node: '>=18.18.0'} '@definitelytyped/utils@0.1.8': @@ -32147,13 +32147,13 @@ snapshots: enabled: 2.0.0 kuler: 2.0.0 - '@definitelytyped/header-parser@0.2.17': + '@definitelytyped/header-parser@0.2.18': dependencies: - '@definitelytyped/typescript-versions': 0.1.7 + '@definitelytyped/typescript-versions': 0.1.8 '@definitelytyped/utils': 0.1.8 semver: 7.7.1 - '@definitelytyped/typescript-versions@0.1.7': {} + '@definitelytyped/typescript-versions@0.1.8': {} '@definitelytyped/utils@0.1.8': dependencies: @@ -38795,7 +38795,7 @@ snapshots: dts-critic@3.3.11(typescript@5.7.2): dependencies: - '@definitelytyped/header-parser': 0.2.17 + '@definitelytyped/header-parser': 0.2.18 command-exists: 1.2.9 rimraf: 3.0.2 semver: 6.3.1 @@ -38805,8 +38805,8 @@ snapshots: dtslint@4.2.1(typescript@5.7.2): dependencies: - '@definitelytyped/header-parser': 0.2.17 - '@definitelytyped/typescript-versions': 0.1.7 + '@definitelytyped/header-parser': 0.2.18 + '@definitelytyped/typescript-versions': 0.1.8 '@definitelytyped/utils': 0.1.8 dts-critic: 3.3.11(typescript@5.7.2) fs-extra: 6.0.1 From 4c0b3abd556b6b7042e5e9495d462222866833a8 Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Wed, 2 Apr 2025 12:20:43 -0300 Subject: [PATCH 10/10] some adjusts --- .../add-data-to-knowledge-base/add-data-to-knowledge-base.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs b/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs index ef60968f6eec0..f2eab6d5a0cd1 100644 --- a/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs +++ b/components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs @@ -31,7 +31,7 @@ export default { label: "Rescrape Frequency", description: "The frequency to rescrape the URL", options: RESCRAPE_FREQUENCY_OPTIONS, - optional: true, + default: RESCRAPE_FREQUENCY_OPTIONS[0], }, }, async run({ $ }) {