New Components - vectorshift

components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs

@@ -0,0 +1,116 @@
+import {
+  DATA_TYPE_OPTIONS, RESCRAPE_FRANQUENCY_OPTIONS,
+} from "../../common/constants.mjs";
+import vectorshift from "../../vectorshift.app.mjs";
+
+export default {
+  key: "vectorshift-add-data-to-knowledge-base",
+  name: "Add Data to Knowledge Base",
+  description: "Adds data to a knowledge base in VectorShift. [See the documentation](https://docs.vectorshift.ai/api-reference/knowledge-bases/index).",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    vectorshift,
+    knowledgeBaseId: {
+      propDefinition: [
+        vectorshift,
+        "knowledgeBaseId",
+      ],
+    },
+    dataType: {
+      type: "string",
+      label: "Data Type",
+      description: "The type of the data to be added.",
+      options: DATA_TYPE_OPTIONS,
+      reloadProps: true,
+    },
+    file: {
+      type: "string",
+      label: "File Data",
+      description: "The file to be uploaded, please provide a file from `/tmp`. To upload a file to `/tmp` folder, [See the documentation](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)",
+      hidden: true,
+    },
+    url: {
+      type: "string",
+      label: "URL",
+      description: "URL to add to the knowledge base",
+      hidden: true,
+    },
+    recursive: {
+      type: "boolean",
+      label: "Recursive",
+      description: "Whether the scrape is recursive or not",
+      default: false,
+    },
+    rescrapeFrequency: {
+      type: "string",
+      label: "Rescrape Frequency",
+      description: "The frequency to rescrape the URL",
+      options: RESCRAPE_FRANQUENCY_OPTIONS,
+      optional: true,
+      hidden: true,
+    },
+    wikipedia: {
+      type: "string",
+      label: "Wikipedia",
+      description: "Wikipedia data to add to the knowledge base",
+      hidden: true,
+    },
+    youtube: {
+      type: "string",
+      label: "YouTube",
+      description: "YouTube data to add to the knowledge base",
+      hidden: true,
+    },
+    arxiv: {
+      type: "string",
+      label: "ArXiv",
+      description: "ArXiv data to add to the knowledge base",
+      hidden: true,
+    },
+    git: {
+      type: "string",
+      label: "Git",
+      description: "Git data to add to the knowledge base",
+      hidden: true,
+    },
+  },
+  async additionalProps(props) {
+    if (this.dataType) {
+      props.url.hidden = true;
+      props.wikipedia.hidden = true;
+      props.youtube.hidden = true;
+      props.arxiv.hidden = true;
+      props.git.hidden = true;
+      props[this.dataType].hidden = false;
+
+      const isUrl = this.dataType === "url";
+      props.rescrapeFrequency.hidden = !isUrl;
+      props.recursive.hidden = !isUrl;
+    }
+    return {};
+  },
+  async run({ $ }) {
+    let data = (this.dataType === "url")
+      ? {
+        url_data: {
+          request: {
+            url: this.url,
+            recursive: this.recursive,
+            return_type: "CONTENT",
+          },
+          rescrape_frequency: this.rescrapeFrequency,
+        },
+      }
+      : this[this.dataType];
+
+    const response = await this.vectorshift.addDataToKnowledgeBase({
+      $,
+      knowledgeBaseId: this.knowledgeBaseId,
+      data,
+    });
+
+    $.export("$summary", `Added ${response.document_ids.length} document(s) to knowledge base ${this.knowledgeBaseId}. Document IDs: ${response.document_ids.join(", ")}`);
+    return response;
+  },
+};

components/vectorshift/actions/create-pipeline/create-pipeline.mjs

@@ -0,0 +1,48 @@
+import { ConfigurationError } from "@pipedream/platform";
+import { parseObject } from "../../common/utils.mjs";
+import vectorshift from "../../vectorshift.app.mjs";
+
+export default {
+  key: "vectorshift-create-pipeline",
+  name: "Create Pipeline",
+  description: "Creates a new pipeline in VectorShift. [See the documentation](https://docs.vectorshift.ai)",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    vectorshift,
+    name: {
+      type: "string",
+      label: "Pipeline Name",
+      description: "Name of the new pipeline",
+    },
+    config: {
+      type: "object",
+      label: "Pipeline Config",
+      description: "Configuration for the new pipeline",
+    },
+    description: {
+      type: "string",
+      label: "Description",
+      description: "Optional description of the new pipeline",
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    try {
+      const response = await this.vectorshift.createPipeline({
+        $,
+        data: {
+          name: this.name,
+          config: parseObject(this.config),
+          description: this.description,
+        },
+      });
+
+      $.export("$summary", `Created pipeline with ID ${response.id}`);
+      return response;
+    } catch ({ message }) {
+      const parsedError = JSON.parse(message).error;
+      throw new ConfigurationError(parsedError);
+    }
+  },
+};

components/vectorshift/actions/run-pipeline/run-pipeline.mjs

@@ -0,0 +1,42 @@
+import { ConfigurationError } from "@pipedream/platform";
+import { parseObject } from "../../common/utils.mjs";
+import vectorshift from "../../vectorshift.app.mjs";
+
+export default {
+  key: "vectorshift-run-pipeline",
+  name: "Run Pipeline",
+  description: "Executes a VectorShift pipeline with specified inputs. [See the documentation](https://docs.vectorshift.ai/api-reference/pipelines/run)",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    vectorshift,
+    pipelineId: {
+      propDefinition: [
+        vectorshift,
+        "pipelineId",
+      ],
+    },
+    inputs: {
+      type: "object",
+      label: "Pipeline Inputs",
+      description: "Inputs for the pipeline execution. [See the documentation](https://docs.vectorshift.ai/platform/pipelines/general/input) for further details",
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    try {
+      const response = await this.vectorshift.executePipeline({
+        $,
+        pipelineId: this.pipelineId,
+        data: {
+          inputs: parseObject(this.inputs),
+        },
+      });
+      $.export("$summary", `Pipeline executed successfully. Run ID: ${response.run_id}`);
+      return response;
+    } catch ({ message }) {
+      const parsedError = JSON.parse(message).error;
+      throw new ConfigurationError(parsedError);
+    }
+  },
+};

components/vectorshift/common/constants.mjs

@@ -0,0 +1,15 @@
+export const DATA_TYPE_OPTIONS = [
+  "url",
+  "wikipedia",
+  "youtube",
+  "arxiv",
+  "git",
+];
+
+export const RESCRAPE_FRANQUENCY_OPTIONS = [
+  "Never",
+  "Hourly",
+  "Daily",
+  "Weekly",
+  "Monthly",
+];

components/vectorshift/common/utils.mjs

@@ -0,0 +1,31 @@
+export const checkTmp = (filename) => {
+  if (!filename.startsWith("/tmp")) {
+    return `/tmp/${filename}`;
+  }
+  return filename;
+};
+
+export const parseObject = (obj) => {
+  if (!obj) return undefined;
+
+  if (Array.isArray(obj)) {
+    return obj.map((item) => {
+      if (typeof item === "string") {
+        try {
+          return JSON.parse(item);
+        } catch (e) {
+          return item;
+        }
+      }
+      return item;
+    });
+  }
+  if (typeof obj === "string") {
+    try {
+      return JSON.parse(obj);
+    } catch (e) {
+      return obj;
+    }
+  }
+  return obj;
+};

components/vectorshift/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/vectorshift",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "description": "Pipedream VectorShift Components",
   "main": "vectorshift.app.mjs",
   "keywords": [
@@ -11,5 +11,9 @@
   "author": "Pipedream <[email protected]> (https://pipedream.com/)",
   "publishConfig": {
     "access": "public"
+  },
+  "dependencies": {
+    "@pipedream/platform": "^3.0.3",
+    "fs": "^0.0.1-security"
   }
-}
+}

components/vectorshift/sources/common/base.mjs

@@ -0,0 +1,42 @@
+import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
+import app from "../../verctorshift.app.mjs";
+
+export default {
+  props: {
+    app,
+    timer: {
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
+      },
+    },
+  },
+  methods: {
+    async emitEvent(maxResults = false) {
+      const fn = this.getFunction();
+      const { objects: response } = await fn();
+
+      if (response.length) {
+        if (maxResults && (response.length > maxResults)) {
+          response.length = maxResults;
+        }
+      }
+
+      for (const item of response) {
+        this.$emit(item, {
+          id: item.id,
+          summary: this.getSummary(item),
+          ts: Date.parse(item.created || new Date()),
+        });
+      }
+    },
+  },
+  hooks: {
+    async deploy() {
+      await this.emitEvent(25);
+    },
+  },
+  async run() {
+    await this.emitEvent();
+  },
+};

components/vectorshift/sources/new-chatbot/new-chatbot.mjs

@@ -0,0 +1,22 @@
+import common from "../common/base.mjs";
+import sampleEmit from "./test-event.mjs";
+
+export default {
+  ...common,
+  key: "vectorshift-new-chatbot",
+  name: "New Chatbot Created",
+  description: "Emit new event when a chatbot is created.",
+  version: "0.0.1",
+  type: "source",
+  dedupe: "unique",
+  methods: {
+    ...common.methods,
+    getFunction() {
+      return this.app.listChatbots;
+    },
+    getSummary(item) {
+      return `New Chatbot: ${item.name || "Unnamed"}`;
+    },
+  },
+  sampleEmit,
+};

components/vectorshift/sources/new-chatbot/test-event.mjs

@@ -0,0 +1,46 @@
+export default {
+  "course_title": "Class title",
+  "course_subject": "Course subject",
+  "course_level": "Course level",
+  "school_id": 16385,
+  "start_date": "2024-10-21",
+  "end_date": "2024-10-28",
+  "recurrence": "weekly",
+  "payment_frequency": "free",
+  "id": 153319,
+  "photo": null,
+  "payment_fee": "0",
+  "course_description": "",
+  "course_full_title": "Class title [Online Lesson]",
+  "created": "2024-10-17 18:20:13",
+  "modified": "2024-10-17 18:20:15",
+  "color": "color1",
+  "course_started": false,
+  "course_ended": false,
+  "course_status": 1,
+  "num_enrolled_students": 0,
+  "teachers": "66792",
+  "classrooms": "39627",
+  "billing_month_start_date": "2024-10-01",
+  "billing_month_end_date": "2024-10-01",
+  "custom_payments": null,
+  "archived": false,
+  "awarding_body": "",
+  "course_code": "",
+  "book_code": "",
+  "total_lessons": 2,
+  "total_lessons_hrs": "02:00",
+  "skype_meeting_link": "",
+  "year": null,
+  "credit_hours": "",
+  "class_type": "",
+  "is_ended": null,
+  "teacher_hourly_fees": null,
+  "is_booking_class": false,
+  "subscription_plan_id": null,
+  "is_stripe_sub_allow": 0,
+  "created_by": 66114,
+  "modified_by": 66114,
+  "exception_dates": null,
+  "removed_exception_dates": null
+}