APIs - 1.1.0-beta

.github/workflows/pull_request.yaml

@@ -24,5 +24,4 @@ jobs:
           cd api-service
           npm install
           npm run actions:test
-          npm run actions:test:v2
           npm run lint

api-service/.eslintignore

@@ -4,6 +4,6 @@ docs
 coverage
 @types
 .nyc_output
-src/v2/tests
+src/tests
 src/v1
 dist

api-service/.eslintrc

@@ -9,7 +9,8 @@
         "plugin:@typescript-eslint/eslint-recommended",
         "plugin:@typescript-eslint/recommended"
     ],
-    "rules": {    
+    "rules": {
+        "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }], 
         "@typescript-eslint/no-explicit-any": ["off"],
         "@typescript-eslint/no-useless-escape": ["off"],
         "@typescript-eslint/quotes": [

api-service/package.json

@@ -5,9 +5,8 @@
   "main": "dist/app.js",
   "scripts": {
     "start": "ts-node ./src/app.ts",
-    "test": "source .env.test && nyc mocha ./src/v1/test/*.spec.ts --exit && nyc mocha ./src/v2/tests/**/*.spec.ts --exit",
-    "actions:test": "nyc mocha ./src/v1/test/*.spec.ts --exit",
-    "actions:test:v2": "nyc mocha ./src/v2/tests/**/*.spec.ts --exit",
+    "test": "source .env.test && nyc mocha ./src/tests/**/*.spec.ts --exit",
+    "actions:test": "nyc mocha ./src/tests/**/*.spec.ts --exit",
     "build": "rm -rf dist && tsc --declaration -P . && cp package.json ./dist/package.json",
     "package": "npm run build && cd dist && npm pack . && cd ..",
     "lint": "eslint . --ext .ts",

api-service/src/app.ts

@@ -8,13 +8,16 @@ import { errorHandler, obsrvErrorHandler } from "./middlewares/errors";
 import { ResponseHandler } from "./helpers/ResponseHandler";
 import { config } from "./configs/Config";
 import { alertsRouter } from "./routes/AlertsRouter";
+import { interceptAuditEvents } from "./services/telemetry";
 
 const app: Application = express();
+
 app.use(bodyParser.json({ limit: config.body_parser_limit}));
 app.use(express.text());
 app.use(express.json());
 app.use(errorHandler)
 
+app.use(interceptAuditEvents());
 app.use("/v2/", v2Router);
 app.use("/", druidProxyRouter);
 app.use("/alerts/v1", alertsRouter);

api-service/src/configs/Config.ts

@@ -109,7 +109,7 @@ export const config = {
     "encryption_algorithm": process.env.encryption_algorithm || "aes-256-ecb",
   },
   "grafana_config": {
-    "dialect": process.env.dialet || 'postgres',
+    "dialect": process.env.dialet || "postgres",
     "url": process.env.grafana_url || "http://localhost:8000",
     "access_token": process.env.grafana_token || ""
   }

api-service/src/connections/grafanaConnection.ts

@@ -5,6 +5,6 @@ const grafanaHttpClient = axios.create({
   baseURL: config.grafana_config.url
 });
 
-grafanaHttpClient.defaults.headers.common['Authorization'] = config.grafana_config.access_token;
+grafanaHttpClient.defaults.headers.common["Authorization"] = config.grafana_config.access_token;
 
 export { grafanaHttpClient };

api-service/src/controllers/Alerts/Alerts.ts

@@ -17,9 +17,9 @@ const createAlertHandler = async (req: Request, res: Response, next: NextFunctio
     updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } });
     ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } });
   } catch (error: any) {
-    let errorMessage = _.get(error, 'message')
-    if (_.get(error, 'name') == "SequelizeUniqueConstraintError") {
-      errorMessage = _.get(error, 'parent.detail')
+    let errorMessage = _.get(error, "message")
+    if (_.get(error, "name") == "SequelizeUniqueConstraintError") {
+      errorMessage = _.get(error, "parent.detail")
     }
     next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage })))
   }
@@ -44,7 +44,7 @@ const publishAlertHandler = async (req: Request, res: Response, next: NextFuncti
 
 const transformAlerts = async (alertModel: any) => {
   const alert = alertModel?.toJSON();
-  const status = _.get(alert, 'status');
+  const status = _.get(alert, "status");
   if (status !== "live") return alert;
   return getAlertsMetadata(alert);
 }
@@ -108,13 +108,13 @@ const updateAlertHandler = async (req: Request, res: Response, next: NextFunctio
       await retireAlertSilence(alertId);
     }
     const updatedPayload = getAlertPayload({ ...req.body, manager: rulePayload?.manager });
-    await Alert.update({ ...updatedPayload, status: 'draft' }, { where: { id: alertId } });
+    await Alert.update({ ...updatedPayload, status: "draft" }, { where: { id: alertId } });
     updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } });
     ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } });
   } catch (error: any) {
-    let errorMessage = _.get(error, 'message')
-    if (_.get(error, 'name') == "SequelizeUniqueConstraintError") {
-      errorMessage = _.get(error, 'parent.detail')
+    let errorMessage = _.get(error, "message")
+    if (_.get(error, "name") == "SequelizeUniqueConstraintError") {
+      errorMessage = _.get(error, "parent.detail")
     }
     next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage })))
   }

api-service/src/controllers/Alerts/Metric.ts

@@ -10,15 +10,15 @@ const telemetryObject = { type: "metric", ver: "1.0.0" };
 
 const createMetricHandler = async (req: Request, res: Response, next: NextFunction) => {
     try {
-        const { component } = req?.body;
+        const { component } = req.body;
         const transformComponent = _.toLower(component);
         const metricsBody = await Metrics.create({ ...(req.body), component: transformComponent });
         updateTelemetryAuditEvent({ request: req, object: { id: metricsBody?.dataValues?.id, ...telemetryObject } });
         ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: metricsBody.dataValues.id } });
     } catch (error: any) {
-        let errorMessage = _.get(error, 'message')
-        if (_.get(error, 'name') == "SequelizeUniqueConstraintError") {
-            errorMessage = _.get(error, 'parent.detail')
+        let errorMessage = _.get(error, "message")
+        if (_.get(error, "name") == "SequelizeUniqueConstraintError") {
+            errorMessage = _.get(error, "parent.detail")
         }
         next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage })))
     }
@@ -30,7 +30,7 @@ const listMetricsHandler = async (req: Request, res: Response, next: NextFunctio
         const metricsPayload = await Metrics.findAll({ limit: limit, offset: offset, ...(filters && { where: filters }) });
         ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { metrics: metricsPayload, count: metricsPayload.length } });
     } catch (error) {
-        const errorMessage = _.get(error, 'message')
+        const errorMessage = _.get(error, "message")
         next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage })))
     }
 }
@@ -50,9 +50,9 @@ const updateMetricHandler = async (req: Request, res: Response, next: NextFuncti
         });
         ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id } });
     } catch (error) {
-        let errorMessage = _.get(error, 'message')
-        if (_.get(error, 'name') == "SequelizeUniqueConstraintError") {
-            errorMessage = _.get(error, 'parent.detail')
+        let errorMessage = _.get(error, "message")
+        if (_.get(error, "name") == "SequelizeUniqueConstraintError") {
+            errorMessage = _.get(error, "parent.detail")
         }
         next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage })))
     }
@@ -66,7 +66,7 @@ const deleteMetricHandler = async (req: Request, res: Response, next: NextFuncti
         await record.destroy();
         ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id } });
     } catch (error) {
-        const errorMessage = _.get(error, 'message')
+        const errorMessage = _.get(error, "message")
         next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage })))
     }
 }
@@ -78,7 +78,7 @@ const deleteMultipleMetricHandler = async (req: Request, res: Response, next: Ne
         await Metrics.destroy({ where: filters });
         ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: {} });
     } catch (error) {
-        const errorMessage = _.get(error, 'message')
+        const errorMessage = _.get(error, "message")
         next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage })))
     }
 }

api-service/src/controllers/Alerts/Silence.ts

@@ -18,8 +18,8 @@ const createHandler = async (request: Request, response: Response, next: NextFun
         const grafanaResponse = await createSilence(payload);
         if (!grafanaResponse) return next({ message: httpStatus[httpStatus.INTERNAL_SERVER_ERROR], statusCode: httpStatus.INTERNAL_SERVER_ERROR })
 
-        let start_date = new Date(startDate);
-        let end_date = new Date(endDate);
+        const start_date = new Date(startDate);
+        const end_date = new Date(endDate);
         const silenceBody = {
             id: grafanaResponse.silenceId,
             manager: grafanaResponse.manager,
@@ -31,7 +31,7 @@ const createHandler = async (request: Request, response: Response, next: NextFun
         updateTelemetryAuditEvent({ request, object: { id: sileneResponse?.dataValues?.id, ...telemetryObject } });
         ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: sileneResponse.dataValues.id } })
     } catch (err) {
-        const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR])
+        const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR])
         next(error);
     }
 }
@@ -45,11 +45,11 @@ const transformSilences = async (silenceModel: any) => {
 const listHandler = async (request: Request, response: Response, next: NextFunction) => {
     try {
         const silences = await Silence.findAll();
-        const count = _.get(silences, 'length');
+        const count = _.get(silences, "length");
         const transformedSilences = await Promise.all(silences.map(transformSilences));
         ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { transformedSilences, ...(count && { count }) } });
     } catch (err) {
-        const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR])
+        const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR])
         next(error);
     }
 }
@@ -62,7 +62,7 @@ const fetchHandler = async (request: Request, response: Response, next: NextFunc
         if (!silenceModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND });
         ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: transformedSilence });
     } catch (err) {
-        const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR])
+        const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR])
         next(error);
     }
 }
@@ -86,7 +86,7 @@ const updateHandler = async (request: Request, response: Response, next: NextFun
         const silenceResponse = await Silence.update(updatedSilence, { where: { id } })
         ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { silenceResponse } })
     } catch (err) {
-        const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR])
+        const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR])
         next(error);
     }
 }
@@ -97,13 +97,13 @@ const deleteHandler = async (request: Request, response: Response, next: NextFun
         const silenceModel = await Silence.findOne({ where: { id } });
         if (!silenceModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND });
         const silenceObject = silenceModel?.toJSON();
-        if (silenceObject?.status === 'expired') return next({ message: "Silence is already expired", statusCode: httpStatus.BAD_REQUEST });
+        if (silenceObject?.status === "expired") return next({ message: "Silence is already expired", statusCode: httpStatus.BAD_REQUEST });
         await deleteSilence(silenceObject);
         await silenceModel.destroy();
         updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: silenceObject });
         ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } })
     } catch (err) {
-        const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR])
+        const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR])
         next(error);
     }
 }

api-service/src/controllers/DataIngestion/DataIngestionController.ts

@@ -29,7 +29,7 @@ const dataIn = async (req: Request, res: Response) => {
     try {
         const requestBody = req.body;
         const datasetId = req.params.datasetId.trim();
-        
+
         const isValidSchema = schemaValidation(requestBody, validationSchema)
         if (!isValidSchema?.isValid) {
             logger.error({ apiId, message: isValidSchema?.message, code: "DATA_INGESTION_INVALID_INPUT" })
@@ -68,18 +68,23 @@ const addMetadataToEvents = (datasetId: string, payload: any) => {
     const obsrvMeta = { syncts: now, flags: {}, timespans: {}, error: {}, source: source };
     if (Array.isArray(validData)) {
         const payloadRef = validData.map((event: any) => {
-            event = _.set(event, "obsrv_meta", obsrvMeta);
-            event = _.set(event, "dataset", datasetId);
-            event = _.set(event, "msgid", mid);
-            return event
+            const payload = {
+                event,
+                "obsrv_meta": obsrvMeta,
+                "dataset": datasetId,
+                "msgid": mid
+            }
+            return payload;
         })
         return payloadRef;
     }
     else {
-        _.set(validData, "msgid", mid);
-        _.set(validData, "obsrv_meta", obsrvMeta);
-        _.set(validData, "dataset", datasetId);
-        return validData
+        return ({
+            "event": validData,
+            "obsrv_meta": obsrvMeta,
+            "dataset": datasetId,
+            "msgid": mid
+        });
     }
 }
 

api-service/src/controllers/DatasetCopy/DatasetCopy.ts

@@ -18,7 +18,7 @@ const validateRequest = (req: Request) => {
     }
 }
 
-const fetchDataset = async (req: Request, newDatasetId: string) => {
+const fetchDataset = async (req: Request) => {
     const datasetId = _.get(req, "body.request.source.datasetId");
     const isLive = _.get(req, "body.request.source.isLive");
 
@@ -39,10 +39,10 @@ const datasetCopy = async (req: Request, res: Response) => {
 
     validateRequest(req);
     const newDatasetId = _.get(req, "body.request.destination.datasetId");
-    const dataset = await fetchDataset(req, newDatasetId);
+    const dataset = await fetchDataset(req);
     updateRecords(dataset, newDatasetId)
     const response = await datasetService.createDraftDataset(dataset).catch(err => {
-        if (err?.name === 'SequelizeUniqueConstraintError') {
+        if (err?.name === "SequelizeUniqueConstraintError") {
             throw obsrvError(newDatasetId, "DATASET_ALREADY_EXISTS", `Dataset with id ${newDatasetId} already exists`, "BAD_REQUEST", 400);
         }
         throw obsrvError(newDatasetId, "DATASET_COPY_FAILURE", `Failed to clone dataset`, "INTERNAL_SERVER_ERROR", 500);

api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts

@@ -6,13 +6,13 @@ const version = defaultDatasetConfig.version;
 
 export const updateRecords = (datasetRecord: Record<string, any>, newDatasetId: string): void => {
     const dataset_id = newDatasetId;
-    _.set(datasetRecord, 'api_version', "v2")
-    _.set(datasetRecord, 'status', DatasetStatus.Draft)
+    _.set(datasetRecord, "api_version", "v2")
+    _.set(datasetRecord, "status", DatasetStatus.Draft)
     _.set(datasetRecord, "dataset_id", dataset_id)
     _.set(datasetRecord, "id", dataset_id)
     _.set(datasetRecord, "name", dataset_id)
     _.set(datasetRecord, "version_key", Date.now().toString())
-    _.set(datasetRecord, 'version', version);
+    _.set(datasetRecord, "version", version);
     _.set(datasetRecord, "entry_topic", config.telemetry_service_config.kafka.topics.createDataset)
     _.set(datasetRecord, "router_config", { topic: newDatasetId })
 }

api-service/src/controllers/DatasetImport/DatasetImport.ts

@@ -25,9 +25,9 @@ const datasetImport = async (req: Request, res: Response) => {
 const importDataset = async (dataset: Record<string, any>, overwrite: string | any) => {
     const dataset_id = _.get(dataset,"dataset_id")
     const response = await datasetService.createDraftDataset(dataset).catch(err => { return err })
-    if (response?.name === 'SequelizeUniqueConstraintError') {
+    if (response?.name === "SequelizeUniqueConstraintError") {
         if (overwrite === "true") {
-            const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(err=>{
+            const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(()=>{
                 throw obsrvError(dataset_id, "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id} as overwrite failed`, "INTERNAL_SERVER_ERROR", 500);
             })
             return _.omit(overwriteRes, ["message"])
@@ -44,7 +44,7 @@ const importDataset = async (dataset: Record<string, any>, overwrite: string | a
 const getResponseData = (ignoredConfigs: Record<string, any>) => {
     const { ignoredConnectors, ignoredTransformations, ignoredDenorms } = ignoredConfigs;
     let successMsg = "Dataset is imported successfully";
-    let partialIgnored: Record<string, any> = {};
+    const partialIgnored: Record<string, any> = {};
 
     if (ignoredConnectors.length || ignoredTransformations.length || ignoredDenorms.length) {
         successMsg = "Dataset is partially imported";

api-service/src/controllers/DatasetImport/DatasetImportHelper.ts

@@ -34,7 +34,7 @@ export const datasetImportValidation = async (payload: Record<string, any>): Pro
         throw obsrvError("", "DATASET_IMPORT_INVALID_CONFIGS", isRequestValid.message, "BAD_REQUEST", 400)
     }
 
-    let datasetConfig = payload.request;
+    const datasetConfig = payload.request;
 
     const connectors = _.get(datasetConfig, "connectors_config", []);
     const transformations = _.get(datasetConfig, "transformations_config", []);
@@ -100,7 +100,7 @@ export const migrateExportedDatasetV1 = (requestPayload: Record<string, any>) =>
     const { dataset_id, timestamp_key = "", data_key = "", type: datasetType } = _.get(datasetPayload, "data.metadata")
     const type = datasetType === "master-dataset" ? DatasetType.master : DatasetType.event
 
-    let dataset: Record<string, any> = {
+    const dataset: Record<string, any> = {
         dataset_id, id: dataset_id, name: dataset_id, type,
         version_key: Date.now().toString(),
         api_version: "v2",