From 0b11f5ec23fd736cd7b29157b63f072565a0e035 Mon Sep 17 00:00:00 2001 From: baalmart Date: Thu, 18 Jan 2024 18:38:57 +0300 Subject: [PATCH 1/2] reduce the logging for last refreshed to only happen on an hourly basis --- src/device-registry/models/Event.js | 102 +--------------------------- 1 file changed, 2 insertions(+), 100 deletions(-) diff --git a/src/device-registry/models/Event.js b/src/device-registry/models/Event.js index db00647491..1c6bf400e8 100644 --- a/src/device-registry/models/Event.js +++ b/src/device-registry/models/Event.js @@ -1756,55 +1756,6 @@ eventSchema.statics = { }) .allowDiskUse(true); - data[0].data.forEach((record) => { - if ( - constants.ENVIRONMENT && - constants.ENVIRONMENT === "PRODUCTION ENVIRONMENT" - ) { - if (record.timeDifferenceHours > 14) { - logObject( - `🪫🪫 Last refreshed time difference exceeds 14 hours for device: ${ - record.device ? record.device : "" - }, frequency ${ - record.frequency ? record.frequency : "" - }, time ${record.time ? record.time : ""} and site ${ - record.siteDetails ? record.siteDetails.name : "" - }` - ); - logger.info( - `🪫🪫 Last refreshed time difference exceeds 14 hours for device: ${ - record.device ? record.device : "" - }, Frequency: ${ - record.frequency ? record.frequency : "" - }, Time: ${record.time ? record.time : ""}, Site Name: ${ - record.siteDetails ? record.siteDetails.name : "" - }` - ); - } - - if (record.pm2_5 === null) { - logObject( - `😲😲 Null pm2_5 value for device: ${ - record.device ? record.device : "" - }, frequency ${ - record.frequency ? record.frequency : "" - }, time ${record.time ? record.time : ""} and site ${ - record.siteDetails ? record.siteDetails.name : "" - }` - ); - logger.info( - `😲😲 Null pm2_5 value for device: ${ - record.device ? record.device : "" - }, Frequency: ${ - record.frequency ? record.frequency : "" - }, Time: ${record.time ? record.time : ""}, Site Name: ${ - record.siteDetails ? record.siteDetails.name : "" - }` - ); - } - } - }); - data[0].data = data[0].data.filter((record) => record.pm2_5 !== null); return { @@ -1934,55 +1885,6 @@ eventSchema.statics = { }) .allowDiskUse(true); - data[0].data.forEach((record) => { - if ( - constants.ENVIRONMENT && - constants.ENVIRONMENT === "PRODUCTION ENVIRONMENT" - ) { - if (record.timeDifferenceHours > 14) { - logObject( - `🪫🪫 Last refreshed time difference exceeds 14 hours for device: ${ - record.device ? record.device : "" - }, frequency ${ - record.frequency ? record.frequency : "" - }, time ${record.time ? record.time : ""} and site ${ - record.siteDetails ? record.siteDetails.name : "" - }` - ); - logger.info( - `🪫🪫 Last refreshed time difference exceeds 14 hours for device: ${ - record.device ? record.device : "" - }, Frequency: ${ - record.frequency ? record.frequency : "" - }, Time: ${record.time ? record.time : ""}, Site Name: ${ - record.siteDetails ? record.siteDetails.name : "" - }` - ); - } - - if (record.pm2_5 === null) { - logObject( - `😲😲 Null pm2_5 value for device: ${ - record.device ? record.device : "" - }, frequency ${ - record.frequency ? record.frequency : "" - }, time ${record.time ? record.time : ""} and site ${ - record.siteDetails ? record.siteDetails.name : "" - }` - ); - logger.info( - `😲😲 Null pm2_5 value for device: ${ - record.device ? record.device : "" - }, Frequency: ${ - record.frequency ? record.frequency : "" - }, Time: ${record.time ? record.time : ""}, Site Name: ${ - record.siteDetails ? record.siteDetails.name : "" - }` - ); - } - } - }); - data[0].data = data[0].data.filter((record) => record.pm2_5 !== null); return { success: true, @@ -2013,7 +1915,7 @@ eventSchema.statics.view = async function(filter, next) { request.limit = filter.limit ? filter.limit : DEFAULT_LIMIT; request.page = filter.page ? filter.page : DEFAULT_PAGE; const result = await fetchData(this, request); - const transformedData = filterNullAndReportOffDevices(result[0].data); + const transformedData = filterNull(result[0].data); result[0].data = transformedData; const calculatedValues = computeAveragePm2_5(transformedData); result[0].meta.pm2_5Avg = calculatedValues; @@ -2043,7 +1945,7 @@ eventSchema.statics.fetch = async function(filter) { request.limit = filter.limit ? filter.limit : DEFAULT_LIMIT; request.page = filter.page ? filter.page : DEFAULT_PAGE; const result = await fetchData(this, request); - const transformedData = filterNull(result[0].data); + const transformedData = filterNullAndReportOffDevices(result[0].data); result[0].data = transformedData; const calculatedValues = computeAveragePm2_5(transformedData); result[0].meta.pm2_5Avg = calculatedValues; From 09103afba70630daa6be1d264ee7db01bf9e8bf3 Mon Sep 17 00:00:00 2001 From: baalmart Date: Thu, 18 Jan 2024 19:08:23 +0300 Subject: [PATCH 2/2] continuing to reduce the logs --- src/device-registry/bin/kafka-consumer.js | 4 +-- src/device-registry/utils/create-event.js | 41 ++++++++++++----------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/device-registry/bin/kafka-consumer.js b/src/device-registry/bin/kafka-consumer.js index 6a8fa5330f..1c7b26690e 100644 --- a/src/device-registry/bin/kafka-consumer.js +++ b/src/device-registry/bin/kafka-consumer.js @@ -143,14 +143,14 @@ const consumeHourlyMeasurements = async (messageData) => { // ); if (responseFromInsertMeasurements.success === false) { - console.log("KAFKA: some failures while inserting data"); + console.log("KAFKA: failed to store the measurements"); // logger.error( // `KAFKA: responseFromInsertMeasurements --- ${jsonify( // responseFromInsertMeasurements // )}` // ); } else if (responseFromInsertMeasurements.success === true) { - console.log("KAFKA: successfully inserted data"); + console.log("KAFKA: successfully stored the measurements"); // logger.info( // `KAFKA: successfully inserted the measurements --- ${jsonify(responseFromInsertMeasurements.message ? // responseFromInsertMeasurements.message : diff --git a/src/device-registry/utils/create-event.js b/src/device-registry/utils/create-event.js index 4260cefa38..c8c2399765 100644 --- a/src/device-registry/utils/create-event.js +++ b/src/device-registry/utils/create-event.js @@ -246,7 +246,7 @@ async function processEvent(event, next) { update, options ); - logObject("addedEvents", addedEvents); + // logObject("addedEvents", addedEvents); if (addedEvents) { return { @@ -1141,7 +1141,7 @@ const createEvent = { request, next ); - logObject("transformEventsResponse man", transformEventsResponse); + // logObject("transformEventsResponse man", transformEventsResponse); if (transformEventsResponse.success === true) { let transformedEvents = transformEventsResponse.data; let nAdded = 0; @@ -1151,7 +1151,7 @@ const createEvent = { for (const event of transformedEvents) { try { - logObject("event", event); + // logObject("event", event); let value = event; let dot = new Dot("."); let options = event.options; @@ -1160,17 +1160,17 @@ const createEvent = { dot.delete(["filter", "update", "options"], value); update["$push"] = { values: value }; - logObject("event.tenant", event.tenant); - logObject("update", update); - logObject("filter", filter); - logObject("options", options); + // logObject("event.tenant", event.tenant); + // logObject("update", update); + // logObject("filter", filter); + // logObject("options", options); const addedEvents = await EventModel(event.tenant).updateOne( filter, update, options ); - logObject("addedEvents", addedEvents); + // logObject("addedEvents", addedEvents); if (addedEvents) { nAdded += 1; eventsAdded.push(event); @@ -1241,7 +1241,7 @@ const createEvent = { }; } } else if (transformEventsResponse.success === false) { - logText("maan, things have jam!"); + // logText("maan, things have jam!"); return transformEventsResponse; } } catch (error) { @@ -1985,7 +1985,7 @@ const createEvent = { }, addEvents: async (request, next) => { try { - logText("adding the events insertTransformedEvents to the util....."); + // logText("adding the events insertTransformedEvents to the util....."); // logger.info(`adding events in the util.....`); /** * Step One: trasform or prepare for insertion into Events collection -- prepare the nesting expexctation @@ -1998,7 +1998,7 @@ const createEvent = { ); if (transformEventsResponses.success === false) { - logElement("transformEventsResponses was false?", true); + // logElement("transformEventsResponses was false?", true); return transformEventsResponses; } else if (transformEventsResponses.success === true) { const transformedMeasurements = transformEventsResponses.data; @@ -2083,7 +2083,7 @@ const createEvent = { } } - if (errors.length > 0) { + if (errors.length > 0 && isEmpty(data)) { logger.error( `finished the operation with some errors -- ${jsonify(errors)}` ); @@ -2095,8 +2095,9 @@ const createEvent = { } else { return { success: true, - message: "successfully added all the events", + message: "successfully added the events", data, + errors, }; } } catch (error) { @@ -2184,7 +2185,7 @@ const createEvent = { for (const measurement of responseFromTransformMeasurements.data) { try { - logObject("the measurement in the insertion process", measurement); + // logObject("the measurement in the insertion process", measurement); const eventsFilter = { day: measurement.day, site_id: measurement.site_id, @@ -2202,9 +2203,9 @@ const createEvent = { let someDeviceDetails = {}; someDeviceDetails["device_id"] = measurement.device_id; someDeviceDetails["site_id"] = measurement.site_id; - logObject("someDeviceDetails", someDeviceDetails); + // logObject("someDeviceDetails", someDeviceDetails); - logObject("the measurement", measurement); + // logObject("the measurement", measurement); const eventsUpdate = { $push: { values: measurement }, @@ -2212,8 +2213,8 @@ const createEvent = { $max: { last: measurement.time }, $inc: { nValues: 1 }, }; - logObject("eventsUpdate", eventsUpdate); - logObject("eventsFilter", eventsFilter); + // logObject("eventsUpdate", eventsUpdate); + // logObject("eventsFilter", eventsFilter); const addedEvents = await EventModel(tenant).updateOne( eventsFilter, @@ -2222,7 +2223,7 @@ const createEvent = { upsert: true, } ); - logObject("addedEvents", addedEvents); + // logObject("addedEvents", addedEvents); if (addedEvents) { nAdded += 1; eventsAdded.push(measurement); @@ -2289,6 +2290,7 @@ const createEvent = { } if (errors.length > 0) { + console.log("API: SOME measurements successfully added"); return { success: false, message: "finished the operation with some errors", @@ -2296,6 +2298,7 @@ const createEvent = { status: httpStatus.INTERNAL_SERVER_ERROR, }; } else { + console.log("API: ALL measurements successfully added"); return { success: true, message: "successfully added all the events",