Skip to content

Commit 7827640

Browse files
natlibfi-jonollilammsalme
natlibfi-jonollil
and
ammsalme
authored
MRA-728 (#142)
* add different test date format for mongo * add JSDoc comments * remove luxon dependency Update serializers (#143) * Update serializers + deps v4.0.8-alpha.12 Co-authored-by: Minttu Hurme <[email protected]>
1 parent ff900ff commit 7827640

20 files changed

+478
-365
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@ coverage
44
dist
55
.nyc_output
66
.DS_Store
7+
.env*

‎package-lock.json

+207-177
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+16-16
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
"url": "[email protected]:natlibfi/melinda-rest-api-commons.git"
1515
},
1616
"license": "AGPL-3.0+",
17-
"version": "4.0.17",
17+
"version": "4.0.18-alpha.12",
1818
"main": "./dist/index.js",
1919
"engines": {
2020
"node": ">=18"
@@ -39,32 +39,32 @@
3939
},
4040
"dependencies": {
4141
"@natlibfi/marc-record": "^8.0.0",
42-
"@natlibfi/marc-record-serializers": "^10.1.1",
43-
"@natlibfi/marc-record-validate": "^8.0.1",
44-
"@natlibfi/marc-record-validators-melinda": "^10.13.0",
45-
"@natlibfi/melinda-backend-commons": "^2.2.1",
46-
"@natlibfi/melinda-commons": "^13.0.6",
42+
"@natlibfi/marc-record-serializers": "^10.1.2-alpha.1",
43+
"@natlibfi/marc-record-validate": "^8.0.3",
44+
"@natlibfi/marc-record-validators-melinda": "^10.15.0",
45+
"@natlibfi/melinda-backend-commons": "^2.2.2",
46+
"@natlibfi/melinda-commons": "^13.0.7",
4747
"amqplib": "^0.10.3",
4848
"debug": "^4.3.4",
49-
"http-status": "^1.7.0",
49+
"http-status": "^1.7.3",
5050
"moment": "^2.29.4",
5151
"mongo-sanitize": "^1.1.0",
5252
"mongodb": "^4.17.1"
5353
},
5454
"devDependencies": {
55-
"@babel/cli": "^7.22.15",
56-
"@babel/core": "^7.22.20",
57-
"@babel/plugin-transform-runtime": "^7.22.15",
58-
"@babel/preset-env": "^7.22.20",
55+
"@babel/cli": "^7.23.0",
56+
"@babel/core": "^7.23.2",
57+
"@babel/plugin-transform-runtime": "^7.23.2",
58+
"@babel/preset-env": "^7.23.2",
5959
"@natlibfi/eslint-config-melinda-backend": "^3.0.2",
60-
"@natlibfi/fixugen": "^2.0.1",
61-
"@natlibfi/fixura": "^3.0.1",
62-
"@natlibfi/fixura-mongo": "^2.0.10-alpha.1",
60+
"@natlibfi/fixugen": "^2.0.2",
61+
"@natlibfi/fixura": "^3.0.2",
62+
"@natlibfi/fixura-mongo": "^2.0.10",
6363
"babel-plugin-istanbul": "^6.1.1",
6464
"babel-plugin-rewire": "^1.2.0",
65-
"chai": "^4.3.8",
65+
"chai": "^4.3.10",
6666
"cross-env": "^7.0.3",
67-
"eslint": "^8.49.0",
67+
"eslint": "^8.52.0",
6868
"mocha": "^10.2.0",
6969
"nodemon": "^3.0.1",
7070
"nyc": "^15.1.0"

‎src/mongoLog.js

+100-26
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
*
55
* Shared modules for microservices of Melinda rest api batch import system
66
*
7-
* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
7+
* Copyright (C) 2020-2023 University Of Helsinki (The National Library Of Finland)
88
*
99
* This file is part of melinda-rest-api-commons
1010
*
@@ -35,6 +35,12 @@ import httpStatus from 'http-status';
3535
import sanitize from 'mongo-sanitize';
3636
import {LOG_ITEM_TYPE} from './constants';
3737

38+
/**
39+
* Create log operator
40+
* @param {String} MONGO_URI connnection address to mongo
41+
* @param {String} dbName Mongo DB name, defaults 'rest-api'
42+
* @returns {Object} containing all log handling functions
43+
*/
3844
export default async function (MONGO_URI, dbName = 'rest-api') {
3945
const logger = createLogger();
4046

@@ -44,6 +50,11 @@ export default async function (MONGO_URI, dbName = 'rest-api') {
4450
const collection = 'logs';
4551
return {addLogItem, query, queryById, getListOfLogs, getListOfCatalogers, getExpandedListOfLogs, protect, remove, removeBySequences};
4652

53+
/**
54+
* Add log item to collection
55+
* @param {Object} logItem contains log item data
56+
* @returns void
57+
*/
4758
async function addLogItem(logItem) {
4859
const time = moment().toDate();
4960
const newLogItem = {
@@ -70,11 +81,17 @@ export default async function (MONGO_URI, dbName = 'rest-api') {
7081
}
7182
}
7283

73-
async function query(params) {
74-
logger.debug(`Query params: ${JSON.stringify(params)}`);
75-
checkLogItemType(params.logItemType, false, false);
76-
const {limit = 5, skip = 0, ...rest} = params;
77-
logger.debug(`Actual query params: ${JSON.stringify(rest)}`);
84+
/**
85+
* Query log items
86+
* @param {Integer} limit defaults 5
87+
* @param {Integer} skip defaults 0
88+
* @param {Object} rest query params
89+
* @returns result array
90+
*/
91+
async function query({limit = 5, skip = 0, ...rest}) {
92+
logger.debug(`Query params: ${JSON.stringify(rest)}`);
93+
logger.debug(`Limit and skip params: ${limit} | ${skip}`);
94+
checkLogItemType(rest.logItemType, false, false);
7895
const result = await db.collection(collection) // eslint-disable-line functional/immutable-data
7996
.find(rest)
8097
.sort({creationTime: 1})
@@ -87,10 +104,18 @@ export default async function (MONGO_URI, dbName = 'rest-api') {
87104
}
88105

89106
// DEVELOP: queryById returns jsut one (randomish) log! is this useful?
90-
async function queryById(correlationIdString, skip = 0, limit = 1) {
91-
logger.debug(`QueryById: ${correlationIdString}`);
107+
/**
108+
* Get single log item by correlationId
109+
* @param {String} correlationId
110+
* @param {String} logItemType constant LOG_ITEM_TYPE defaults LOG_ITEM_TYPE.MERGE_LOG
111+
* @param {Integer} skip defaults 0
112+
* @param {Integer} limit defaults 1
113+
* @returns query result array
114+
*/
115+
async function queryById(correlationId, logItemType = LOG_ITEM_TYPE.MERGE_LOG, skip = 0, limit = 1) {
116+
logger.debug(`QueryById: ${correlationId}`);
92117
const result = await db.collection(collection) // eslint-disable-line functional/immutable-data
93-
.find({correlationId: correlationIdString})
118+
.find({correlationId, logItemType})
94119
.sort({blobSequence: 1})
95120
.skip(parseInt(skip, 10))
96121
.limit(parseInt(limit, 10))
@@ -101,14 +126,23 @@ export default async function (MONGO_URI, dbName = 'rest-api') {
101126

102127
// getListOfLogs returns list of correlationIds that have logs of given logItemType
103128
// if logItemType is not given, getListOfLogs returns list of correlationIds that have MERGE_LOG
104-
async function getListOfLogs(logItemType = 'MERGE_LOG') {
105-
checkLogItemType(logItemType, false, false);
129+
/**
130+
* Get list of correlationIds of logs filtered by logItemType
131+
* @param {String} logItemType constant LOG_ITEM_TYPE defaults LOG_ITEM_TYPE.MERGE_LOG
132+
* @returns Array of query results
133+
*/
134+
async function getListOfLogs(logItemType = LOG_ITEM_TYPE.MERGE_LOG) {
135+
// checkLogItemType(logItemType, false, false);
106136
const result = await db.collection(collection) // eslint-disable-line functional/immutable-data
107137
.distinct('correlationId', {logItemType});
108138
logger.debug(`Query result: ${result.length > 0 ? `Found ${result.length} log items!` : 'Not found!'}`);
109-
return {status: result.length > 0 ? httpStatus.OK : httpStatus.NOT_FOUND, payload: result.length > 0 ? result : 'No logs found'};
139+
return result;
110140
}
111141

142+
/**
143+
* Get list of catalogers from logs
144+
* @returns Array of query results
145+
*/
112146
async function getListOfCatalogers() {
113147
logger.debug(`Getting list of Catalogers`);
114148

@@ -118,15 +152,23 @@ export default async function (MONGO_URI, dbName = 'rest-api') {
118152
return result;
119153
}
120154

121-
// getExpandedListOfLogs returns groped MERGE_LOGs and MATCH_LOGs
122-
async function getExpandedListOfLogs({logItemTypes = [LOG_ITEM_TYPE.MERGE_LOG, LOG_ITEM_TYPE.MATCH_LOG], catalogers = [], dateBefore = new Date(), dateAfter = new Date('2000-01-01')}) {
155+
/**
156+
* Get filtered list of logs with extra info. Give params in Object
157+
* @param {[String]} logItemTypes: String array of logItemTypes. Defaults [LOG_ITEM_TYPE.MERGE_LOG, LOG_ITEM_TYPE.MATCH_LOG]
158+
* @param {[String]} catalogers: String array of wanted catalogers. Defaults []
159+
* @param {String} dateBefore: String date schema 'YYYY-MM-DD'. Defaults new Date()
160+
* @param {String} dateAfter: String date schema 'YYYY-MM-DD'. Defaults '2000-01-01'
161+
* @param {Boolean} test: Boolean is this test run. Defaults false
162+
* @returns Parsed Object {'correlationId', 'logItemType', 'creationTime', 'cataloger', 'logCount'}
163+
*/
164+
async function getExpandedListOfLogs({logItemTypes = [LOG_ITEM_TYPE.MERGE_LOG, LOG_ITEM_TYPE.MATCH_LOG], catalogers = [], dateBefore = new Date(), dateAfter = '2000-01-01', test = false}) {
123165
logger.debug(`commons: logItemTypes: ${JSON.stringify(logItemTypes)}, dateAfter: ${dateAfter}, dateBefore: ${dateBefore}}, catalogers: ${JSON.stringify(catalogers)}`);
124-
logger.debug(JSON.stringify(generateMatchObject(logItemTypes, catalogers, dateBefore, dateAfter))); // eslint-disable-line
166+
logger.debug(JSON.stringify(generateMatchObject({logItemTypes, catalogers, dateBefore, dateAfter}))); // eslint-disable-line
125167
//checkLogItemType(logItemType, false, false);
126168
logger.debug(`Getting expanded list of logs`);
127169
const pipeline = [
128170
// currently return only MERGE_LOG and MATCH_LOG
129-
generateMatchObject(logItemTypes, catalogers, dateBefore, dateAfter),
171+
generateMatchObject({logItemTypes, catalogers, dateBefore, dateAfter, test}),
130172
{
131173
'$sort':
132174
{'correlationId': 1, 'logItemType': 1, 'creationTime': 1}
@@ -160,18 +202,25 @@ export default async function (MONGO_URI, dbName = 'rest-api') {
160202
logger.silly(`Query result: ${JSON.stringify(fixedResult)}`);
161203

162204
logger.debug(`Query result: ${fixedResult.length > 0 ? `Found ${fixedResult.length} log items!` : 'Not found!'}`);
163-
return {status: fixedResult.length > 0 ? httpStatus.OK : httpStatus.NOT_FOUND, payload: fixedResult.length > 0 ? fixedResult : 'No logs found'};
205+
return fixedResult;
164206

165-
function generateMatchObject(logItemTypes, catalogers, dateBefore, dateAfter) {
166-
const dateBeforeIso = new Date(dateBefore).toISOString();
167-
const dateAfterIso = new Date(dateAfter).toISOString();
207+
/**
208+
* Generate match object for aggregation pipeline
209+
* @param {[String]} logItemTypes: String array of logItemTypes.
210+
* @param {[String]} catalogers: String array of wanted catalogers
211+
* @param {String} dateBefore: String date schema 'YYYY-MM-DD'
212+
* @param {String} dateAfter: String date schema 'YYYY-MM-DD'
213+
* @param {Boolean} test: Boolean is this test run. Defaults false
214+
* @returns Match Object
215+
*/
216+
function generateMatchObject({logItemTypes, catalogers, dateBefore, dateAfter, test = false}) {
168217
const matchOptions = {
169218
'$match': {
170219
'logItemType': logItemTypes.length > 0 ? {'$in': logItemTypes} : /.*/ui,
171220
'cataloger': catalogers.length > 0 ? {'$in': catalogers} : /.*/ui,
172221
'creationTime': {
173-
'$gte': dateAfterIso,
174-
'$lte': dateBeforeIso
222+
'$gte': test ? new Date(dateAfter).toISOString() : new Date(dateAfter),
223+
'$lte': test ? new Date(dateBefore).toISOString() : new Date(dateBefore)
175224
}
176225
}
177226
};
@@ -180,6 +229,12 @@ export default async function (MONGO_URI, dbName = 'rest-api') {
180229
}
181230
}
182231

232+
/**
233+
* Protect logs from cleaning. Forced cleaning defaults 30days
234+
* @param {String} correlationId
235+
* @param {Integer} blobSequence
236+
* @returns result
237+
*/
183238
async function protect(correlationId, blobSequence) {
184239
logger.info(`Protecting in Mongo (${collection}) correlationId: ${correlationId}, blobSequence: ${blobSequence}`);
185240
const cleanCorrelationId = sanitize(correlationId);
@@ -198,29 +253,42 @@ export default async function (MONGO_URI, dbName = 'rest-api') {
198253
}
199254
]
200255
);
201-
return {status: result.modifiedCount > 0 ? httpStatus.OK : httpStatus.NOT_FOUND, payload: result.modifiedCount > 0 ? result : 'No logs found'};
256+
return result;
202257
} catch (error) {
203258
const errorMessage = error.payload || error.message || '';
204259
logError(error);
205260
throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);
206261
}
207262
}
208263

264+
/**
265+
* Remove logs
266+
* @param {String} correlationId
267+
* @param {Boolean} force defaults false
268+
* @returns result
269+
*/
209270
async function remove(correlationId, force = false) {
210271
logger.info(`Removing from Mongo (${collection}) correlationId: ${correlationId}`);
211272
const clean = sanitize(correlationId);
212273
const filter = force ? {correlationId: clean} : {correlationId: clean, protected: {$ne: true}};
213274

214275
try {
215276
const result = await db.collection(collection).deleteMany(filter);
216-
return {status: result.deletedCount > 0 ? httpStatus.OK : httpStatus.NOT_FOUND, payload: result.deletedCount > 0 ? result : 'No logs found'};
277+
return result;
217278
} catch (error) {
218279
const errorMessage = error.payload || error.message || '';
219280
logError(error);
220281
throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);
221282
}
222283
}
223284

285+
/**
286+
* Remove logs by sequences
287+
* @param {String} correlationId
288+
* @param {[Integer]} sequences
289+
* @param {Boolean} force defaults false
290+
* @returns result
291+
*/
224292
async function removeBySequences(correlationId, sequences = [], force = false) {
225293
logger.info(`Removing from Mongo (${collection}) correlationId: ${correlationId}, sequences: ${sequences.length}`);
226294
const clean = sanitize(correlationId);
@@ -236,16 +304,22 @@ export default async function (MONGO_URI, dbName = 'rest-api') {
236304

237305
try {
238306
const result = await db.collection(collection).deleteMany(filter);
239-
return {status: result.deletedCount > 0 ? httpStatus.OK : httpStatus.NOT_FOUND, payload: result.deletedCount > 0 ? result : 'No logs found'};
307+
return result;
240308
} catch (error) {
241309
const errorMessage = error.payload || error.message || '';
242310
logError(error);
243311
throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);
244312
}
245313
}
246314

315+
/**
316+
* Checks existence and validity of logItemType
317+
* @param {String} logItemType constant LOG_ITEM_TYPE
318+
* @param {Boolean} errorUnknown defaults false
319+
* @param {Boolean} errorNotExisting defaults false
320+
* @returns void
321+
*/
247322
function checkLogItemType(logItemType, errorUnknown = false, errorNotExisting = false) {
248-
249323
if (logItemType) {
250324
const typeInLogItemTypes = Object.values(LOG_ITEM_TYPE).indexOf(logItemType) > -1;
251325
if (typeInLogItemTypes) {

‎test-fixtures/mongoLog/03/expectedResult.json

+23-26
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,23 @@
1-
{
2-
"payload": [
3-
{
4-
"cataloger": "IMP_TEST1",
5-
"correlationId": "bdce85eb-0ca2-4525-b00e-ae67d43b68ec",
6-
"creationTime": "2023-08-07T01:21:38.392Z",
7-
"logCount": 1,
8-
"logItemType": "MERGE_LOG"
9-
},
10-
{
11-
"cataloger": "IMP_TEST2",
12-
"correlationId": "f63bad0b-8820-428a-b570-c172f209ee94",
13-
"creationTime": "2023-08-08T01:25:59.348Z",
14-
"logCount": 1,
15-
"logItemType": "MERGE_LOG"
16-
},
17-
{
18-
"cataloger": "IMP_TEST3",
19-
"correlationId": "7207c1f2-cbc4-4a0c-bd64-344a746ee605",
20-
"creationTime": "2023-09-21T01:07:26.693Z",
21-
"logCount": 1,
22-
"logItemType": "MATCH_LOG"
23-
}
24-
],
25-
"status": 200
26-
}
1+
[
2+
{
3+
"cataloger": "IMP_TEST1",
4+
"correlationId": "bdce85eb-0ca2-4525-b00e-ae67d43b68ec",
5+
"creationTime": "2023-08-07T01:21:38.392Z",
6+
"logCount": 1,
7+
"logItemType": "MERGE_LOG"
8+
},
9+
{
10+
"cataloger": "IMP_TEST2",
11+
"correlationId": "f63bad0b-8820-428a-b570-c172f209ee94",
12+
"creationTime": "2023-08-08T01:25:59.348Z",
13+
"logCount": 1,
14+
"logItemType": "MERGE_LOG"
15+
},
16+
{
17+
"cataloger": "IMP_TEST3",
18+
"correlationId": "7207c1f2-cbc4-4a0c-bd64-344a746ee605",
19+
"creationTime": "2023-09-21T01:07:26.693Z",
20+
"logCount": 1,
21+
"logItemType": "MATCH_LOG"
22+
}
23+
]

‎test-fixtures/mongoLog/03/metadata.json

+3-1
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,7 @@
44
"skip": false,
55
"functionName": "getExpandedListOfLogs",
66
"preFillDb": true,
7-
"params": []
7+
"params": {
8+
"test": true
9+
}
810
}

0 commit comments

Comments
 (0)