From 5afc48dbf27356eef8de6c1f3da05d46b9be6b69 Mon Sep 17 00:00:00 2001 From: jonathangoulding Date: Mon, 3 Feb 2025 15:28:07 +0000 Subject: [PATCH] Refactor existing CSV converter https://eaflood.atlassian.net/browse/WATER-4776 As part of the work to implement notifications in the system repo we have found some logic that will need to be duplicated in upcoming work. This change lifts the convert to csv service into the lib. It has been renamed to transform to csv, The existing logic has been updated to be more precise in the transformation it is doing (transforms an array into a CSV row). --- .../jobs/export/convert-to-csv.service.js | 76 --------- .../export/write-table-to-file.service.js | 6 +- test/lib/transform-to-csv.lib.test.js | 158 ++++++++++++++++++ .../export/convert-to-csv.service.test.js | 106 ------------ 4 files changed, 161 insertions(+), 185 deletions(-) delete mode 100644 app/services/jobs/export/convert-to-csv.service.js create mode 100644 test/lib/transform-to-csv.lib.test.js delete mode 100644 test/services/jobs/export/convert-to-csv.service.test.js diff --git a/app/services/jobs/export/convert-to-csv.service.js b/app/services/jobs/export/convert-to-csv.service.js deleted file mode 100644 index 58d783fb44..0000000000 --- a/app/services/jobs/export/convert-to-csv.service.js +++ /dev/null @@ -1,76 +0,0 @@ -'use strict' - -/** - * Convert data to CSV format - * @module ConvertToCSVService - */ - -/** - * Converts data to a CSV formatted string - * - * @param {string[]} data - An array representing either the headers or rows from a db table - * - * @returns {string} A CSV formatted string - */ -function go(data) { - if (!data) { - return undefined - } - - return _transformDataToCSV(data) -} - -/** - * Transforms each row or header to CSV format and joins the values with commas - * - * @private - */ -function _transformDataToCSV(data) { - const transformedRow = data - .map((value) => { - return _transformValueToCSV(value) - }) - .join(',') - - return transformedRow + '\n' -} - -/** - * Transform a value to CSV format - * - * @private - */ -function _transformValueToCSV(value) { - // Return empty string for undefined or null values - if (!value && value !== false) { - return '' - } - - // Return ISO date if value is a date object - if (value instanceof Date) { - return value.toISOString() - } - - // Return integers and booleans as they are (not converted to a string) - if (Number.isInteger(value) || typeof value === 'boolean') { - return `${value}` - } - - // Return objects by serializing them to JSON - if (typeof value === 'object') { - const objectToString = JSON.stringify(value) - - const escapedObjectToString = objectToString.replace(/"/g, '""').replace(/:/g, ': ').replace(/,/g, ', ') - - return `"${escapedObjectToString}"` - } - - // Return strings by quoting them and escaping any double quotes - const stringValue = value.toString().replace(/"/g, '""') - - return `"${stringValue}"` -} - -module.exports = { - go -} diff --git a/app/services/jobs/export/write-table-to-file.service.js b/app/services/jobs/export/write-table-to-file.service.js index 0b9aaa6113..f9fe8048ea 100644 --- a/app/services/jobs/export/write-table-to-file.service.js +++ b/app/services/jobs/export/write-table-to-file.service.js @@ -11,7 +11,7 @@ const { pipeline, Transform } = require('stream') const path = require('path') const util = require('util') -const ConvertToCSVService = require('./convert-to-csv.service.js') +const { TransformArrayToCSVRow } = require('../../../lib/transform-to-csv.lib.js') /** * Converts data into CSV format and writes it to a file @@ -30,7 +30,7 @@ async function go(headers, rows, schemaFolderPath, tableName) { const transformDataStream = _transformDataStream() - const convertedHeaders = ConvertToCSVService.go(headers) + const convertedHeaders = TransformArrayToCSVRow(headers) writeToFileStream.write(convertedHeaders) @@ -48,7 +48,7 @@ function _transformDataStream() { return new Transform({ objectMode: true, transform: function (row, _encoding, callback) { - const datRow = ConvertToCSVService.go(Object.values(row)) + const datRow = TransformArrayToCSVRow(Object.values(row)) callback(null, datRow) } diff --git a/test/lib/transform-to-csv.lib.test.js b/test/lib/transform-to-csv.lib.test.js new file mode 100644 index 0000000000..be64145c67 --- /dev/null +++ b/test/lib/transform-to-csv.lib.test.js @@ -0,0 +1,158 @@ +'use strict' + +// Test framework dependencies +const Lab = require('@hapi/lab') +const Code = require('@hapi/code') + +const { describe, it, beforeEach } = (exports.lab = Lab.script()) +const { expect } = Code + +// Thing under test +const { TransformArrayToCSVRow } = require('../../app/lib/transform-to-csv.lib.js') + +describe('Transform to csv', () => { + describe('#TransformArrayToCSVRow', () => { + let testArray + + beforeEach(() => { + testArray = _complexArray() + }) + + it('correctly transforms all data types to csv', () => { + const result = TransformArrayToCSVRow(testArray) + + expect(result).to.equal( + '"20146cdc-9b40-4769-aa78-b51c17080d56",' + + '"4.1.1",' + + '9700,' + + '"Low loss tidal abstraction of water up to and ""including"" 25,002 megalitres, known as ML/yr a year where no model applies",' + + '2022-12-14T18:39:45.000Z,' + + 'true,' + + ',' + + ',' + + 'false,' + + ',' + + '25002,' + + '"{""message"": ""a json object""}"' + + '\n' + ) + }) + + describe('when the data type is', () => { + describe('an object', () => { + it('correctly formats the object to a string', () => { + const result = TransformArrayToCSVRow([{ message: 'a json object' }]) + + expect(result).to.equal('"{""message"": ""a json object""}"\n') + }) + }) + + describe('a UUID', () => { + it('correctly formats the UUID to a string', () => { + const result = TransformArrayToCSVRow(['20146cdc-9b40-4769-aa78-b51c17080d56']) + + expect(result).to.equal('"20146cdc-9b40-4769-aa78-b51c17080d56"\n') + }) + }) + + describe('a boolean', () => { + it('correctly formats the boolean to a string', () => { + const result = TransformArrayToCSVRow([true]) + + expect(result).to.equal('true\n') + }) + }) + + describe('a number', () => { + it('correctly formats the boolean to a string', () => { + const result = TransformArrayToCSVRow([100]) + + expect(result).to.equal('100\n') + }) + }) + + describe('a string containing', () => { + describe('a comma', () => { + it('correctly formats the string', () => { + const result = TransformArrayToCSVRow(['I am a, comma seperated sentence.']) + + expect(result).to.equal('"I am a, comma seperated sentence."\n') + }) + }) + + describe('a single double quote', () => { + it('correctly formats the string', () => { + const result = TransformArrayToCSVRow(['I am a " double quote sentence.']) + + expect(result).to.equal('"I am a "" double quote sentence."\n') + }) + }) + + describe('a double double quote', () => { + it('correctly formats the string', () => { + const result = TransformArrayToCSVRow(['I am a "" double quote sentence.']) + + expect(result).to.equal('"I am a """" double quote sentence."\n') + }) + }) + + describe('a back slash', () => { + it('correctly formats the string', () => { + const result = TransformArrayToCSVRow(['I am a "\\" back slash sentence.']) + + expect(result).to.equal('"I am a ""\\"" back slash sentence."\n') + }) + }) + }) + + describe('a date', () => { + it('correctly formats the date to an iso string', () => { + const result = TransformArrayToCSVRow([new Date('2021-02-01')]) + + expect(result).to.equal('2021-02-01T00:00:00.000Z\n') + }) + }) + }) + + describe('when an array of strings us provided', () => { + beforeEach(() => { + testArray = _arrayOfStrings() + }) + + it('converts the data to a CSV format', () => { + const result = TransformArrayToCSVRow(testArray) + + expect(result).to.equal('"name","age"\n') + }) + }) + + describe('when no array is provided', () => { + it('returns undefined', () => { + const result = TransformArrayToCSVRow() + + expect(result).to.equal(undefined) + }) + }) + }) +}) + +function _arrayOfStrings() { + return ['name', 'age'] +} + +function _complexArray() { + return [ + '20146cdc-9b40-4769-aa78-b51c17080d56', + '4.1.1', + 9700, + 'Low loss tidal abstraction of water up to and "including" 25,002 megalitres, known as ML/yr a year where no model applies', + new Date(2022, 11, 14, 18, 39, 45), + true, + null, + undefined, + false, + '', + 25002, + { message: 'a json object' } + ] +} diff --git a/test/services/jobs/export/convert-to-csv.service.test.js b/test/services/jobs/export/convert-to-csv.service.test.js deleted file mode 100644 index 52b0b1082a..0000000000 --- a/test/services/jobs/export/convert-to-csv.service.test.js +++ /dev/null @@ -1,106 +0,0 @@ -'use strict' - -// Test framework dependencies -const Lab = require('@hapi/lab') -const Code = require('@hapi/code') - -const { describe, it } = (exports.lab = Lab.script()) -const { expect } = Code - -// Thing under test -const ConvertToCSVService = require('../../../../app/services/jobs/export/convert-to-csv.service.js') - -/** - * billingChargeCategoriesTable has all data types we are testing for - * objects{Date} - dateCreated - * String - description - * String (with added quotes)- shortDescription - * Integer - maxVolume - * Null - lossFactor - * Undefined - modelTier - */ -const billingChargeCategoryRow = [ - '20146cdc-9b40-4769-aa78-b51c17080d56', - '4.1.1', - 9700, - 'Low loss tidal abstraction of water up to and "including" 25,002 megalitres a year where no model applies', - 'Low loss, tidal, up to and including 25,002 ML/yr', - new Date(2022, 11, 14, 18, 39, 45), - new Date(2022, 11, 14, 18, 39, 45), - true, - null, - undefined, - false, - '', - 25002 -] - -const billingChargeCategoriesColumnInfo = [ - 'billingChargeCategoryId', - 'reference', - 'subsistenceCharge', - 'description', - 'shortDescription', - 'dateCreated', - 'dateUpdated', - 'isTidal', - 'lossFactor', - 'modelTier', - 'isRestrictedSource', - 'minVolume', - 'maxVolume' -] - -const csvHeaders = - '"billingChargeCategoryId",' + - '"reference",' + - '"subsistenceCharge",' + - '"description",' + - '"shortDescription",' + - '"dateCreated",' + - '"dateUpdated",' + - '"isTidal",' + - '"lossFactor",' + - '"modelTier",' + - '"isRestrictedSource",' + - '"minVolume",' + - '"maxVolume"\n' - -const csvValues = - '"20146cdc-9b40-4769-aa78-b51c17080d56",' + - '"4.1.1",9700,' + - '"Low loss tidal abstraction of water up to and ""including"" 25,002 megalitres a year where no model applies",' + - '"Low loss, tidal, up to and including 25,002 ML/yr",' + - '2022-12-14T18:39:45.000Z,2022-12-14T18:39:45.000Z,' + - 'true,' + - ',' + - ',' + - 'false,' + - ',' + - '25002\n' - -describe('Convert to CSV service', () => { - describe('when given a row of data to convert', () => { - it('convert the data to a CSV format', () => { - const result = ConvertToCSVService.go(billingChargeCategoryRow) - - expect(result).to.equal(csvValues) - }) - }) - - describe('when given headers to convert', () => { - it('converts the data to a CSV format', () => { - const result = ConvertToCSVService.go(billingChargeCategoriesColumnInfo) - - expect(result).to.equal(csvHeaders) - }) - }) - - describe('when not given data to convert', () => { - it('exports the table to CSV without any rows', () => { - const result = ConvertToCSVService.go() - - expect(result).to.equal(undefined) - }) - }) -})