Refactor existing CSV converter (#1677)

* Refactor existing CSV converter

https://eaflood.atlassian.net/browse/WATER-4776

As part of the work to implement notifications in the system repo we have found some logic that will need to be duplicated in upcoming work.

This change lifts the convert to csv service into the lib.

It has been renamed to transform to csv,

The existing logic has been updated to be more precise in the transformation it is doing (transforms an array into a CSV row).

Author: jonathangoulding

app/services/jobs/export/convert-to-csv.service.js renamed to app/lib/transform-to-csv.lib.js

@@ -1,32 +1,33 @@
 'use strict'
 
 /**
- * Convert data to CSV format
- * @module ConvertToCSVService
- */
-
-/**
- * Converts data to a CSV formatted string
+ * Transforms an array into a CSV formatted string.
+ *
+ * A CSV header / row is the same regardless of transformation. The header is commonly the first row.
+ * Therefore, this function works for both a header and row.
  *
- * @param {string[]} data - An array representing either the headers or rows from a db table
+ * The function can transform most common types:
+ * - Objects are stringified and parsed
+ * - Numbers are converted to strings
+ * - Booleans are converted to strings
+ * - Falsey values are converted to empty strings
+ * - Dates are converted into ISO string format
+ * - Strings are escaped to handle special characters (e.g., ',', '/', '.', '\\', '"', '\n')
  *
- * @returns {string} A CSV formatted string
+ * @param {Array} arrayToTransform - An array of data to transform. Each element can be of any type
+ * (string, number, object, boolean, date, etc.)
+ *
+ * @returns {string} A CSV formatted string with each value separated by commas,
+ * with a separated by newline characters (`\n`) to signify the end of the row.
+ *
+ * @private
  */
-function go(data) {
-  if (!data) {
+function transformArrayToCSVRow(arrayToTransform) {
+  if (!arrayToTransform) {
     return undefined
   }
 
-  return _transformDataToCSV(data)
-}
-
-/**
- * Transforms each row or header to CSV format and joins the values with commas
- *
- * @private
- */
-function _transformDataToCSV(data) {
-  const transformedRow = data
+  const transformedRow = arrayToTransform
     .map((value) => {
       return _transformValueToCSV(value)
     })
@@ -72,5 +73,5 @@ function _transformValueToCSV(value) {
 }
 
 module.exports = {
-  go
+  transformArrayToCSVRow
 }

app/services/jobs/export/write-table-to-file.service.js

@@ -11,7 +11,7 @@ const { pipeline, Transform } = require('stream')
 const path = require('path')
 const util = require('util')
 
-const ConvertToCSVService = require('./convert-to-csv.service.js')
+const { transformArrayToCSVRow } = require('../../../lib/transform-to-csv.lib.js')
 
 /**
  * Converts data into CSV format and writes it to a file
@@ -30,7 +30,7 @@ async function go(headers, rows, schemaFolderPath, tableName) {
 
   const transformDataStream = _transformDataStream()
 
-  const convertedHeaders = ConvertToCSVService.go(headers)
+  const convertedHeaders = transformArrayToCSVRow(headers)
 
   writeToFileStream.write(convertedHeaders)
 
@@ -48,7 +48,7 @@ function _transformDataStream() {
   return new Transform({
     objectMode: true,
     transform: function (row, _encoding, callback) {
-      const datRow = ConvertToCSVService.go(Object.values(row))
+      const datRow = transformArrayToCSVRow(Object.values(row))
 
       callback(null, datRow)
     }

test/lib/transform-to-csv.lib.test.js

@@ -0,0 +1,150 @@
+'use strict'
+
+// Test framework dependencies
+const Lab = require('@hapi/lab')
+const Code = require('@hapi/code')
+
+const { describe, it, beforeEach } = (exports.lab = Lab.script())
+const { expect } = Code
+
+// Thing under test
+const { transformArrayToCSVRow } = require('../../app/lib/transform-to-csv.lib.js')
+
+describe('Transform to csv', () => {
+  describe('#transformArrayToCSVRow', () => {
+    let testArray
+
+    beforeEach(() => {
+      testArray = _testArray()
+    })
+
+    it('correctly transforms all data types to csv', () => {
+      const result = transformArrayToCSVRow(testArray)
+
+      expect(result).to.equal(
+        '"20146cdc-9b40-4769-aa78-b51c17080d56",' +
+          '"4.1.1",' +
+          '9700,' +
+          '"Low loss tidal abstraction of water up to and ""including"" 25,002 megalitres, known as ML/yr a year where no model applies",' +
+          '2022-12-14T18:39:45.000Z,' +
+          'true,' +
+          ',' +
+          ',' +
+          'false,' +
+          ',' +
+          '25002,' +
+          '"{""message"": ""a json object""}"' +
+          '\n'
+      )
+    })
+
+    describe('when the data type is', () => {
+      describe('an object', () => {
+        it('correctly formats the object to a string', () => {
+          const result = transformArrayToCSVRow([{ message: 'a json object' }])
+
+          expect(result).to.equal('"{""message"": ""a json object""}"\n')
+        })
+      })
+
+      describe('a UUID', () => {
+        it('correctly formats the UUID to a string', () => {
+          const result = transformArrayToCSVRow(['20146cdc-9b40-4769-aa78-b51c17080d56'])
+
+          expect(result).to.equal('"20146cdc-9b40-4769-aa78-b51c17080d56"\n')
+        })
+      })
+
+      describe('a boolean', () => {
+        it('correctly formats the boolean to a string', () => {
+          const result = transformArrayToCSVRow([true])
+
+          expect(result).to.equal('true\n')
+        })
+      })
+
+      describe('a number', () => {
+        it('correctly formats the number to a string', () => {
+          const result = transformArrayToCSVRow([100])
+
+          expect(result).to.equal('100\n')
+        })
+      })
+
+      describe('a string containing', () => {
+        describe('a comma ,', () => {
+          it('correctly formats the string', () => {
+            const result = transformArrayToCSVRow(['I am a, comma seperated sentence.'])
+
+            expect(result).to.equal('"I am a, comma seperated sentence."\n')
+          })
+        })
+
+        describe('a single double quote "', () => {
+          it('correctly formats the string', () => {
+            const result = transformArrayToCSVRow(['I am a " double quote sentence.'])
+
+            expect(result).to.equal('"I am a "" double quote sentence."\n')
+          })
+        })
+
+        describe('a double double quote ""', () => {
+          it('correctly formats the string', () => {
+            const result = transformArrayToCSVRow(['I am a "" double quote sentence.'])
+
+            expect(result).to.equal('"I am a """" double quote sentence."\n')
+          })
+        })
+
+        describe('a back slash "\\" ', () => {
+          it('correctly formats the string', () => {
+            const result = transformArrayToCSVRow(['I am a  "\\"  back slash sentence.'])
+
+            expect(result).to.equal('"I am a  ""\\""  back slash sentence."\n')
+          })
+        })
+      })
+
+      describe('a date', () => {
+        it('correctly formats the date to an iso string', () => {
+          const result = transformArrayToCSVRow([new Date('2021-02-01')])
+
+          expect(result).to.equal('2021-02-01T00:00:00.000Z\n')
+        })
+      })
+    })
+
+    describe('when an array of strings us provided', () => {
+      it('converts the data to a CSV format', () => {
+        const result = transformArrayToCSVRow(['name', 'age'])
+
+        expect(result).to.equal('"name","age"\n')
+      })
+    })
+
+    describe('when no array is provided', () => {
+      it('returns undefined', () => {
+        const result = transformArrayToCSVRow()
+
+        expect(result).to.equal(undefined)
+      })
+    })
+  })
+})
+
+function _testArray() {
+  return [
+    '20146cdc-9b40-4769-aa78-b51c17080d56',
+    '4.1.1',
+    9700,
+    'Low loss tidal abstraction of water up to and "including" 25,002 megalitres, known as ML/yr a year where no model applies',
+    new Date(2022, 11, 14, 18, 39, 45),
+    true,
+    null,
+    undefined,
+    false,
+    '',
+    25002,
+    { message: 'a json object' }
+  ]
+}