From d97e82f56a40e3d4d33c86c82c609a587d5530fa Mon Sep 17 00:00:00 2001
From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com>
Date: Sun, 14 Jul 2024 05:06:43 -0700
Subject: [PATCH] metrics: download processed-logs in batches of 50 each
 (#3812)

---
 .../metrics/files/summaries/summaries.js      | 28 +++++++++++++------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/ansible/roles/metrics/files/summaries/summaries.js b/ansible/roles/metrics/files/summaries/summaries.js
index f5baebce0..1430e39ee 100755
--- a/ansible/roles/metrics/files/summaries/summaries.js
+++ b/ansible/roles/metrics/files/summaries/summaries.js
@@ -76,15 +76,25 @@ async function collectData (date) {
   const filePrefix = date.toString().concat('/')
   console.log(filePrefix)
   const [files] = await storage.bucket('processed-logs-nodejs').getFiles({ prefix: `${filePrefix}`})
-  for (const file of files) {
-    const data = await storage.bucket('processed-logs-nodejs').file(file.name).download()
-    const stringContents = data[0].toString()
-    const contentsArray = stringContents.split('\n')
-    for (const line of contentsArray) {
-      try {
-        const csvparse = csvStream(line)
-        if (csvparse !== undefined && csvparse[0][0] !== '') { summary(date, csvparse) }
-      } catch (err) { console.log(err) }
+
+  const filesBatches = []
+  for (let i = 0; i < files.length; i += 50) {
+    filesBatches.push(files.slice(i, i + 50))
+  }
+
+  for (const fileBatch of filesBatches) {
+    const dataBatch = await Promise.all(
+      fileBatch.map(file => storage.bucket('processed-logs-nodejs').file(file.name).download())
+    )
+    for (const data of dataBatch) {
+      const stringContents = data[0].toString()
+      const contentsArray = stringContents.split('\n')
+      for (const line of contentsArray) {
+        try {
+          const csvparse = csvStream(line)
+          if (csvparse !== undefined && csvparse[0][0] !== '') { summary(date, csvparse) }
+        } catch (err) { console.log(err) }
+      }
     }
   }
 }