From c253b1ae480ffe11a3c4faf95da25c4f67348e3a Mon Sep 17 00:00:00 2001
From: chrismclarke <chris.m.clarke@live.co.uk>
Date: Thu, 27 Feb 2025 16:35:20 -0800
Subject: [PATCH] chore: climate functions tidying

---
 .../app/modules/climate/climate-api.mapping.ts |  2 +-
 .../pages/forecast/forecast.component.ts       |  1 +
 .../functions/dashboard/forecast-db.ts         |  4 ++--
 .../functions/dashboard/forecast-storage.ts    | 18 +++++++++++++-----
 4 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/apps/picsa-apps/dashboard/src/app/modules/climate/climate-api.mapping.ts b/apps/picsa-apps/dashboard/src/app/modules/climate/climate-api.mapping.ts
index 560c779d..91d304ed 100644
--- a/apps/picsa-apps/dashboard/src/app/modules/climate/climate-api.mapping.ts
+++ b/apps/picsa-apps/dashboard/src/app/modules/climate/climate-api.mapping.ts
@@ -110,7 +110,7 @@ export const ApiMapping = (
       // setup metadata
       const fileBlob = data as any as Blob;
       const bucketId = country_code as string;
-      const folderPath = 'climate/forecasts';
+      const folderPath = 'forecasts/daily';
       // upload to storage
       const { fullPath } = await storage.putFile({ bucketId, fileBlob, filename: filepath, folderPath });
 
diff --git a/apps/picsa-apps/dashboard/src/app/modules/climate/pages/forecast/forecast.component.ts b/apps/picsa-apps/dashboard/src/app/modules/climate/pages/forecast/forecast.component.ts
index 8fba1bc9..f381581b 100644
--- a/apps/picsa-apps/dashboard/src/app/modules/climate/pages/forecast/forecast.component.ts
+++ b/apps/picsa-apps/dashboard/src/app/modules/climate/pages/forecast/forecast.component.ts
@@ -155,6 +155,7 @@ export class ClimateForecastPageComponent {
   }
 
   private async downloadStorageFile(row: IForecastRow) {
+    // TODO - invoke cloud function instead of direct
     this.activeDownloads.update((v) => ({ ...v, [row.id]: 'pending' }));
     const storagePath = await this.service.loadFromAPI.forecast_file(row);
     this.activeDownloads.update((v) => ({ ...v, [row.id]: 'complete' }));
diff --git a/apps/picsa-server/supabase/functions/dashboard/forecast-db.ts b/apps/picsa-server/supabase/functions/dashboard/forecast-db.ts
index 38691bc5..aaa9b484 100644
--- a/apps/picsa-server/supabase/functions/dashboard/forecast-db.ts
+++ b/apps/picsa-server/supabase/functions/dashboard/forecast-db.ts
@@ -61,9 +61,9 @@ async function getCountryUpdates(country_code: string, query_prefix: string) {
   // filter results to only include api forecasts not present on db
   const apiForecasts = await getApiForecasts({ country_code, query_prefix });
   const dbForecasts = await getDBForecasts({ country_code, query_prefix });
-
   const dbForecastIds = dbForecasts.map((v) => v.id);
   const newForecasts = apiForecasts.filter((v) => !dbForecastIds.includes(v.name));
+  console.log(`${country_code}: ${newForecasts.length} New Forecasts`);
   if (newForecasts.length === 0) {
     return [];
   }
@@ -92,12 +92,12 @@ async function getApiForecasts(query: { country_code: string; query_prefix?: str
 async function getDBForecasts(query: { country_code: string; query_prefix: string }) {
   const supabaseClient = getClient();
   const { country_code, query_prefix } = query;
-  console.log('db query', query_prefix, country_code);
   const { data, error } = await supabaseClient
     .from('forecasts')
     .select('*')
     .like('id', `${query_prefix}%`)
     .eq('country_code', country_code)
+    .eq('forecast_type', 'daily')
     .order('id', { ascending: false });
 
   if (error) {
diff --git a/apps/picsa-server/supabase/functions/dashboard/forecast-storage.ts b/apps/picsa-server/supabase/functions/dashboard/forecast-storage.ts
index 187b2434..46d538ed 100644
--- a/apps/picsa-server/supabase/functions/dashboard/forecast-storage.ts
+++ b/apps/picsa-server/supabase/functions/dashboard/forecast-storage.ts
@@ -32,13 +32,12 @@ class ForecastStorageUpdate {
   }
 
   async populateStorageFiles(params: IReqParams) {
-    const { limit = 5 } = params;
+    const { limit = 20 } = params;
     const pending = await this.listPendingFiles(limit);
 
     const updates: IDBClimateForecastRow[] = [];
     const errors: any[] = [];
-    // TODO - make parallel and allow failure
-    for (const { country_code, id } of pending) {
+    const promises = pending.map(async ({ country_code, id }) => {
       const { data, error } = await this.storeForecast(country_code, id);
       if (error) {
         errors.push(error);
@@ -46,12 +45,17 @@ class ForecastStorageUpdate {
       if (data) {
         updates.push(data);
       }
+    });
+    await Promise.allSettled(promises);
+    for (const error of errors) {
+      console.error(error);
     }
+    console.log(`[${updates.length}] storage files populates\n[${errors.length}] errors recorded`);
     return { data: updates, error: errors };
   }
 
   /** Check all climate forecast db entries for any that are missing corresponding storage files */
-  private async listPendingFiles(limit = 5) {
+  private async listPendingFiles(limit: number) {
     const query = this.table.select('*').is('storage_file', null).order('id', { ascending: false }).limit(limit);
     const { data, error } = await query;
     if (error) {
@@ -78,9 +82,13 @@ class ForecastStorageUpdate {
     if (fileData) {
       // upload to supabase storage
       const contentType = apiResponse.headers.get('content-type') as string;
+      const year = id.substring(0, 4);
+      const month = id.substring(4, 6);
+      const day = id.substring(6, 8);
+      const filename = id.substring(9);
       const { data: uploadData, error: uploadError } = await supabaseClient.storage
         .from(country_code)
-        .upload(`climate/forecasts/${id}`, fileData, { contentType, upsert: true });
+        .upload(`forecasts/daily/${year}/${month}/${day}/${filename}`, fileData, { contentType, upsert: true });
       if (uploadError) {
         return { error: uploadError };
       }