chore: climate functions tidying

apps/picsa-apps/dashboard/src/app/modules/climate/climate-api.mapping.ts

@@ -110,7 +110,7 @@ export const ApiMapping = (
       // setup metadata
       const fileBlob = data as any as Blob;
       const bucketId = country_code as string;
-      const folderPath = 'climate/forecasts';
+      const folderPath = 'forecasts/daily';
       // upload to storage
       const { fullPath } = await storage.putFile({ bucketId, fileBlob, filename: filepath, folderPath });
 

apps/picsa-apps/dashboard/src/app/modules/climate/pages/forecast/forecast.component.ts

@@ -155,6 +155,7 @@ export class ClimateForecastPageComponent {
   }
 
   private async downloadStorageFile(row: IForecastRow) {
+    // TODO - invoke cloud function instead of direct
     this.activeDownloads.update((v) => ({ ...v, [row.id]: 'pending' }));
     const storagePath = await this.service.loadFromAPI.forecast_file(row);
     this.activeDownloads.update((v) => ({ ...v, [row.id]: 'complete' }));

apps/picsa-server/supabase/functions/dashboard/forecast-db.ts

@@ -61,9 +61,9 @@ async function getCountryUpdates(country_code: string, query_prefix: string) {
   // filter results to only include api forecasts not present on db
   const apiForecasts = await getApiForecasts({ country_code, query_prefix });
   const dbForecasts = await getDBForecasts({ country_code, query_prefix });
-
   const dbForecastIds = dbForecasts.map((v) => v.id);
   const newForecasts = apiForecasts.filter((v) => !dbForecastIds.includes(v.name));
+  console.log(`${country_code}: ${newForecasts.length} New Forecasts`);
   if (newForecasts.length === 0) {
     return [];
   }
@@ -92,12 +92,12 @@ async function getApiForecasts(query: { country_code: string; query_prefix?: str
 async function getDBForecasts(query: { country_code: string; query_prefix: string }) {
   const supabaseClient = getClient();
   const { country_code, query_prefix } = query;
-  console.log('db query', query_prefix, country_code);
   const { data, error } = await supabaseClient
     .from('forecasts')
     .select('*')
     .like('id', `${query_prefix}%`)
     .eq('country_code', country_code)
+    .eq('forecast_type', 'daily')
     .order('id', { ascending: false });
 
   if (error) {

apps/picsa-server/supabase/functions/dashboard/forecast-storage.ts

@@ -32,26 +32,30 @@ class ForecastStorageUpdate {
   }
 
   async populateStorageFiles(params: IReqParams) {
-    const { limit = 5 } = params;
+    const { limit = 20 } = params;
     const pending = await this.listPendingFiles(limit);
 
     const updates: IDBClimateForecastRow[] = [];
     const errors: any[] = [];
-    // TODO - make parallel and allow failure
-    for (const { country_code, id } of pending) {
+    const promises = pending.map(async ({ country_code, id }) => {
       const { data, error } = await this.storeForecast(country_code, id);
       if (error) {
         errors.push(error);
       }
       if (data) {
         updates.push(data);
       }
+    });
+    await Promise.allSettled(promises);
+    for (const error of errors) {
+      console.error(error);
     }
+    console.log(`[${updates.length}] storage files populates\n[${errors.length}] errors recorded`);
     return { data: updates, error: errors };
   }
 
   /** Check all climate forecast db entries for any that are missing corresponding storage files */
-  private async listPendingFiles(limit = 5) {
+  private async listPendingFiles(limit: number) {
     const query = this.table.select('*').is('storage_file', null).order('id', { ascending: false }).limit(limit);
     const { data, error } = await query;
     if (error) {
@@ -78,9 +82,13 @@ class ForecastStorageUpdate {
     if (fileData) {
       // upload to supabase storage
       const contentType = apiResponse.headers.get('content-type') as string;
+      const year = id.substring(0, 4);
+      const month = id.substring(4, 6);
+      const day = id.substring(6, 8);
+      const filename = id.substring(9);
       const { data: uploadData, error: uploadError } = await supabaseClient.storage
         .from(country_code)
-        .upload(`climate/forecasts/${id}`, fileData, { contentType, upsert: true });
+        .upload(`forecasts/daily/${year}/${month}/${day}/${filename}`, fileData, { contentType, upsert: true });
       if (uploadError) {
         return { error: uploadError };
       }