Add Asset Regenerate Metadata Task

Lessify · Feb 2, 2025 · ec2ee2c · ec2ee2c
1 parent afe1c97
commit ec2ee2c
Show file tree

Hide file tree

Showing 19 changed files with 200 additions and 98 deletions.
diff --git a/functions/package-lock.json b/functions/package-lock.json
diff --git a/functions/package.json b/functions/package.json
@@ -1,6 +1,6 @@
 {
   "name": "functions",
-  "version": "2.5.0",
+  "version": "2.5.1",
   "scripts": {
     "lint": "eslint --ext .js,.ts .",
     "lint:fix": "eslint --fix --ext .js,.ts .",

diff --git a/functions/src/models/task.model.ts b/functions/src/models/task.model.ts
@@ -3,6 +3,7 @@ import { Timestamp } from 'firebase-admin/firestore';
 export enum TaskKind {
   ASSET_EXPORT = 'ASSET_EXPORT',
   ASSET_IMPORT = 'ASSET_IMPORT',
+  ASSET_REGEN_METADATA = 'ASSET_REGEN_METADATA',
   CONTENT_EXPORT = 'CONTENT_EXPORT',
   CONTENT_IMPORT = 'CONTENT_IMPORT',
   SCHEMA_EXPORT = 'SCHEMA_EXPORT',

diff --git a/functions/src/services/asset.service.ts b/functions/src/services/asset.service.ts
@@ -1,11 +1,12 @@
-import { DocumentReference, Query, Timestamp } from 'firebase-admin/firestore';
+import { DocumentReference, FieldValue, Query, Timestamp, UpdateData } from 'firebase-admin/firestore';
 import { logger } from 'firebase-functions/v2';
-import { firestoreService } from '../config';
-import { Asset, AssetExport, AssetFileExport, AssetFolderExport, AssetKind } from '../models';
+import { bucket, firestoreService } from '../config';
+import { Asset, AssetExport, AssetFile, AssetFileExport, AssetFolderExport, AssetKind } from '../models';
 import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
 import fs from 'fs';
 import os from 'os';
 import ffmpegStatic from 'ffmpeg-static';
+import sharp from 'sharp';
 
 /**
  * find Content by Full Slug
@@ -102,11 +103,9 @@ export function extractThumbnail(videoPath: string, outputImageName: string, tim
   return new Promise((resolve, reject) => {
     ffmpeg.setFfmpegPath(ffmpegStatic!);
     ffmpeg(videoPath)
-      .on('start', command => console.log(`FFmpeg command: ${command}`))
-      .on('progress', progress => console.log(`Processing: ${progress.percent}% done`))
       .on('end', () => {
         if (fs.existsSync(outputPath)) {
-          console.log(`Thumbnail saved at: ${outputPath}`);
+          logger.info(`Thumbnail saved at: ${outputPath}`);
           resolve();
         } else {
           reject(new Error('Error: Screenshot was not generated!'));
@@ -126,7 +125,7 @@ export function extractThumbnail(videoPath: string, outputImageName: string, tim
  * @param {string} video
  * @return {Promise<FfprobeData>} - metadata
  */
-export function extractMetadata(video: string): Promise<FfprobeData> {
+export function extractVideoMetadata(video: string): Promise<FfprobeData> {
   return new Promise((resolve, reject) => {
     ffmpeg.setFfmpegPath(ffmpegStatic!);
     ffmpeg.ffprobe(video, (err, metadata) => {
@@ -138,3 +137,84 @@ export function extractMetadata(video: string): Promise<FfprobeData> {
     });
   });
 }
+
+/**
+ * Update Asset Metadata
+ * @param {string} assetRef - firestore asset reference
+ */
+export async function updateMetadataByRef(assetRef: DocumentReference): Promise<void> {
+  const storagePath = `${assetRef.path}/original`;
+  const assetDocSnapshot = await assetRef.get();
+  const asset = assetDocSnapshot.data() as Asset;
+  const update: UpdateData<AssetFile> = {
+    inProgress: FieldValue.delete(),
+    updatedAt: FieldValue.serverTimestamp(),
+  };
+  if (asset.kind === AssetKind.FILE) {
+    if (asset.type.startsWith('image/')) {
+      // Image
+      const [file] = await bucket.file(storagePath).download();
+      const { size, width, height, format, pages, isProgressive } = await sharp(file).metadata();
+      if (size) {
+        update.size = size;
+      }
+      update.metadata = {
+        type: 'image',
+        format: format,
+        height: height,
+        width: width,
+        animated: pages !== undefined,
+        progressive: isProgressive,
+      };
+      // calculate orientation
+      if (width && height) {
+        if (width > height) {
+          update.metadata.orientation = 'landscape';
+        } else if (height > width) {
+          update.metadata.orientation = 'portrait';
+        } else {
+          update.metadata.orientation = 'squarish';
+        }
+      }
+    } else if (asset.type.startsWith('video/')) {
+      // Video
+      const tempFilePath = `${os.tmpdir()}/assets-tmp`;
+      await bucket.file(storagePath).download({ destination: tempFilePath });
+      const metadata = await extractVideoMetadata(tempFilePath);
+      update.metadata = {
+        type: 'video',
+        format: metadata.format.format_name,
+        formatLong: metadata.format.format_long_name,
+        duration: metadata.format.duration,
+        bitRate: metadata.format.bit_rate,
+      };
+      if (metadata.streams.length > 0) {
+        const video = metadata.streams[0];
+        update.metadata.height = video.height;
+        update.metadata.width = video.width;
+        // calculate orientation
+        const { width, height } = video;
+        if (width && height) {
+          if (width > height) {
+            update.metadata.orientation = 'landscape';
+          } else if (height > width) {
+            update.metadata.orientation = 'portrait';
+          } else {
+            update.metadata.orientation = 'squarish';
+          }
+        }
+      }
+    }
+  } else {
+    return;
+  }
+  await assetRef.update(update);
+}
+/**
+ * Update Asset Metadata
+ * @param {string} assetRefPath - firestore asset reference path
+ */
+export async function updateMetadataByPath(assetRefPath: string): Promise<void> {
+  const assetRef = firestoreService.doc(assetRefPath);
+  return updateMetadataByRef(assetRef);
+}
diff --git a/functions/src/services/open-api.service.ts b/functions/src/services/open-api.service.ts
@@ -302,7 +302,7 @@ export function generateOpenApi(schemasById: Map<string, Schema>): OpenAPIObject
     openapi: '3.0.3',
     info: {
       title: 'Localess Open API Specification',
-      version: '2.5.0',
+      version: '2.5.1',
       description: 'Fetch data from Localess via REST API',
       contact: {
         name: 'Lessify Team',
@@ -695,7 +695,7 @@ export function generateOpenApi(schemasById: Map<string, Schema>): OpenAPIObject
             {
               name: 'thumbnail',
               in: 'query',
-              description: 'In case you have animated image like WebP/Gif, and you wish to generate non animated thumbnail.',
+              description: 'In case you have a video or animated image like WebP/Gif, and you wish to generate thumbnail.',
               required: false,
               schema: {
                 type: 'boolean',

diff --git a/functions/src/storage.ts b/functions/src/storage.ts
@@ -1,80 +1,16 @@
 import { logger } from 'firebase-functions/v2';
 import { onObjectFinalized } from 'firebase-functions/v2/storage';
-import { FieldValue, UpdateData } from 'firebase-admin/firestore';
-import sharp from 'sharp';
-import { bucket, firestoreService } from './config';
-import { AssetFile } from './models';
-import { extractMetadata } from './services';
-import os from 'os';
+import { updateMetadataByPath } from './services';
 
 const onFileUpload = onObjectFinalized(async event => {
   logger.info(`[Storage::onFileUpload] name : ${event.data.name}`);
   // logger.info(event.data);
-  const { name, contentType } = event.data;
+  const { name } = event.data;
   // Spaces Assets
   // spaces/eo42RwNL8XHD7Cdvd8eO/assets/RpMDPKkmDM66Vc1jgDpo/original
   if (name && name.startsWith('spaces/') && name.includes('assets') && name.endsWith('/original')) {
     const assetPath = name.slice(0, -9); // remove '/original'
-    const assetRef = firestoreService.doc(assetPath);
-    const update: UpdateData<AssetFile> = {
-      inProgress: FieldValue.delete(),
-      updatedAt: FieldValue.serverTimestamp(),
-    };
-    if (contentType) {
-      if (contentType.startsWith('image/')) {
-        const [file] = await bucket.file(name).download();
-        const { size, width, height, format, pages, isProgressive } = await sharp(file).metadata();
-        if (size) {
-          update.size = size;
-        }
-        update.metadata = {
-          type: 'image',
-          format: format,
-          height: height,
-          width: width,
-          animated: pages !== undefined,
-          progressive: isProgressive,
-        };
-        // calculate orientation
-        if (width && height) {
-          if (width > height) {
-            update.metadata.orientation = 'landscape';
-          } else if (height > width) {
-            update.metadata.orientation = 'portrait';
-          } else {
-            update.metadata.orientation = 'squarish';
-          }
-        }
-      } else if (contentType.startsWith('video/')) {
-        const tempFilePath = `${os.tmpdir()}/assets-tmp`;
-        await bucket.file(name).download({ destination: tempFilePath });
-        const metadata = await extractMetadata(tempFilePath);
-        update.metadata = {
-          type: 'video',
-          format: metadata.format.format_name,
-          formatLong: metadata.format.format_long_name,
-          duration: metadata.format.duration,
-          bitRate: metadata.format.bit_rate,
-        };
-        if (metadata.streams.length > 0) {
-          const video = metadata.streams[0];
-          update.metadata.height = video.height;
-          update.metadata.width = video.width;
-          // calculate orientation
-          const { width, height } = video;
-          if (width && height) {
-            if (width > height) {
-              update.metadata.orientation = 'landscape';
-            } else if (height > width) {
-              update.metadata.orientation = 'portrait';
-            } else {
-              update.metadata.orientation = 'squarish';
-            }
-          }
-        }
-      }
-    }
-    await assetRef.update(update);
+    await updateMetadataByPath(assetPath);
   }
   return;
 });

diff --git a/functions/src/tasks.ts b/functions/src/tasks.ts
@@ -46,7 +46,7 @@ import {
   findSchemaById,
   findSchemas,
   findTranslationById,
-  findTranslations,
+  findTranslations, updateMetadataByRef,
 } from './services';
 import { tmpdir } from 'os';
 import { ZodError } from 'zod/lib/ZodError';
@@ -109,6 +109,8 @@ const onTaskCreate = onDocumentCreated(
           updateToFinished.trace = JSON.stringify(errors.format());
         }
       }
+    } else if (task.kind === TaskKind.ASSET_REGEN_METADATA) {
+      await assetRegenerateMetadata(spaceId);
     } else if (task.kind === TaskKind.CONTENT_EXPORT) {
       const metadata = await contentsExport(spaceId, taskId, task);
       updateToFinished['file'] = {
@@ -358,6 +360,19 @@ async function assetsImport(spaceId: string, taskId: string): Promise<ZodError |
   return undefined;
 }
 
+/**
+ * Asset Regenerate Metadata Job
+ * @param {string} spaceId original task
+ */
+async function assetRegenerateMetadata(spaceId: string): Promise<void> {
+  const assetsSnapshot = await findAssets(spaceId, AssetKind.FILE).get();
+  for (const assetSnapshot of assetsSnapshot.docs) {
+    logger.info('[Task:onCreate:assetsRegenMetadata] asset : ' + assetSnapshot.ref.path);
+    await updateMetadataByRef(assetSnapshot.ref);
+  }
+  return undefined;
+}
+
 /**
  * contentExport Job
  * @param {string} spaceId original task

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "localess",
-  "version": "2.5.0",
+  "version": "2.5.1",
   "engines": {
     "node": "20"
   },

diff --git a/src/app/features/admin/users/user-dialog/user-dialog.component.html b/src/app/features/admin/users/user-dialog/user-dialog.component.html
@@ -150,6 +150,10 @@ <h2 mat-dialog-title>User</h2>
           <span matListItemTitle>Export</span>
           <span matListItemLine>Export Asset.</span>
         </mat-list-option>
+        <mat-list-option value="ASSET_REGEN_METADATA">
+          <span matListItemTitle>Regenerate Metadata</span>
+          <span matListItemLine>Regenerate Metadata related to media files.</span>
+        </mat-list-option>
         <mat-list-option value="ASSET_IMPORT">
           <span matListItemTitle>Import</span>
           <span matListItemLine>Import Asset.</span>

diff --git a/src/app/features/admin/users/user-invite-dialog/user-invite-dialog.component.html b/src/app/features/admin/users/user-invite-dialog/user-invite-dialog.component.html
@@ -172,6 +172,10 @@ <h2 mat-dialog-title>User Invite</h2>
           <span matListItemTitle>Import</span>
           <span matListItemLine>Import Asset.</span>
         </mat-list-option>
+        <mat-list-option value="ASSET_REGEN_METADATA">
+          <span matListItemTitle>Regenerate Metadata</span>
+          <span matListItemLine>Regenerate Metadata related to media files.</span>
+        </mat-list-option>
 
         <mat-divider></mat-divider>
         <div mat-subheader>Development</div>

diff --git a/src/app/features/spaces/assets/assets.component.html b/src/app/features/spaces/assets/assets.component.html
@@ -72,6 +72,13 @@
             <span>Export</span>
           </button>
         }
+        @if ('ASSET_REGEN_METADATA' | canUserPerform | async) {
+          <mat-divider />
+          <button mat-menu-item (click)="openRegenerateMetadataDialog()">
+            <mat-icon>sync_alt</mat-icon>
+            <span>Regenerate Metadata</span>
+          </button>
+        }
       </mat-menu>
     </div>
   </mat-toolbar-row>

diff --git a/src/app/features/spaces/assets/assets.component.ts b/src/app/features/spaces/assets/assets.component.ts
@@ -499,6 +499,37 @@ export class AssetsComponent implements OnInit {
       });
   }
 
+  openRegenerateMetadataDialog(): void {
+    this.dialog
+      .open<ConfirmationDialogComponent, ConfirmationDialogModel, boolean>(ConfirmationDialogComponent, {
+        data: {
+          title: 'Regenerate Metadata',
+          content: `Are you sure about regenerating assets metadata? It is a long running job, it may take from few minutes till one hour.`,
+        },
+      })
+      .afterClosed()
+      .pipe(
+        filter(it => it || false),
+        switchMap(() => this.taskService.createAssetRegenerateMetadataTask(this.spaceId())),
+      )
+      .subscribe({
+        next: () => {
+          this.selection.clear();
+          this.cd.markForCheck();
+          this.notificationService.success('Assets Regenerate Metadata Task has been created.', [
+            {
+              label: 'To Tasks',
+              link: `/features/spaces/${this.spaceId()}/tasks`,
+            },
+          ]);
+        },
+        error: (err: unknown) => {
+          console.error(err);
+          this.notificationService.error(`Assets Regenerate Metadata Task can not be created.`);
+        },
+      });
+  }
+
   onDownload(event: Event, element: Asset): void {
     // Prevent Default
     event.preventDefault();