Merge pull request #161 from wchaws/dev

feat: support parameter store to bypass large gif processing

Author: wchaws

source/constructs/cdk.context.json

@@ -12,6 +12,7 @@
   "//enable_cloudfront": true,
   "secret_arn": "arn:aws:secretsmanager:<region>:<account-id-number>:secret:<secret-name>-<random-6-characters>",
   "stack_tags":{"key1":"value1", "key2":"value2"},
+  "config_json_parameter_name": "<parameter-name>",
   "ecs_desired_count": 10,
   "env": {
     "SHARP_QUEUE_LIMIT": "1"

source/constructs/lib/ecs-image-handler.ts

@@ -7,6 +7,7 @@ import * as ecs from '@aws-cdk/aws-ecs';
 import * as ecsPatterns from '@aws-cdk/aws-ecs-patterns';
 import * as iam from '@aws-cdk/aws-iam';
 import * as s3 from '@aws-cdk/aws-s3';
+import * as ssm from '@aws-cdk/aws-ssm';
 import * as secretsmanager from '@aws-cdk/aws-secretsmanager';
 import { Aws, CfnOutput, Construct, Duration, Stack } from '@aws-cdk/core';
 
@@ -34,6 +35,7 @@ export class ECSImageHandler extends Construct {
 
     this.cfnOutput('StyleConfig', table.tableName, 'The DynamoDB table for processing style');
 
+    const configJsonParameter = this.getConfigJsonParameter();
     const vpc = getOrCreateVpc(this);
     const taskSubnets = getTaskSubnets(this, vpc);
     const albFargateService = new ecsPatterns.ApplicationLoadBalancedFargateService(this, 'Service', {
@@ -59,12 +61,15 @@ export class ECSImageHandler extends Construct {
           SRC_BUCKET: buckets[0].bucketName,
           STYLE_TABLE_NAME: table.tableName,
           SECRET_NAME: secret?.secretArn,
+          CONFIG_JSON_PARAMETER_NAME: configJsonParameter.parameterName,
         }, scope.node.tryGetContext('env')),
       },
     });
     albFargateService.targetGroup.configureHealthCheck({
       path: '/',
       healthyThresholdCount: 3,
+      timeout: Duration.seconds(10),
+      interval: Duration.seconds(60),
     });
     albFargateService.service.autoScaleTaskCount({
       minCapacity: 8,
@@ -76,6 +81,8 @@ export class ECSImageHandler extends Construct {
 
     const taskRole = albFargateService.taskDefinition.taskRole;
     table.grantReadData(taskRole);
+    configJsonParameter.grantRead(taskRole);
+
     for (const bkt of buckets) {
 
       taskRole.addToPrincipalPolicy(new iam.PolicyStatement({
@@ -156,6 +163,15 @@ export class ECSImageHandler extends Construct {
     o.overrideLogicalId(id);
     return o;
   }
+
+  private getConfigJsonParameter() {
+    const name = this.node.tryGetContext('config_json_parameter_name');
+    if (name) {
+      return ssm.StringParameter.fromStringParameterName(this, 'ConfigJsonParameter', name);
+    } else {
+      throw new Error('Missing "config_json_parameter_name" in cdk.context.json');
+    }
+  }
 }
 
 function getOrCreateVpc(scope: Construct): ec2.IVpc {

source/constructs/package.json

@@ -7,6 +7,7 @@ test('Snapshot', () => {
     context: {
       buckets: ['bucket-0'],
       secret_arn: 'arn:aws:secretsmanager:us-east-9:123456789012:secret:test-aaabbb',
+      config_json_parameter_name: 'config_json_parameter_name',
     },
   });
   const stack = new ECSImageHandlerStack(app, 'test');

source/constructs/test/ecs-image-handler.test.ts

@@ -7,17 +7,19 @@ export interface IConfig {
   autoWebp: boolean;
   secretName: string;
   sharpQueueLimit: number;
+  configJsonParameterName: string;
 }
 
 const conf: IConfig = {
   port: 8080,
-  region: process.env.REGION ?? 'us-west-2',
+  region: process.env.REGION ?? process.env.AWS_REGION ?? 'us-west-2',
   isProd: process.env.NODE_ENV === 'production',
   srcBucket: process.env.BUCKET || process.env.SRC_BUCKET || 'sih-input',
   styleTableName: process.env.STYLE_TABLE_NAME || 'style-table-name',
   autoWebp: ['yes', '1', 'true'].includes((process.env.AUTO_WEBP ?? '').toLowerCase()),
   secretName: process.env.SECRET_NAME ?? 'X-Client-Authorization',
   sharpQueueLimit: Number.parseInt(process.env.SHARP_QUEUE_LIMIT ?? '1', 10),
+  configJsonParameterName: process.env.CONFIG_JSON_PARAMETER_NAME ?? '',
 };
 
 export default conf;

source/new-image-handler/src/config.ts

@@ -14,6 +14,14 @@ const PROCESSOR_MAP: { [key: string]: IProcessor } = {
   [VideoProcessor.getInstance().name]: VideoProcessor.getInstance(),
 };
 
+export function setMaxGifSizeMB(value: number) {
+  ImageProcessor.getInstance().setMaxGifSizeMB(value);
+}
+
+export function setMaxGifPages(value: number) {
+  ImageProcessor.getInstance().setMaxGifPages(value);
+}
+
 export function getProcessor(name: string): IProcessor {
   const processor = PROCESSOR_MAP[name];
   if (!processor) {

source/new-image-handler/src/default.ts

@@ -1,5 +1,6 @@
 import * as S3 from 'aws-sdk/clients/s3';
 import * as SecretsManager from 'aws-sdk/clients/secretsmanager';
+import * as SSM from 'aws-sdk/clients/ssm';
 import * as HttpErrors from 'http-errors';
 import * as Koa from 'koa'; // http://koajs.cn
 import * as bodyParser from 'koa-bodyparser';
@@ -8,9 +9,13 @@ import * as Router from 'koa-router';
 import * as sharp from 'sharp';
 import config from './config';
 import debug from './debug';
-import { bufferStore, getProcessor, parseRequest } from './default';
+import { bufferStore, getProcessor, parseRequest, setMaxGifSizeMB, setMaxGifPages } from './default';
+import * as is from './is';
 import { IHttpHeaders, InvalidArgument } from './processor';
 
+const ssm = new SSM({ region: config.region });
+const smclient = new SecretsManager({ region: config.region });
+
 const DefaultBufferStore = bufferStore();
 const app = new Koa();
 const router = new Router();
@@ -47,6 +52,12 @@ router.post('/images', async (ctx) => {
 
 router.get(['/', '/ping'], async (ctx) => {
   ctx.body = 'ok';
+
+  try {
+    await setMaxGifLimit();
+  } catch (err: any) {
+    console.error(err);
+  }
 });
 
 router.get(['/debug', '/_debug'], async (ctx) => {
@@ -162,11 +173,6 @@ function bypass() {
   throw new HttpErrors[403]('Please visit s3 directly');
 }
 
-// Create a Secrets Manager client
-const smclient = new SecretsManager({
-  region: config.region,
-});
-
 async function getSecretFromSecretsManager() {
   // Load the AWS SDK
   const secretName = config.secretName;
@@ -178,4 +184,21 @@ async function getHeaderFromSecretsManager() {
   const secretString = secret.SecretString!;
   const keypair = JSON.parse(secretString);
   return keypair['X-Client-Authorization'];
+}
+
+async function setMaxGifLimit() {
+  if (config.configJsonParameterName) {
+    const data = await ssm.getParameter({ Name: config.configJsonParameterName }).promise();
+    if (data.Parameter) {
+      const configJson = JSON.parse(data.Parameter.Value ?? '{}');
+      const maxGifSizeMB = configJson.max_gif_size_mb;
+      if (is.number(maxGifSizeMB)) {
+        setMaxGifSizeMB(maxGifSizeMB);
+      }
+      const maxGifPages = configJson.max_gif_pages;
+      if (is.number(maxGifPages)) {
+        setMaxGifPages(maxGifPages);
+      }
+    }
+  }
 }

source/new-image-handler/src/index.ts

@@ -42,11 +42,29 @@ export class ImageProcessor implements IProcessor {
   }
   private static _instance: ImageProcessor;
   private readonly _actions: { [name: string]: IAction } = {};
+  private _maxGifSizeMB: number = 5;
+  private _maxGifPages: number = 100;
 
   public readonly name: string = 'image';
 
   private constructor() { }
 
+  public setMaxGifSizeMB(value: number) {
+    if (value > 0) {
+      this._maxGifSizeMB = value;
+    } else {
+      console.warn(`Max gif size must > 0, but the value is ${value}`);
+    }
+  }
+
+  public setMaxGifPages(value: number) {
+    if (value > 0) {
+      this._maxGifPages = value;
+    } else {
+      console.warn(`Max gif pages must > 0, but the value is ${value}`);
+    }
+  }
+
   public async newContext(uri: string, actions: string[], bufferStore: IBufferStore): Promise<IImageContext> {
     const ctx: IProcessContext = {
       uri,
@@ -94,6 +112,14 @@ export class ImageProcessor implements IProcessor {
     }
     if ('gif' === metadata.format) {
       image.gif({ effort: 1 }); // https://github.com/lovell/sharp/issues/3176
+
+      if (metadata.size && metadata.size > (this._maxGifSizeMB * MB)) {
+        console.log(`Gif processing skipped. The image size exceeds ${this._maxGifSizeMB} MB`);
+        ctx.mask.disableAll();
+      } else if (metadata.pages && metadata.pages > this._maxGifPages) {
+        console.log(`Gif processing skipped. The image pages exceeds ${this._maxGifPages}`);
+        ctx.mask.disableAll();
+      }
     }
     if ('png' === metadata.format && metadata.size && metadata.size > (5 * MB)) {
       image.png({ adaptiveFiltering: true });
@@ -135,7 +161,7 @@ export class ImageProcessor implements IProcessor {
       act.beforeProcess.bind(act)(ctx, params, index);
     });
     const enabledActions = ctx.mask.filterEnabledActions();
-    const nothing2do = (enabledActions.length === 1) && (this.name === enabledActions[0]);
+    const nothing2do = (enabledActions.length === 0) || ((enabledActions.length === 1) && (this.name === enabledActions[0]));
 
     if (nothing2do && (!ctx.features[Features.AutoWebp])) {
       const { buffer } = await ctx.bufferStore.get(ctx.uri);