Improve handling of animated GIF and WEBP images (matrix-org#8153)

Author: t3chguy

package.json

@@ -166,6 +166,7 @@
     "@wojtekmaj/enzyme-adapter-react-17": "^0.6.1",
     "allchange": "^1.0.6",
     "babel-jest": "^26.6.3",
+    "blob-polyfill": "^6.0.20211015",
     "chokidar": "^3.5.1",
     "concurrently": "^5.3.0",
     "enzyme": "^3.11.0",

src/ContentMessages.ts

@@ -106,15 +106,17 @@ interface IThumbnail {
  * @param {HTMLElement} element The element to thumbnail.
  * @param {number} inputWidth The width of the image in the input element.
  * @param {number} inputHeight the width of the image in the input element.
- * @param {String} mimeType The mimeType to save the blob as.
+ * @param {string} mimeType The mimeType to save the blob as.
+ * @param {boolean} calculateBlurhash Whether to calculate a blurhash of the given image too.
  * @return {Promise} A promise that resolves with an object with an info key
  *  and a thumbnail key.
  */
-async function createThumbnail(
+export async function createThumbnail(
     element: ThumbnailableElement,
     inputWidth: number,
     inputHeight: number,
     mimeType: string,
+    calculateBlurhash = true,
 ): Promise<IThumbnail> {
     let targetWidth = inputWidth;
     let targetHeight = inputHeight;
@@ -152,7 +154,7 @@ async function createThumbnail(
 
     const imageData = context.getImageData(0, 0, targetWidth, targetHeight);
     // thumbnailPromise and blurhash promise are being awaited concurrently
-    const blurhash = await BlurhashEncoder.instance.getBlurhash(imageData);
+    const blurhash = calculateBlurhash ? await BlurhashEncoder.instance.getBlurhash(imageData) : undefined;
     const thumbnail = await thumbnailPromise;
 
     return {

src/components/views/messages/MImageBody.tsx

@@ -41,14 +41,12 @@ export default class MImageReplyBody extends MImageBody {
     }
 
     render() {
-        if (this.state.error !== null) {
+        if (this.state.error) {
             return super.render();
         }
 
         const content = this.props.mxEvent.getContent<IMediaEventContent>();
-
-        const contentUrl = this.getContentUrl();
-        const thumbnail = this.messageContent(contentUrl, this.getThumbUrl(), content, FORCED_IMAGE_HEIGHT);
+        const thumbnail = this.messageContent(this.state.contentUrl, this.state.thumbUrl, content, FORCED_IMAGE_HEIGHT);
         const fileBody = this.getFileBody();
         const sender = <SenderProfile
             mxEvent={this.props.mxEvent}

src/components/views/messages/MImageReplyBody.tsx

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2022 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export function mayBeAnimated(mimeType: string): boolean {
+    return ["image/gif", "image/webp"].includes(mimeType);
+}
+
+function arrayBufferRead(arr: ArrayBuffer, start: number, len: number): Uint8Array {
+    return new Uint8Array(arr.slice(start, start + len));
+}
+
+function arrayBufferReadStr(arr: ArrayBuffer, start: number, len: number): string {
+    return String.fromCharCode.apply(null, arrayBufferRead(arr, start, len));
+}
+
+export async function blobIsAnimated(mimeType: string, blob: Blob): Promise<boolean> {
+    if (mimeType === "image/webp") {
+        // Only extended file format WEBP images support animation, so grab the expected data range and verify header.
+        // Based on https://developers.google.com/speed/webp/docs/riff_container#extended_file_format
+        const arr = await blob.slice(0, 17).arrayBuffer();
+        if (
+            arrayBufferReadStr(arr, 0, 4) === "RIFF" &&
+            arrayBufferReadStr(arr, 8, 4) === "WEBP" &&
+            arrayBufferReadStr(arr, 12, 4) === "VP8X"
+        ) {
+            const [flags] = arrayBufferRead(arr, 16, 1);
+            // Flags: R R I L E X _A_ R (reversed)
+            const animationFlagMask = 1 << 1;
+            return (flags & animationFlagMask) != 0;
+        }
+    } else if (mimeType === "image/gif") {
+        // Based on https://gist.github.com/zakirt/faa4a58cec5a7505b10e3686a226f285
+        // More info at http://www.matthewflickinger.com/lab/whatsinagif/bits_and_bytes.asp
+        const dv = new DataView(await blob.arrayBuffer(), 10);
+
+        const globalColorTable = dv.getUint8(0);
+        let globalColorTableSize = 0;
+        // check first bit, if 0, then we don't have a Global Color Table
+        if (globalColorTable & 0x80) {
+            // grab the last 3 bits, to calculate the global color table size -> RGB * 2^(N+1)
+            // N is the value in the last 3 bits.
+            globalColorTableSize = 3 * Math.pow(2, (globalColorTable & 0x7) + 1);
+        }
+
+        // move on to the Graphics Control Extension
+        const offset = 3 + globalColorTableSize;
+
+        const extensionIntroducer = dv.getUint8(offset);
+        const graphicsControlLabel = dv.getUint8(offset + 1);
+        let delayTime = 0;
+
+        // Graphics Control Extension section is where GIF animation data is stored
+        // First 2 bytes must be 0x21 and 0xF9
+        if ((extensionIntroducer & 0x21) && (graphicsControlLabel & 0xF9)) {
+            // skip to the 2 bytes with the delay time
+            delayTime = dv.getUint16(offset + 4);
+        }
+
+        return !!delayTime;
+    }
+
+    return false;
+}

src/utils/Image.ts

@@ -52,6 +52,7 @@ const ALLOWED_BLOB_MIMETYPES = [
     'image/jpeg',
     'image/gif',
     'image/png',
+    'image/webp',
 
     'video/mp4',
     'video/webm',

src/utils/blobs.ts

@@ -0,0 +1,61 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import fs from "fs";
+import path from "path";
+
+import './skinned-sdk';
+import { blobIsAnimated, mayBeAnimated } from "../src/utils/Image";
+
+describe("Image", () => {
+    describe("mayBeAnimated", () => {
+        it("image/gif", async () => {
+            expect(mayBeAnimated("image/gif")).toBeTruthy();
+        });
+        it("image/webp", async () => {
+            expect(mayBeAnimated("image/webp")).toBeTruthy();
+        });
+        it("image/png", async () => {
+            expect(mayBeAnimated("image/png")).toBeFalsy();
+        });
+        it("image/jpeg", async () => {
+            expect(mayBeAnimated("image/jpeg")).toBeFalsy();
+        });
+    });
+
+    describe("blobIsAnimated", () => {
+        const animatedGif = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.gif"))]);
+        const animatedWebp = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.webp"))]);
+        const staticGif = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.gif"))]);
+        const staticWebp = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.webp"))]);
+
+        it("Animated GIF", async () => {
+            expect(await blobIsAnimated("image/gif", animatedGif)).toBeTruthy();
+        });
+
+        it("Static GIF", async () => {
+            expect(await blobIsAnimated("image/gif", staticGif)).toBeFalsy();
+        });
+
+        it("Animated WEBP", async () => {
+            expect(await blobIsAnimated("image/webp", animatedWebp)).toBeTruthy();
+        });
+
+        it("Static WEBP", async () => {
+            expect(await blobIsAnimated("image/webp", staticWebp)).toBeFalsy();
+        });
+    });
+});

test/Image-test.ts

@@ -1,6 +1,7 @@
 import { TextEncoder, TextDecoder } from 'util';
 import Adapter from "@wojtekmaj/enzyme-adapter-react-17";
 import { configure } from "enzyme";
+import "blob-polyfill"; // https://github.com/jsdom/jsdom/issues/2555
 
 import * as languageHandler from "../src/languageHandler";
 import SdkConfig, { DEFAULTS } from '../src/SdkConfig';

test/images/animated-logo.gif

@@ -2627,6 +2627,11 @@ binary-extensions@^2.0.0:
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
   integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==
 
+blob-polyfill@^6.0.20211015:
+  version "6.0.20211015"
+  resolved "https://registry.yarnpkg.com/blob-polyfill/-/blob-polyfill-6.0.20211015.tgz#7c47e62347e302e8d1d1ee5e140b881f74bdb23e"
+  integrity sha512-OGL4bm6ZNpdFAvQugRlQy5MNly8gk15aWi/ZhQHimQsrx9WKD05r+v+xNgHCChLER3MH+9KLAhzuFlwFKrH1Yw==
+
 bluebird@^3.5.0:
   version "3.7.2"
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"