[tfjs-react-native] make camera utils robust to unmounting

BUG
* Close image tensor generator when component is unmounted
* Make camera utils robust to unmounting
* Enhancement. pass camera texture to callback

Author: tafsiri

tfjs-react-native/integration_rn59/components/webcam/realtime_demo.tsx

@@ -16,7 +16,7 @@
  */
 
 import React from 'react';
-import {ActivityIndicator, StyleSheet, View, Platform } from 'react-native';
+import {ActivityIndicator, Button, StyleSheet, View, Platform } from 'react-native';
 import Svg, { Circle, Rect, G, Line} from 'react-native-svg';
 
 import * as Permissions from 'expo-permissions';
@@ -54,6 +54,8 @@ const AUTORENDER = true;
 const TensorCamera = cameraWithTensors(Camera);
 
 export class RealtimeDemo extends React.Component<ScreenProps,ScreenState> {
+  rafID?: number;
+
   constructor(props: ScreenProps) {
     super(props);
     this.state = {
@@ -95,8 +97,6 @@ export class RealtimeDemo extends React.Component<ScreenProps,ScreenState> {
           const flipHorizontal = Platform.OS === 'ios' ? false : true;
           const pose = await this.state.posenetModel.estimateSinglePose(
             imageTensor, { flipHorizontal });
-
-          // console.log('pose', pose);
           this.setState({pose});
           tf.dispose([imageTensor]);
         }
@@ -115,12 +115,18 @@ export class RealtimeDemo extends React.Component<ScreenProps,ScreenState> {
       if(!AUTORENDER) {
         gl.endFrameEXP();
       }
-      requestAnimationFrame(loop);
+      this.rafID = requestAnimationFrame(loop);
     };
 
     loop();
   }
 
+  componentWillUnmount() {
+    if(this.rafID) {
+      cancelAnimationFrame(this.rafID);
+    }
+  }
+
   async componentDidMount() {
     const { status } = await Permissions.askAsync(Permissions.CAMERA);
 
@@ -254,11 +260,17 @@ export class RealtimeDemo extends React.Component<ScreenProps,ScreenState> {
       />
       <View style={styles.modelResults}>
         {modelName === 'posenet' ? this.renderPose() : this.renderFaces()}
-
       </View>
     </View>;
+
     return (
       <View style={{width:'100%'}}>
+        <View style={styles.sectionContainer}>
+          <Button
+            onPress={this.props.returnToMain}
+            title='Back'
+          />
+        </View>
         {isLoading ? <View style={[styles.loadingIndicator]}>
           <ActivityIndicator size='large' color='#FF0266' />
         </View> : camView}
@@ -275,6 +287,10 @@ const styles = StyleSheet.create({
     right: 20,
     zIndex: 200,
   },
+  sectionContainer: {
+    marginTop: 32,
+    paddingHorizontal: 24,
+  },
   cameraContainer: {
     display: 'flex',
     flexDirection: 'column',

tfjs-react-native/src/camera/camera_stream.tsx

@@ -43,7 +43,8 @@ interface Props {
   onReady: (
     images: IterableIterator<tf.Tensor3D>,
     updateCameraPreview: () => void,
-    gl: ExpoWebGLRenderingContext
+    gl: ExpoWebGLRenderingContext,
+    cameraTexture: WebGLTexture,
   ) => void;
 }
 
@@ -77,13 +78,14 @@ const DEFAULT_RESIZE_DEPTH = 3;
  * - __resizeHeight__: number — the height of the output tensor
  * - __resizeDepth__: number — the depth (num of channels) of the output tensor.
  *    Should be 3 or 4.
- * - __autorender__: boolean — if true the view will be automatically updated with
- *   the contents of the camera. Set this to false if you want more direct
+ * - __autorender__: boolean — if true the view will be automatically updated
+ *   with the contents of the camera. Set this to false if you want more direct
  *   control on when rendering happens.
  * - __onReady__: (
  *    images: IterableIterator<tf.Tensor3D>,
  *    updateCameraPreview: () => void,
- *    gl: ExpoWebGLRenderingContext
+ *    gl: ExpoWebGLRenderingContext,
+ *    cameraTexture: WebGLTexture
  *  ) => void — When the component is mounted and ready this callback will
  *  be called and recieve the following 3 elements:
  *    - __images__ is a (iterator)[https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Iterators_and_Generators]
@@ -95,8 +97,9 @@ const DEFAULT_RESIZE_DEPTH = 3;
  *      `updateCameraPreview` and any other operations you want to synchronize
  *      to the camera rendering you must call gl.endFrameExp() to display it
  *      on the screen. This is also provided in case you want to do other
- *      rendering using WebGL.
- *      Not needed when `autorender` is true.
+ *      rendering using WebGL. Not needed when `autorender` is true.
+ *    - __cameraTexture__ The underlying cameraTexture. This can be used to
+ *      implement your own __updateCameraPreview__.
  *
  * ```js
  * import { Camera } from 'expo-camera';
@@ -171,6 +174,7 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
     extends React.Component<T & Props, State> {
     camera: Camera;
     glView: GLView;
+    glContext: ExpoWebGLRenderingContext;
     rafID: number;
 
     constructor(props: T & Props) {
@@ -184,9 +188,13 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
     }
 
     componentWillUnmount() {
+      cancelAnimationFrame(this.rafID);
+      if(this.glContext) {
+        GLView.destroyContextAsync(this.glContext);
+      }
       this.camera = null;
       this.glView = null;
-      cancelAnimationFrame(this.rafID);
+      this.glContext = null;
     }
 
     /*
@@ -219,6 +227,7 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
      * @param gl
      */
     async onGLContextCreate(gl: ExpoWebGLRenderingContext) {
+      this.glContext = gl;
       const cameraTexture = await this.createCameraTexture();
       await detectGLCapabilities(gl);
 
@@ -250,6 +259,7 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
       //  Set up a generator function that yields tensors representing the
       // camera on demand.
       //
+      const cameraStreamView = this;
       function* nextFrameGenerator() {
         const RGBA_DEPTH = 4;
         const textureDims = {
@@ -264,7 +274,7 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
           depth: resizeDepth || DEFAULT_RESIZE_DEPTH,
         };
 
-        while (true) {
+        while (cameraStreamView.glContext != null) {
           const imageTensor = fromTexture(
             gl,
             cameraTexture,
@@ -277,7 +287,7 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
       const nextFrameIterator = nextFrameGenerator();
 
       // Pass the utility functions to the caller provided callback
-      this.props.onReady(nextFrameIterator, updatePreview, gl);
+      this.props.onReady(nextFrameIterator, updatePreview, gl, cameraTexture);
     }
 
     /**

tfjs-react-native/src/camera/camera_webgl_util.ts

@@ -29,11 +29,12 @@ interface Dimensions {
 }
 
 // Shared cached frameBuffer object from external context
-let fbo: WebGLFramebuffer;
+const fboCache = new WeakMap<WebGL2RenderingContext, WebGLFramebuffer>();
 
 // Internal target texture used for resizing camera texture input
-let resizeTexture: WebGLTexture;
-let resizeTextureDims: {width: number, height: number};
+const resizeTextureCache = new WeakMap<WebGL2RenderingContext, WebGLTexture>();
+const resizeTextureDimsCache =
+    new WeakMap<WebGL2RenderingContext, {width: number, height: number}>();
 
 interface ProgramObjects {
   program: WebGLProgram;
@@ -43,7 +44,9 @@ interface ProgramObjects {
 }
 
 // Cache for shader programs and associated vertex array buffers.
-const programCache: Map<string, ProgramObjects> = new Map();
+const programCacheByContext:
+    WeakMap<WebGL2RenderingContext, Map<string, ProgramObjects>> =
+        new WeakMap();
 
 /**
  * Download data from an texture.
@@ -58,9 +61,10 @@ export function downloadTextureData(
   const {width, height, depth} = dims;
   const pixels = new Uint8Array(width * height * depth);
 
-  if (fbo == null) {
-    fbo = createFrameBuffer(gl);
+  if (!fboCache.has(gl)) {
+    fboCache.set(gl, createFrameBuffer(gl));
   }
+  const fbo = fboCache.get(gl);
 
   const debugMode = getDebugMode();
 
@@ -197,9 +201,11 @@ export function runResizeProgram(
   //
   // Set up output texture.
   //
-  if (resizeTexture == null) {
-    resizeTexture = gl.createTexture();
+  if (!resizeTextureCache.has(gl)) {
+    resizeTextureCache.set(gl, gl.createTexture());
   }
+  const resizeTexture = resizeTextureCache.get(gl);
+
   const targetTexture = resizeTexture;
   const targetTextureWidth = outputDims.width;
   const targetTextureHeight = outputDims.height;
@@ -213,6 +219,11 @@ export function runResizeProgram(
   gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
 
   // Reallocate texture storage if target size has changed.
+  if (!resizeTextureDimsCache.has(gl)) {
+    resizeTextureDimsCache.set(gl, {width: -1, height: -1});
+  }
+  const resizeTextureDims = resizeTextureDimsCache.get(gl);
+
   if (resizeTextureDims == null ||
       resizeTextureDims.width !== targetTextureWidth ||
       resizeTextureDims.height !== targetTextureHeight) {
@@ -228,18 +239,17 @@ export function runResizeProgram(
           targetTextureHeight, border, format, type, null);
     });
 
-    resizeTextureDims = {
-      width: targetTextureWidth,
-      height: targetTextureHeight
-    };
+    resizeTextureDimsCache.set(
+        gl, {width: targetTextureWidth, height: targetTextureHeight});
   }
 
   //
   // Render to output texture
   //
-  if (fbo == null) {
-    fbo = createFrameBuffer(gl);
+  if (!fboCache.has(gl)) {
+    fboCache.set(gl, createFrameBuffer(gl));
   }
+  const fbo = fboCache.get(gl);
 
   gl.viewport(0, 0, targetTextureWidth, targetTextureHeight);
   gl.bindFramebuffer(gl.FRAMEBUFFER, fbo);
@@ -248,7 +258,6 @@ export function runResizeProgram(
 
   const fboComplete = gl.checkFramebufferStatus(gl.FRAMEBUFFER);
   if (fboComplete !== gl.FRAMEBUFFER_COMPLETE) {
-    console.log('checkFramebufferStatus is not complete', fboComplete);
     switch (fboComplete) {
       case gl.FRAMEBUFFER_INCOMPLETE_ATTACHMENT:
         throw new Error(
@@ -294,6 +303,11 @@ function createFrameBuffer(gl: WebGL2RenderingContext): WebGLFramebuffer {
 
 function drawTextureProgram(
     gl: WebGL2RenderingContext, flipHorizontal: boolean): ProgramObjects {
+  if (!programCacheByContext.has(gl)) {
+    programCacheByContext.set(gl, new Map());
+  }
+  const programCache = programCacheByContext.get(gl);
+
   const cacheKey = `drawTexture_${flipHorizontal}`;
   if (!programCache.has(cacheKey)) {
     const vertSource =
@@ -315,6 +329,11 @@ function resizeProgram(
     gl: WebGL2RenderingContext, sourceDims: Dimensions, targetDims: Dimensions,
     alignCorners: boolean,
     interpolation: 'nearest_neighbor'|'bilinear'): ProgramObjects {
+  if (!programCacheByContext.has(gl)) {
+    programCacheByContext.set(gl, new Map());
+  }
+  const programCache = programCacheByContext.get(gl);
+
   const cacheKey = `resize_${sourceDims.width}_${sourceDims.height}_${
       sourceDims.depth}_${targetDims.width}_${targetDims.height}_${
       targetDims.depth}_${alignCorners}_${interpolation}`;