Javascript__Image Display

The goal in mkturk is to deliver images or frames of a movie at the fastest speed and highest precision possible. Because of advances in display hardware (retina display) and software (javascript canvas), the speed and precision of image display in the browser has steadily improved. Here are a few notable improvements:

timing: window.requestAnimationFrame can display images locked to the next screen refresh thanks in part to improved timing precision from the performance.now() function
resolution: retina displays and in particular those on tablets and smartphones are pushing dpi to new levels from >200 on tablets up to 538 dpi on the newest google pixel phone. This allows greater detail in image content. Alternatively, this allows viewers to be positioned much closer to screen, reducing space requirements, while still delivering rich content.
speed & memory: improvements in the canvas feature in javascript to utilize offScreenCanvas can reduce memory load of images (offScreenCanvas renders images followed by no copy bitmap transfers for distribution to visible canvases) and/or increase speed by rendering on a separate thread from main (webgl running on a web worker),

Currently, two rendering modes are generally useful. The first is for drawing the pixel based content of images, the second is the rendering of 3D scenes using webgl graphics api.

If you are really interested in knowing more about the issues around image displays in javascript, this is a very good video overviewing canvas optimization - especially see last slide summary.

2D Image Context -- canvas.getContext('2d')

The drawImage command will render an image into a 2D canvas context = canvasobject.getContext('2d'). When showing a sequence of images, it would be nice to load the upcoming image offscreen and concurrently (parallel thread) while the current image displays. However, because of limitations on how web workers operate, they aren't compatible with 2D contexts. It is still possible to use an offScreenCanvas to feed a visible canvas. While the current frame is waiting to be displayed on the next refresh, the offScreenCanvas can prepare the upcoming frame (think double buffering). This transfer is done via a no copy operation of the bitmap, and the offscreenCanvas can feed multiple canvases, making this a memory efficient solution. The only drawback is that rendering, though happening offscreen, is still done using the main thread.

// CANVAS SETUP

// The drawing workspace is the total physical pixels and is the size of the desired bitmap from offscreenCanvas.
var visiblecanvas.width = window.innerWidth*window.devicePixelRatio
var visiblecanvas.height = window.innerHeight*window.devicePixelRatio

// The CSS size of the canvas is in logical pixels and is actual canvas coordinates (e.g. locations of user touches)
visiblecanvas.style.width = window.innerWidth + "px"
visiblecanvas.style.height = window.innerHeight + "px"

var destinationcontext = visiblecanvas.getContext("bitmaprenderer");

var offscreen = new OffscreenCanvas(visiblecanvas.width,visiblecanvas.height); //should be # of physical pixels 
var offscreenContext = offscreen.getContext('2d');
offscreen.commitTo = function(destinationcontext) {
    var bitmap = this.transferToImageBitmap();
    destionationcontext.transferFromImageBitmap(bitmap);
}
offscreen.renderFrame = function(args){
//drawing commands
};

// ANIMATION LOOP
offscreen.renderFrame(args); //prepare first frame
function displayImages(){
    window.requestAnimationFrame(updateCanvas)
    function updateCanvas(timestamp){
        offscreen.commitTo(destinationcontext)
        if still rendering {
            offscreen.renderFrame(args)
            window.requestAnimationFrame(updateCanvas)
        }
        else if done rendering{//done}
    }
}
displayImages() //kick off display loop

In our speed tests, the offscreen method versus direct rendering is achieving similar frame rates. If you can limit your frame operation to simple image draws with limited other computations, you should experience good display performance given the cpu/gpu capabilities of most modern devices.

Webgl Context -- canvas.getContext('webgl')

Webgl is meant for rendering 2D & 3D graphics. If you're using webgl, then more options open up as far as working off the main thread using webworkers (which currently do not work with 2D contexts). In main, you would do something like:

// SEND OFFSCREEN CANVAS TO WORKER
var offscreen = visiblecanvas.transferControlToOffscreen();
var worker = new Worker('offscreenworker.js');
worker.postMessage({cmd: 'start', canvas: offscreen, 
					innerWidth: window.innerWidth,
					innerHeight: window.innerHeight,
					devicePixelRatio: window.devicePixelRatio
			}, [offscreen]); // where the second argument are transferable (no copy) data
//when ready to draw
worker.postMessage({cmd: 'renderFrame',args: renderingargs});

//ANIMATION LOOP
function displayImages() {
	window.requestAnimationFrame(updateCanvas)
	var starttime = null
	function updateCanvas(timestamp){
		worker.postMessage({cmd: 'renderFrame',idx: canvas.currentframe});

		if still rendering {
			window.requestAnimationFrame(updateCanvasOrder)
		}
		else {//done}
	}
}

And in the worker, you would handle all drawing operations offscreen prior to committing to the visible parent canvas:

onmessage = function(e) {
	var data = e.data;
	switch (data.cmd) {
	case 'start':
		initCanvas(e.data.canvas,e.data.devicePixelRatio,
					e.data.innerWidth,e.data.innerHeight)
		self.postMessage('WORKER STARTED: ' + data.cmd);
		break;
	case 'render':
		renderFrame(i); //,images,canvasWidth,canvasHeight,canvasScale)
	case 'stop':
	  self.postMessage('WORKER STOPPED: ' + data.cmd +
					   '. (buttons will no longer work)');
	  self.close(); // Terminates the worker.
	  break;   
	default:
	  self.postMessage('Unknown command: ' + data.cmd);
	}
  postMessage(workerResult);
}
function initCanvas(args){
//canvas initialization, formatting
}

function renderFrame(i,canvasWidth,canvasHeight,canvasScale){
//rendering operations
	offscreencanvas.getContext('webgl').commit();
}

This code pattern would be useful for animations or rendering of parametrically generated stimuli. For using webgl with retina displays, you would follow the same canvas and backingStore set up as for 2D contexts. The logical screen size would be specificed in the canvas.style.width/height property. The physical pixel (retina display) workspace would be defined for the backingStore (canvas.width/height) and webgl context sizes.

DPI for tablets and smartphones

264 dpi - Samsung Galaxy Tab S2 9.7
281 dpi - Nexus 9
287 dpi - Samsung Galaxy S 10.5
308 dpi - Google Pixel-C
359 dpi - Samsung Galaxy S 8.4
458 dpi - Apple iPhone X
538 dpi - Google Pixel 2 XL

Unit tests: qr code, white pixel noise, bump on line for different number of pixels, vernier at different pixel spacings

Standard canvas updating without using offscreenCanvas API

mkturk initially worked by swapping the depth order between pre-rendered canvases to display images. Timing was still controlled by window.requestAnimationFrame.

On a macbook pro, requestAnimationFrame does not drop frames (i.e. updates every 16.7 ms on a 60Hz lcd). This works for basic rsvp type use where image on/off is 100ms so that can buffer a handful of images and canvases do not need to be swapped on every frame and do not need to be rendered on the fly as in a movie.

Note: A slowdown is observed when using chrome developer tools on tablet device (e.g. Samsung Galaxy Tab S 10.5). When usb debugging is not connected, tablet performance is faster.

Approaches for requestAnimationFrame: I've tried 3 approaches to improve display performance prior to the availability of offscreenCanvas:
1 - Swap current and new canvas using zIndex: Pre-render canvases. Bring the new canvas to foreground and move the current canvas back by setting CSS zIndex property.

2 - Render new canvas into destination using drawImage: Have a fixed foreground (destination) canvas. Use drawImage to render the new canvas into the destination canvas.

3 - Move new canvases into foreground only. In later cleanup, move to background: Only move new canvases to foreground. Once display sequence is done, move all canvases to background prior to next display issue.

Method 1 performed as well or better than Methods 2 & 3, so it is still the preferred method. Simply changing the canvas depth (3) or re-rendering into a fixed canvas (2), which involve less operations than (1), ultimately seem to require similar amount of graphics resources.

Pseudo-code for methods 1-3

1 - Swap current and new canvas using zIndex: Pre-render canvases. Bring the new canvas to foreground and move the current canvas back by setting CSS zIndex property.

// ========== 1 ===========  
// Promise: display trial images  
function displayTrial(sequence,tsequence){
	var resolveFunc
	var errFunc
p = new Promise(function(resolve,reject){
	resolveFunc = resolve;
	errFunc = reject;
}).then(function(){});  

	var start = null;
	function updateCanvas(timestamp){
		if (!start) start = timestamp;
		console.log((timestamp-start) + ' ms')
		if (timestamp - start > tsequence[frame.current]){

			// Move canvas in front
			var prev_canvasobj=document.getElementById("canvas"+canvas.front);
			var curr_canvasobj=document.getElementById("canvas"+sequence[frame.current]);
			prev_canvasobj.style.zIndex="0";

			curr_canvasobj.style.zIndex="12";
			canvas.front = sequence[frame.current];
		} // move to front
			
		frame.shown[frame.current]=1;
		frame.current++;
		console.log("******Frame" + frame.current + " t=" + (timestamp-start))
	}; // if show new frame

	// continue if not all frames shown
	if (frame.shown[frame.shown.length-1] != 1){
		window.requestAnimationFrame(updateCanvas);
	}
	else{
		resolveFunc(1);
	}

//requestAnimationFrame advantages: goes on next screen refresh and syncs to browsers refresh rate on separate clock (not js clock)
window.requestAnimationFrame(updateCanvas); // kick off async work
return p
} //displayTrial
// ========== 1 ===========

2 - Render new canvas into destination using drawImage: Have a fixed foreground (destination) canvas. Use drawImage to render the new canvas into the destination canvas.

// ========== 2 ===========  
// Promise: display trial images  
function displayTrial(sequence,tsequence){  
	var resolveFunc  
	var errFunc  
p = new Promise(function(resolve,reject){
	resolveFunc = resolve;
	errFunc = reject;  
}).then(function(){});  

	var start = null;
	function updateCanvas(timestamp){
	if (!start) start = timestamp;
	console.log((timestamp-start) + ' ms')
	if (timestamp - start > tsequence[frame.current]){

		var destinationCanvas=document.getElementById("canvas" + canvas.view)
		var destCtx = destinationCanvas.getContext('2d');

		if (sequence[frame.current] == canvas.sample || 
			sequence[frame.current] == canvas.test){
			destinationCanvas.width=windowWidth*canvasScale
			destinationCanvas.height=windowHeight*canvasScale
			destinationCanvas.style.width = windowWidth + "px";
			destinationCanvas.style.height = windowHeight + "px";
		}
		else {
			destinationCanvas.width=windowWidth
			destinationCanvas.height=windowHeight
		}
			destCtx.drawImage(curr_canvasobj,0,0);
			canvas.front = sequence[frame.current];
			
			frame.shown[frame.current]=1;
			console.log("******Frame" + frame.current + " t=" + (timestamp-start))
			frame.current++;
	}; // if show new frame
}

//requestAnimationFrame advantages: goes on next screen refresh and syncs to browsers refresh rate on separate clock (not js clock)
window.requestAnimationFrame(updateCanvas); // kick off async work
return p
} //displayTrial  
//=========== 2 =============

3 - Move new canvases into foreground only. In later cleanup, move to background: Only move new canvases to foreground. Once display sequence is done, move all canvases to background prior to next display issue.

// ========== 3 ===========  
function displayTrial(sequence,tsequence){  
	var resolveFunc  
	var errFunc  
	p = new Promise(function(resolve,reject){
	resolveFunc = resolve;
	errFunc = reject;  
}).then(moveCanvasesToBack());
  
	var start = null;
	function updateCanvas(timestamp){
		if (!start) start = timestamp;
		console.log((timestamp-start) + ' ms')
		if (timestamp - start > tsequence[frame.current]){
			//console.log('displaying frame' + currframe + ' ' + timestamp);

			// Move canvas in front
			var curr_canvasobj=document.getElementById("canvas"+sequence[frame.current])
			var zCanvas = 21 + frame.current
			curr_canvasobj.style.zIndex = zCanvas.toString();
			canvas.front = sequence[frame.current];
			
			frame.shown[frame.current]=1;
			frame.current++
			console.log("******Frame" + frame.current + " t=" + (timestamp-start))
		} // if show new frame

		// continue if not all frames shown
		if (frame.shown[frame.shown.length-1] != 1){
			window.requestAnimationFrame(updateCanvas);
		}
		else{
			resolveFunc(1)
		}
	}

//requestAnimationFrame advantages: goes on next screen refresh and syncs to browsers refresh rate on separate clock (not js clock)
window.requestAnimationFrame(updateCanvas); // kick off async work
return p
} //displayTrial
  
function moveCanvasesToBack(){
var resolveFunc
var errFunc  
p = new Promise(function(resolve,reject){
	resolveFunc = resolve;
	errFunc = reject;
}).then(function(){  
});
  
	//move all canvases except front canvas to default background position  
	function updateCanvasStack(timestamp){
		console.log('sending canvases to back')
		for (var i = 0; i <= canvas.blank; i++){
			if (i != canvas.front){
				var curr_canvasobj=document.getElementById("canvas"+i)
				curr_canvasobj.style.zIndex = i.toString()
			}
		}

		//move front canvas to default foreground position
		var curr_canvasobj=document.getElementById("canvas"+canvas.front)
		var zCanvas = 11 + canvas.front
		curr_canvasobj.style.zIndex = zCanvas.toString()
		resolveFunc(1)
	}  
window.requestAnimationFrame(updateCanvasStack); // kick off async work
return p  
} //moveCanvasesToBack  
// ========== 3 ===========

Provide feedback

Saved searches

Use saved searches to filter your results more quickly