CocoaSpice: moved screen translation to shaders

Previously, we were using viewports to handle translation and scaling. This
led to issues where if the VM display is larger than the device's display
then the screen would be cut off. We move the affine transformation to part
of the vertex shader now.

Fixes #181

Author: osy

Renderer/UTMRenderer.m

@@ -102,6 +102,39 @@ - (void)mtkView:(nonnull MTKView *)view drawableSizeWillChange:(CGSize)size
     _viewportSize.y = size.height;
 }
 
+/// Create a translation+scale matrix
+static matrix_float4x4 matrix_scale_translate(CGFloat scale, CGPoint translate)
+{
+    matrix_float4x4 m = {
+        .columns[0] = {
+            scale,
+            0,
+            0,
+            0
+        },
+        .columns[1] = {
+            0,
+            scale,
+            0,
+            0
+        },
+        .columns[2] = {
+            0,
+            0,
+            1,
+            0
+        },
+        .columns[3] = {
+            translate.x,
+            -translate.y, // y flipped
+            0,
+            1
+        }
+        
+    };
+    return m;
+}
+
 /// Called whenever the view needs to render a frame
 - (void)drawInMTKView:(nonnull MTKView *)view
 {
@@ -133,18 +166,9 @@ - (void)drawInMTKView:(nonnull MTKView *)view
             dispatch_semaphore_wait(screenLock, DISPATCH_TIME_FOREVER);
 
             // Render the screen first
-            // Set the region of the drawable to which we'll draw.
-            CGSize scaled = CGSizeMake(_viewportSize.x * self.sourceScreen.viewportScale,
-                                       _viewportSize.y * self.sourceScreen.viewportScale);
-            MTLViewport viewport = {
-                self.sourceScreen.viewportOrigin.x + -scaled.width /2 + _viewportSize.x/2,
-                self.sourceScreen.viewportOrigin.y + -scaled.height/2 + _viewportSize.y/2,
-                scaled.width,
-                scaled.height,
-                -1.0,
-                1.0
-            };
-            [renderEncoder setViewport:viewport];
+            
+            matrix_float4x4 transform = matrix_scale_translate(self.sourceScreen.viewportScale,
+                                                               self.sourceScreen.viewportOrigin);
 
             [renderEncoder setRenderPipelineState:_pipelineState];
 
@@ -156,6 +180,10 @@ - (void)drawInMTKView:(nonnull MTKView *)view
                                    length:sizeof(_viewportSize)
                                   atIndex:UTMVertexInputIndexViewportSize];
 
+            [renderEncoder setVertexBytes:&transform
+                                   length:sizeof(transform)
+                                  atIndex:UTMVertexInputIndexTransform];
+
             [renderEncoder setVertexBytes:&hasAlpha
                                    length:sizeof(hasAlpha)
                                   atIndex:UTMVertexInputIndexHasAlpha];
@@ -179,23 +207,20 @@ - (void)drawInMTKView:(nonnull MTKView *)view
             dispatch_semaphore_wait(cursorLock, DISPATCH_TIME_FOREVER);
 
             // Next render the cursor
-            CGSize scaled = CGSizeMake(_viewportSize.x * self.sourceCursor.viewportScale,
-                                       _viewportSize.y * self.sourceCursor.viewportScale);
-            MTLViewport cursorViewport = {
-                self.sourceCursor.viewportOrigin.x + self.sourceScreen.viewportOrigin.x + -scaled.width /2 + _viewportSize.x/2,
-                self.sourceCursor.viewportOrigin.y + self.sourceScreen.viewportOrigin.y + -scaled.height/2 + _viewportSize.y/2,
-                scaled.width,
-                scaled.height,
-                -1.0,
-                1.0
-            };
-            [renderEncoder setViewport:cursorViewport];
+            matrix_float4x4 transform = matrix_scale_translate(self.sourceScreen.viewportScale,
+                                                               CGPointMake(self.sourceScreen.viewportOrigin.x +
+                                                                           self.sourceCursor.viewportOrigin.x,
+                                                                           self.sourceScreen.viewportOrigin.y +
+                                                                           self.sourceCursor.viewportOrigin.y));
             [renderEncoder setVertexBuffer:self.sourceCursor.vertices
                                     offset:0
                                   atIndex:UTMVertexInputIndexVertices];
             [renderEncoder setVertexBytes:&_viewportSize
                                    length:sizeof(_viewportSize)
                                   atIndex:UTMVertexInputIndexViewportSize];
+            [renderEncoder setVertexBytes:&transform
+                                 length:sizeof(transform)
+                                atIndex:UTMVertexInputIndexTransform];
             [renderEncoder setVertexBytes:&hasAlpha
                                  length:sizeof(hasAlpha)
                                 atIndex:UTMVertexInputIndexHasAlpha];

Renderer/UTMShaderTypes.h

@@ -16,7 +16,8 @@ typedef enum UTMVertexInputIndex
 {
     UTMVertexInputIndexVertices     = 0,
     UTMVertexInputIndexViewportSize = 1,
-    UTMVertexInputIndexHasAlpha     = 2,
+    UTMVertexInputIndexTransform    = 2,
+    UTMVertexInputIndexHasAlpha     = 3,
 } UTMVertexInputIndex;
 
 // Texture index values shared between shader and C code to ensure Metal shader buffer inputs match

Renderer/UTMShaders.metal

@@ -35,16 +35,20 @@ vertex RasterizerData
 vertexShader(uint vertexID [[ vertex_id ]],
              constant UTMVertex *vertexArray [[ buffer(UTMVertexInputIndexVertices) ]],
              constant vector_uint2 *viewportSizePointer  [[ buffer(UTMVertexInputIndexViewportSize) ]],
+             constant matrix_float4x4 &transformation [[ buffer(UTMVertexInputIndexTransform) ]],
              constant bool *hasAlpha [[ buffer(UTMVertexInputIndexHasAlpha) ]])
 
 {
 
     RasterizerData out;
+    
+    // Transform the vertex
+    vector_float4 position = transformation * float4(vertexArray[vertexID].position,0,1);
 
     // Index into our array of positions to get the current vertex
     //   Our positions are specified in pixel dimensions (i.e. a value of 100 is 100 pixels from
     //   the origin)
-    float2 pixelSpacePosition = vertexArray[vertexID].position.xy;
+    float2 pixelSpacePosition = position.xy;
 
     // Get the size of the drawable so that we can convert to normalized device coordinates,
     float2 viewportSize = float2(*viewportSizePointer);