Make web UI usable from touch devices (#1783)

Resolves #270.

Currently, the TinyPilot web UI is almost unusable from a touch device
(e.g., a tablet). We want to improve this situation, and based on our
discussion in the [exploratory proof-of-concept
branch](#1779) we want to do
this in multiple steps. The first step (this PR) includes:

- (a) Resolving the most major UI issues that basically prevent usage
from a touch device altogether. The main problems we identified are:
- The mobile OS pulls up the native keyboard as soon as the remote
screen receives focus.
- The mobile OS intercepts and handles touch actions on with its own
native logic
- (b) Interpreting single taps as single left clicks

(a) is mainly achieved by calling `preventDefault()` on the touch
events. For (b), we can introduce an adapter class that translates touch
events into synthetic mouse events. The idea is that we don’t clutter
the `<remote-screen>` component (which is already quite complex) with
more logic, but that we separate the touch logic as best as possible.

So as of this PR, you would be able to use the TinyPilot web UI on a
touch device, and issue single left clicks on the remote screen.

For testing this PR, it’s probably best to use a real touch device. As
an alternative, the browser dev tools offer a touch device emulation
(see [this Chrome
guide](https://developer.chrome.com/docs/devtools/device-mode) for
example), but that has some limitations.

If you don’t have a tablet at hand, it would suffice for me if you do a
code check. I’ve tested the changes using an iPad and a macOS computer
as target machine. It’s also hard to really break anything with this PR,
as the current touch experience is basically non-existent.
<a data-ca-tag
href="https://codeapprove.com/pr/tiny-pilot/tinypilot/1783"><img
src="https://codeapprove.com/external/github-tag-allbg.png" alt="Review
on CodeApprove" /></a>

---------

Co-authored-by: Jan Heuermann <[email protected]>

app/static/js/mouse.js

@@ -217,6 +217,7 @@ function normalizeWheelDelta(delta) {
  * @property {number} horizontalWheelDelta - A -1, 0, or 1 representing movement
  *     of the mouse's horizontal scroll wheel.
  *
+ * @param {MouseEvent} evt - https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent
  * @returns {MouseEventData}
  */
 function parseMouseEvent(evt) {

app/static/js/touch.js

@@ -0,0 +1,59 @@
+/**
+ * Adapter class that transforms touch events into synthetic mouse events.
+ *
+ * The idea behind having an adapter class like this is to stash away the touch
+ * handling logic as good as possible, and to keep the complexity away from the
+ * “regular” mouse handling code in the remote screen component.
+ *
+ * We currently only provide rudimentary support for touch devices. So for now,
+ * this adapter is only capable of emulating single left clicks.
+ */
+export class TouchToMouseAdapter {
+  _lastTouchPosition = { clientX: 0, clientY: 0 };
+
+  /**
+   * Synthetic mouse event that includes all properties that the
+   * `RateLimitedMouse.parseMouseEvent()` method relies on.
+   * @typedef {Object} SyntheticMouseEvent
+   */
+
+  /**
+   * @param {TouchEvent} evt - See:
+   *     - https://developer.mozilla.org/en-US/docs/Web/API/TouchEvent
+   *     - https://developer.mozilla.org/en-US/docs/Web/API/Element/touchstart_event
+   * @returns {SyntheticMouseEvent}
+   */
+  fromTouchStart(evt) {
+    // The corresponding `touchend` event won’t have the `touches` property
+    // set, so we need to preserve the latest one to be able to reconstruct the
+    // cursor position for the touch/mouse release.
+    const touch = evt.touches[0];
+    this._lastTouchPosition = {
+      clientX: touch.clientX,
+      clientY: touch.clientY,
+    };
+    return mouseClickEvent(evt.target, this._lastTouchPosition, 1);
+  }
+
+  /**
+   * @param {TouchEvent} evt - See:
+   *     - https://developer.mozilla.org/en-US/docs/Web/API/TouchEvent
+   *     - https://developer.mozilla.org/en-US/docs/Web/API/Element/touchend_event
+   *     - https://developer.mozilla.org/en-US/docs/Web/API/Element/touchcancel_event
+   * @returns {SyntheticMouseEvent}
+   */
+  fromTouchEndOrCancel(evt) {
+    return mouseClickEvent(evt.target, this._lastTouchPosition, 0);
+  }
+}
+
+function mouseClickEvent(target, touchPosition, buttons) {
+  return {
+    target,
+    buttons,
+    clientX: touchPosition.clientX,
+    clientY: touchPosition.clientY,
+    deltaX: 0,
+    deltaY: 0,
+  };
+}

app/templates/custom-elements/remote-screen.html

@@ -71,6 +71,7 @@
 
 <script type="module">
   import { RateLimitedMouse } from "/js/mouse.js";
+  import { TouchToMouseAdapter } from "/js/touch.js";
   import { VideoStreamingModeChangedEvent } from "/js/events.js";
 
   (function () {
@@ -201,6 +202,24 @@
             this.rateLimitedMouse.onWheel(evt);
           });
 
+          // Process touch activity and forward it as if it was mouse input.
+          const touchToMouseAdapter = new TouchToMouseAdapter();
+          screenElement.addEventListener("touchstart", (evt) => {
+            evt.preventDefault();
+            const mouseEvent = touchToMouseAdapter.fromTouchStart(evt);
+            this.rateLimitedMouse.onMouseDown(mouseEvent);
+          });
+          screenElement.addEventListener("touchend", (evt) => {
+            evt.preventDefault();
+            const mouseEvent = touchToMouseAdapter.fromTouchEndOrCancel(evt);
+            this.rateLimitedMouse.onMouseUp(mouseEvent);
+          });
+          screenElement.addEventListener("touchcancel", (evt) => {
+            evt.preventDefault();
+            const mouseEvent = touchToMouseAdapter.fromTouchEndOrCancel(evt);
+            this.rateLimitedMouse.onMouseUp(mouseEvent);
+          });
+
           // Ignore the context menu so that it doesn't block the screen when
           // the user right-clicks.
           screenElement.addEventListener("contextmenu", (evt) => {