From ae5f65e19d2e834086f690f6d7bf4464e6f44d85 Mon Sep 17 00:00:00 2001 From: jotaen4tinypilot <83721279+jotaen4tinypilot@users.noreply.github.com> Date: Mon, 15 Apr 2024 17:17:32 +0200 Subject: [PATCH] Make web UI usable from touch devices (#1783) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves https://github.com/tiny-pilot/tinypilot/issues/270. Currently, the TinyPilot web UI is almost unusable from a touch device (e.g., a tablet). We want to improve this situation, and based on our discussion in the [exploratory proof-of-concept branch](https://github.com/tiny-pilot/tinypilot/pull/1779) we want to do this in multiple steps. The first step (this PR) includes: - (a) Resolving the most major UI issues that basically prevent usage from a touch device altogether. The main problems we identified are: - The mobile OS pulls up the native keyboard as soon as the remote screen receives focus. - The mobile OS intercepts and handles touch actions on with its own native logic - (b) Interpreting single taps as single left clicks (a) is mainly achieved by calling `preventDefault()` on the touch events. For (b), we can introduce an adapter class that translates touch events into synthetic mouse events. The idea is that we don’t clutter the `` component (which is already quite complex) with more logic, but that we separate the touch logic as best as possible. So as of this PR, you would be able to use the TinyPilot web UI on a touch device, and issue single left clicks on the remote screen. For testing this PR, it’s probably best to use a real touch device. As an alternative, the browser dev tools offer a touch device emulation (see [this Chrome guide](https://developer.chrome.com/docs/devtools/device-mode) for example), but that has some limitations. If you don’t have a tablet at hand, it would suffice for me if you do a code check. I’ve tested the changes using an iPad and a macOS computer as target machine. It’s also hard to really break anything with this PR, as the current touch experience is basically non-existent. Review on CodeApprove --------- Co-authored-by: Jan Heuermann --- app/static/js/mouse.js | 1 + app/static/js/touch.js | 59 +++++++++++++++++++ .../custom-elements/remote-screen.html | 19 ++++++ 3 files changed, 79 insertions(+) create mode 100644 app/static/js/touch.js diff --git a/app/static/js/mouse.js b/app/static/js/mouse.js index 5c2f674d7..bd37c0516 100644 --- a/app/static/js/mouse.js +++ b/app/static/js/mouse.js @@ -217,6 +217,7 @@ function normalizeWheelDelta(delta) { * @property {number} horizontalWheelDelta - A -1, 0, or 1 representing movement * of the mouse's horizontal scroll wheel. * + * @param {MouseEvent} evt - https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent * @returns {MouseEventData} */ function parseMouseEvent(evt) { diff --git a/app/static/js/touch.js b/app/static/js/touch.js new file mode 100644 index 000000000..a495617bb --- /dev/null +++ b/app/static/js/touch.js @@ -0,0 +1,59 @@ +/** + * Adapter class that transforms touch events into synthetic mouse events. + * + * The idea behind having an adapter class like this is to stash away the touch + * handling logic as good as possible, and to keep the complexity away from the + * “regular” mouse handling code in the remote screen component. + * + * We currently only provide rudimentary support for touch devices. So for now, + * this adapter is only capable of emulating single left clicks. + */ +export class TouchToMouseAdapter { + _lastTouchPosition = { clientX: 0, clientY: 0 }; + + /** + * Synthetic mouse event that includes all properties that the + * `RateLimitedMouse.parseMouseEvent()` method relies on. + * @typedef {Object} SyntheticMouseEvent + */ + + /** + * @param {TouchEvent} evt - See: + * - https://developer.mozilla.org/en-US/docs/Web/API/TouchEvent + * - https://developer.mozilla.org/en-US/docs/Web/API/Element/touchstart_event + * @returns {SyntheticMouseEvent} + */ + fromTouchStart(evt) { + // The corresponding `touchend` event won’t have the `touches` property + // set, so we need to preserve the latest one to be able to reconstruct the + // cursor position for the touch/mouse release. + const touch = evt.touches[0]; + this._lastTouchPosition = { + clientX: touch.clientX, + clientY: touch.clientY, + }; + return mouseClickEvent(evt.target, this._lastTouchPosition, 1); + } + + /** + * @param {TouchEvent} evt - See: + * - https://developer.mozilla.org/en-US/docs/Web/API/TouchEvent + * - https://developer.mozilla.org/en-US/docs/Web/API/Element/touchend_event + * - https://developer.mozilla.org/en-US/docs/Web/API/Element/touchcancel_event + * @returns {SyntheticMouseEvent} + */ + fromTouchEndOrCancel(evt) { + return mouseClickEvent(evt.target, this._lastTouchPosition, 0); + } +} + +function mouseClickEvent(target, touchPosition, buttons) { + return { + target, + buttons, + clientX: touchPosition.clientX, + clientY: touchPosition.clientY, + deltaX: 0, + deltaY: 0, + }; +} diff --git a/app/templates/custom-elements/remote-screen.html b/app/templates/custom-elements/remote-screen.html index cfacc98ed..c6f64fbef 100644 --- a/app/templates/custom-elements/remote-screen.html +++ b/app/templates/custom-elements/remote-screen.html @@ -71,6 +71,7 @@