compositing/
touch.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use embedder_traits::{TouchId, TouchSequenceId};
6use euclid::{Point2D, Scale, Vector2D};
7use log::{debug, error, warn};
8use rustc_hash::FxHashMap;
9use webrender_api::units::{DeviceIntPoint, DevicePixel, DevicePoint, LayoutVector2D};
10
11use self::TouchSequenceState::*;
12
13// TODO: All `_SCREEN_PX` units below are currently actually used as `DevicePixel`
14// without multiplying with the `hidpi_factor`. This should be fixed and the
15// constants adjusted accordingly.
16/// Minimum number of `DeviceIndependentPixel` to begin touch scrolling.
17const TOUCH_PAN_MIN_SCREEN_PX: f32 = 20.0;
18/// Factor by which the flinging velocity changes on each tick.
19const FLING_SCALING_FACTOR: f32 = 0.95;
20/// Minimum velocity required for transitioning to fling when panning ends.
21const FLING_MIN_SCREEN_PX: f32 = 3.0;
22/// Maximum velocity when flinging.
23const FLING_MAX_SCREEN_PX: f32 = 4000.0;
24
25pub struct TouchHandler {
26    pub current_sequence_id: TouchSequenceId,
27    // todo: VecDeque + modulo arithmetic would be more efficient.
28    touch_sequence_map: FxHashMap<TouchSequenceId, TouchSequenceInfo>,
29}
30
31/// Whether the default move action is allowed or not.
32#[derive(Debug, Eq, PartialEq)]
33pub enum TouchMoveAllowed {
34    /// The default move action is prevented by script
35    Prevented,
36    /// The default move action is allowed
37    Allowed,
38    /// The initial move handler result is still pending
39    Pending,
40}
41
42pub struct TouchSequenceInfo {
43    /// touch sequence state
44    pub(crate) state: TouchSequenceState,
45    /// touch sequence active touch points
46    active_touch_points: Vec<TouchPoint>,
47    /// The script thread is already processing a touchmove operation.
48    ///
49    /// We use this to skip sending the event to the script thread,
50    /// to prevent overloading script.
51    handling_touch_move: bool,
52    /// Do not perform a click action.
53    ///
54    /// This happens when
55    /// - We had a touch move larger than the minimum distance OR
56    /// - We had multiple active touchpoints OR
57    /// - `preventDefault()` was called in a touch_down or touch_up handler
58    pub prevent_click: bool,
59    /// Whether move is allowed, prevented or the result is still pending.
60    /// Once the first move has been processed by script, we can transition to
61    /// non-cancellable events, and directly perform the pan without waiting for script.
62    pub prevent_move: TouchMoveAllowed,
63    /// Move operation waiting to be processed in the touch sequence.
64    ///
65    /// This is only used while the first touch move is processed in script.
66    /// Todo: It would be nice to merge this into the TouchSequenceState, but
67    /// this requires some additional work to handle the merging of pending
68    /// touch move events. Presumably if we keep a history of previous touch points,
69    /// this would allow a better fling algorithm and easier merging of zoom events.
70    pending_touch_move_action: Option<TouchMoveAction>,
71}
72
73impl TouchSequenceInfo {
74    fn touch_count(&self) -> usize {
75        self.active_touch_points.len()
76    }
77
78    fn pinch_distance_and_center(&self) -> (f32, Point2D<f32, DevicePixel>) {
79        debug_assert_eq!(self.touch_count(), 2);
80        let p0 = self.active_touch_points[0].point;
81        let p1 = self.active_touch_points[1].point;
82        let center = p0.lerp(p1, 0.5);
83        let distance = (p0 - p1).length();
84
85        (distance, center)
86    }
87
88    fn update_pending_touch_move_action(&mut self, action: TouchMoveAction) {
89        debug_assert!(self.prevent_move == TouchMoveAllowed::Pending);
90
91        if let Some(pre_action) = self.pending_touch_move_action {
92            let combine_action = match (pre_action, action) {
93                (TouchMoveAction::NoAction, _) | (_, TouchMoveAction::NoAction) => action,
94                // Combine touch move action.
95                (TouchMoveAction::Scroll(delta, point), TouchMoveAction::Scroll(delta_new, _)) => {
96                    TouchMoveAction::Scroll(delta + delta_new, point)
97                },
98                (
99                    TouchMoveAction::Scroll(delta, _),
100                    TouchMoveAction::Zoom(magnification, scroll_delta),
101                ) |
102                (
103                    TouchMoveAction::Zoom(magnification, scroll_delta),
104                    TouchMoveAction::Scroll(delta, _),
105                ) => {
106                    // Todo: It's unclear what the best action would be. Should we keep both
107                    // scroll and zoom?
108                    TouchMoveAction::Zoom(magnification, delta + scroll_delta)
109                },
110                (
111                    TouchMoveAction::Zoom(magnification, scroll_delta),
112                    TouchMoveAction::Zoom(magnification_new, scroll_delta_new),
113                ) => TouchMoveAction::Zoom(
114                    magnification * magnification_new,
115                    scroll_delta + scroll_delta_new,
116                ),
117            };
118            self.pending_touch_move_action = Some(combine_action);
119        } else {
120            self.pending_touch_move_action = Some(action);
121        }
122    }
123
124    /// Returns true when all touch events of a sequence have been received.
125    /// This does not mean that all event handlers have finished yet.
126    fn is_finished(&self) -> bool {
127        matches!(
128            self.state,
129            Finished | Flinging { .. } | PendingFling { .. } | PendingClick(_)
130        )
131    }
132}
133
134/// An action that can be immediately performed in response to a touch move event
135/// without waiting for script.
136#[derive(Clone, Copy, Debug, PartialEq)]
137pub enum TouchMoveAction {
138    /// Scroll by the provided offset.
139    Scroll(Vector2D<f32, DevicePixel>, DevicePoint),
140    /// Zoom by a magnification factor and scroll by the provided offset.
141    Zoom(f32, Vector2D<f32, DevicePixel>),
142    /// Don't do anything.
143    NoAction,
144}
145
146#[derive(Clone, Copy, Debug)]
147pub struct TouchPoint {
148    pub id: TouchId,
149    pub point: Point2D<f32, DevicePixel>,
150}
151
152impl TouchPoint {
153    pub fn new(id: TouchId, point: Point2D<f32, DevicePixel>) -> Self {
154        TouchPoint { id, point }
155    }
156}
157
158/// The states of the touch input state machine.
159#[derive(Clone, Copy, Debug, PartialEq)]
160pub(crate) enum TouchSequenceState {
161    /// touch point is active but does not start moving
162    Touching,
163    /// A single touch point is active and has started panning.
164    Panning {
165        velocity: Vector2D<f32, DevicePixel>,
166    },
167    /// A two-finger pinch zoom gesture is active.
168    Pinching,
169    /// A multi-touch gesture is in progress.
170    MultiTouch,
171    // All states below here are reached after a touch-up, i.e. all events of the sequence
172    // have already been received.
173    /// The initial touch move handler has not finished processing yet, so we need to wait
174    /// for the result in order to transition to fling.
175    PendingFling {
176        velocity: Vector2D<f32, DevicePixel>,
177        cursor: DeviceIntPoint,
178    },
179    /// No active touch points, but there is still scrolling velocity
180    Flinging {
181        velocity: Vector2D<f32, DevicePixel>,
182        cursor: DeviceIntPoint,
183    },
184    /// The touch sequence is finished, but a click is still pending, waiting on script.
185    PendingClick(DevicePoint),
186    /// touch sequence finished.
187    Finished,
188}
189
190pub(crate) struct FlingAction {
191    pub delta: LayoutVector2D,
192    pub cursor: DeviceIntPoint,
193}
194
195impl TouchHandler {
196    pub fn new() -> Self {
197        let finished_info = TouchSequenceInfo {
198            state: TouchSequenceState::Finished,
199            active_touch_points: vec![],
200            handling_touch_move: false,
201            prevent_click: false,
202            prevent_move: TouchMoveAllowed::Pending,
203            pending_touch_move_action: None,
204        };
205        // We insert a simulated initial touch sequence, which is already finished,
206        // so that we always have one element in the map, which simplifies creating
207        // a new touch sequence on touch_down.
208        let mut touch_sequence_map = FxHashMap::default();
209        touch_sequence_map.insert(TouchSequenceId::new(), finished_info);
210        TouchHandler {
211            current_sequence_id: TouchSequenceId::new(),
212            touch_sequence_map,
213        }
214    }
215
216    pub(crate) fn set_handling_touch_move(&mut self, sequence_id: TouchSequenceId, flag: bool) {
217        if let Some(sequence) = self.touch_sequence_map.get_mut(&sequence_id) {
218            sequence.handling_touch_move = flag;
219        }
220    }
221
222    pub(crate) fn is_handling_touch_move(&self, sequence_id: TouchSequenceId) -> bool {
223        if let Some(sequence) = self.touch_sequence_map.get(&sequence_id) {
224            sequence.handling_touch_move
225        } else {
226            false
227        }
228    }
229
230    pub(crate) fn prevent_click(&mut self, sequence_id: TouchSequenceId) {
231        if let Some(sequence) = self.touch_sequence_map.get_mut(&sequence_id) {
232            sequence.prevent_click = true;
233        } else {
234            warn!("TouchSequenceInfo corresponding to the sequence number has been deleted.");
235        }
236    }
237
238    pub(crate) fn prevent_move(&mut self, sequence_id: TouchSequenceId) {
239        if let Some(sequence) = self.touch_sequence_map.get_mut(&sequence_id) {
240            sequence.prevent_move = TouchMoveAllowed::Prevented;
241        } else {
242            warn!("TouchSequenceInfo corresponding to the sequence number has been deleted.");
243        }
244    }
245
246    /// Returns true if default move actions are allowed, false if prevented or the result
247    /// is still pending.,
248    pub(crate) fn move_allowed(&mut self, sequence_id: TouchSequenceId) -> bool {
249        if let Some(sequence) = self.touch_sequence_map.get_mut(&sequence_id) {
250            sequence.prevent_move == TouchMoveAllowed::Allowed
251        } else {
252            true
253        }
254    }
255
256    pub(crate) fn pending_touch_move_action(
257        &mut self,
258        sequence_id: TouchSequenceId,
259    ) -> Option<TouchMoveAction> {
260        match self.touch_sequence_map.get(&sequence_id) {
261            Some(sequence) => sequence.pending_touch_move_action,
262            None => None,
263        }
264    }
265
266    pub(crate) fn remove_pending_touch_move_action(&mut self, sequence_id: TouchSequenceId) {
267        if let Some(sequence) = self.touch_sequence_map.get_mut(&sequence_id) {
268            sequence.pending_touch_move_action = None;
269        }
270    }
271
272    // try to remove touch sequence, if touch sequence end and not has pending action.
273    pub(crate) fn try_remove_touch_sequence(&mut self, sequence_id: TouchSequenceId) {
274        if let Some(sequence) = self.touch_sequence_map.get(&sequence_id) {
275            if sequence.pending_touch_move_action.is_none() && sequence.state == Finished {
276                self.touch_sequence_map.remove(&sequence_id);
277            }
278        }
279    }
280
281    pub(crate) fn remove_touch_sequence(&mut self, sequence_id: TouchSequenceId) {
282        let old = self.touch_sequence_map.remove(&sequence_id);
283        debug_assert!(old.is_some(), "Sequence already removed?");
284    }
285
286    pub fn get_current_touch_sequence_mut(&mut self) -> &mut TouchSequenceInfo {
287        self.touch_sequence_map
288            .get_mut(&self.current_sequence_id)
289            .expect("Current Touch sequence does not exist")
290    }
291
292    fn try_get_current_touch_sequence_mut(&mut self) -> Option<&mut TouchSequenceInfo> {
293        self.touch_sequence_map.get_mut(&self.current_sequence_id)
294    }
295
296    pub(crate) fn get_touch_sequence(&self, sequence_id: TouchSequenceId) -> &TouchSequenceInfo {
297        self.touch_sequence_map
298            .get(&sequence_id)
299            .expect("Touch sequence not found.")
300    }
301    pub(crate) fn get_touch_sequence_mut(
302        &mut self,
303        sequence_id: TouchSequenceId,
304    ) -> Option<&mut TouchSequenceInfo> {
305        self.touch_sequence_map.get_mut(&sequence_id)
306    }
307
308    pub fn on_touch_down(&mut self, id: TouchId, point: Point2D<f32, DevicePixel>) {
309        // if the current sequence ID does not exist in the map, then it was already handled
310        if !self
311            .touch_sequence_map
312            .contains_key(&self.current_sequence_id) ||
313            self.get_touch_sequence(self.current_sequence_id)
314                .is_finished()
315        {
316            self.current_sequence_id.next();
317            debug!("Entered new touch sequence: {:?}", self.current_sequence_id);
318            let active_touch_points = vec![TouchPoint::new(id, point)];
319            self.touch_sequence_map.insert(
320                self.current_sequence_id,
321                TouchSequenceInfo {
322                    state: Touching,
323                    active_touch_points,
324                    handling_touch_move: false,
325                    prevent_click: false,
326                    prevent_move: TouchMoveAllowed::Pending,
327                    pending_touch_move_action: None,
328                },
329            );
330        } else {
331            debug!("Touch down in sequence {:?}.", self.current_sequence_id);
332            let touch_sequence = self.get_current_touch_sequence_mut();
333            touch_sequence
334                .active_touch_points
335                .push(TouchPoint::new(id, point));
336            match touch_sequence.active_touch_points.len() {
337                2.. => {
338                    touch_sequence.state = MultiTouch;
339                },
340                0..2 => {
341                    unreachable!("Secondary touch_down event with less than 2 fingers active?");
342                },
343            }
344            // Multiple fingers prevent a click.
345            touch_sequence.prevent_click = true;
346        }
347    }
348
349    pub fn on_vsync(&mut self) -> Option<FlingAction> {
350        let touch_sequence = self.touch_sequence_map.get_mut(&self.current_sequence_id)?;
351
352        let Flinging { velocity, cursor } = &mut touch_sequence.state else {
353            return None;
354        };
355        if velocity.length().abs() < FLING_MIN_SCREEN_PX {
356            touch_sequence.state = Finished;
357            // If we were flinging previously, there could still be a touch_up event result
358            // coming in after we stopped flinging
359            self.try_remove_touch_sequence(self.current_sequence_id);
360            None
361        } else {
362            // TODO: Probably we should multiply with the current refresh rate (and divide on each frame)
363            // or save a timestamp to account for a potentially changing display refresh rate.
364            *velocity *= FLING_SCALING_FACTOR;
365            debug_assert!(velocity.length() <= FLING_MAX_SCREEN_PX);
366            Some(FlingAction {
367                delta: LayoutVector2D::new(velocity.x, velocity.y),
368                cursor: *cursor,
369            })
370        }
371    }
372
373    pub fn on_touch_move(
374        &mut self,
375        id: TouchId,
376        point: Point2D<f32, DevicePixel>,
377    ) -> TouchMoveAction {
378        // As `TouchHandler` is per `WebViewRenderer` which is per `WebView` we might get a Touch Sequence Move that
379        // started with a down on a different webview. As the touch_sequence id is only changed on touch_down this
380        // move event gets a touch id which is already cleaned up.
381        let Some(touch_sequence) = self.try_get_current_touch_sequence_mut() else {
382            return TouchMoveAction::NoAction;
383        };
384        let idx = match touch_sequence
385            .active_touch_points
386            .iter_mut()
387            .position(|t| t.id == id)
388        {
389            Some(i) => i,
390            None => {
391                error!("Got a touchmove event for a non-active touch point");
392                return TouchMoveAction::NoAction;
393            },
394        };
395        let old_point = touch_sequence.active_touch_points[idx].point;
396        let delta = point - old_point;
397
398        let action = match touch_sequence.touch_count() {
399            1 => {
400                if let Panning { ref mut velocity } = touch_sequence.state {
401                    // TODO: Probably we should track 1-3 more points and use a smarter algorithm
402                    *velocity += delta;
403                    *velocity /= 2.0;
404                    // update the touch point every time when panning.
405                    touch_sequence.active_touch_points[idx].point = point;
406
407                    // Scroll offsets are opposite to the direction of finger motion.
408                    TouchMoveAction::Scroll(-delta, point)
409                } else if delta.x.abs() > TOUCH_PAN_MIN_SCREEN_PX ||
410                    delta.y.abs() > TOUCH_PAN_MIN_SCREEN_PX
411                {
412                    touch_sequence.state = Panning {
413                        velocity: Vector2D::new(delta.x, delta.y),
414                    };
415                    // No clicks should be issued after we transitioned to move.
416                    touch_sequence.prevent_click = true;
417                    // update the touch point
418                    touch_sequence.active_touch_points[idx].point = point;
419
420                    // Scroll offsets are opposite to the direction of finger motion.
421                    TouchMoveAction::Scroll(-delta, point)
422                } else {
423                    // We don't update the touchpoint, so multiple small moves can
424                    // accumulate and merge into a larger move.
425                    TouchMoveAction::NoAction
426                }
427            },
428            2 => {
429                if touch_sequence.state == Pinching ||
430                    delta.x.abs() > TOUCH_PAN_MIN_SCREEN_PX ||
431                    delta.y.abs() > TOUCH_PAN_MIN_SCREEN_PX
432                {
433                    touch_sequence.state = Pinching;
434                    let (d0, c0) = touch_sequence.pinch_distance_and_center();
435                    // update the touch point with the enough distance or pinching.
436                    touch_sequence.active_touch_points[idx].point = point;
437                    let (d1, c1) = touch_sequence.pinch_distance_and_center();
438                    let magnification = d1 / d0;
439
440                    let scroll_delta = c1 - c0 * Scale::new(magnification);
441
442                    // Scroll offsets are opposite to the direction of finger motion.
443                    TouchMoveAction::Zoom(magnification, -scroll_delta)
444                } else {
445                    // We don't update the touchpoint, so multiple small moves can
446                    // accumulate and merge into a larger move.
447                    TouchMoveAction::NoAction
448                }
449            },
450            _ => {
451                touch_sequence.active_touch_points[idx].point = point;
452                touch_sequence.state = MultiTouch;
453                TouchMoveAction::NoAction
454            },
455        };
456        // If the touch action is not `NoAction` and the first move has not been processed,
457        //  set pending_touch_move_action.
458        if TouchMoveAction::NoAction != action &&
459            touch_sequence.prevent_move == TouchMoveAllowed::Pending
460        {
461            touch_sequence.update_pending_touch_move_action(action);
462        }
463
464        action
465    }
466
467    pub fn on_touch_up(&mut self, id: TouchId, point: Point2D<f32, DevicePixel>) {
468        let touch_sequence = self.get_current_touch_sequence_mut();
469        let old = match touch_sequence
470            .active_touch_points
471            .iter()
472            .position(|t| t.id == id)
473        {
474            Some(i) => Some(touch_sequence.active_touch_points.swap_remove(i).point),
475            None => {
476                warn!("Got a touch up event for a non-active touch point");
477                None
478            },
479        };
480        match touch_sequence.state {
481            Touching => {
482                if touch_sequence.prevent_click {
483                    touch_sequence.state = Finished;
484                } else {
485                    touch_sequence.state = PendingClick(point);
486                }
487            },
488            Panning { velocity } => {
489                if velocity.length().abs() >= FLING_MIN_SCREEN_PX {
490                    // TODO: point != old. Not sure which one is better to take as cursor for flinging.
491                    debug!(
492                        "Transitioning to Fling. Cursor is {point:?}. Old cursor was {old:?}. \
493                            Raw velocity is {velocity:?}."
494                    );
495                    debug_assert!((point.x as i64) < (i32::MAX as i64));
496                    debug_assert!((point.y as i64) < (i32::MAX as i64));
497                    let cursor = DeviceIntPoint::new(point.x as i32, point.y as i32);
498                    // Multiplying the initial velocity gives the fling a much more snappy feel
499                    // and serves well as a poor-mans acceleration algorithm.
500                    let velocity = (velocity * 2.0).with_max_length(FLING_MAX_SCREEN_PX);
501                    match touch_sequence.prevent_move {
502                        TouchMoveAllowed::Allowed => {
503                            touch_sequence.state = Flinging { velocity, cursor }
504                            // todo: return Touchaction here, or is it sufficient to just
505                            // wait for the next vsync?
506                        },
507                        TouchMoveAllowed::Pending => {
508                            touch_sequence.state = PendingFling { velocity, cursor }
509                        },
510                        TouchMoveAllowed::Prevented => touch_sequence.state = Finished,
511                    }
512                } else {
513                    touch_sequence.state = Finished;
514                }
515            },
516            Pinching => {
517                touch_sequence.state = Touching;
518            },
519            MultiTouch => {
520                // We stay in multi-touch mode once we entered it until all fingers are lifted.
521                if touch_sequence.active_touch_points.is_empty() {
522                    touch_sequence.state = Finished;
523                }
524            },
525            PendingFling { .. } | Flinging { .. } | PendingClick(_) | Finished => {
526                error!("Touch-up received, but touch handler already in post-touchup state.")
527            },
528        }
529        #[cfg(debug_assertions)]
530        if touch_sequence.active_touch_points.is_empty() {
531            debug_assert!(
532                touch_sequence.is_finished(),
533                "Did not transition to a finished state: {:?}",
534                touch_sequence.state
535            );
536        }
537        debug!(
538            "Touch up with remaining active touchpoints: {:?}, in sequence {:?}",
539            touch_sequence.active_touch_points.len(),
540            self.current_sequence_id
541        );
542    }
543
544    pub fn on_touch_cancel(&mut self, id: TouchId, _point: Point2D<f32, DevicePixel>) {
545        // A similar thing with touch move can happen here where the event is coming from a different webview.
546        let Some(touch_sequence) = self.try_get_current_touch_sequence_mut() else {
547            return;
548        };
549        match touch_sequence
550            .active_touch_points
551            .iter()
552            .position(|t| t.id == id)
553        {
554            Some(i) => {
555                touch_sequence.active_touch_points.swap_remove(i);
556            },
557            None => {
558                warn!("Got a touchcancel event for a non-active touch point");
559                return;
560            },
561        }
562        if touch_sequence.active_touch_points.is_empty() {
563            touch_sequence.state = Finished;
564        }
565    }
566}
compositing/touch.rs

compositing/
touch.rs
