script/dom/html/
htmlvideoelement.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::cell::Cell;
6use std::sync::Arc;
7
8use dom_struct::dom_struct;
9use euclid::default::Size2D;
10use html5ever::{LocalName, Prefix, local_name, ns};
11use js::rust::HandleObject;
12use layout_api::{HTMLMediaData, MediaMetadata};
13use net_traits::image_cache::{
14    ImageCache, ImageCacheResult, ImageLoadListener, ImageOrMetadataAvailable, ImageResponse,
15    PendingImageId, UsePlaceholder,
16};
17use net_traits::request::{CredentialsMode, Destination, RequestBuilder, RequestId};
18use net_traits::{
19    CoreResourceThread, FetchMetadata, FetchResponseListener, FetchResponseMsg, NetworkError,
20    ResourceFetchTiming, ResourceTimingType,
21};
22use pixels::{Snapshot, SnapshotAlphaMode, SnapshotPixelFormat};
23use servo_media::player::video::VideoFrame;
24use servo_url::ServoUrl;
25use style::attr::{AttrValue, LengthOrPercentageOrAuto};
26
27use crate::document_loader::{LoadBlocker, LoadType};
28use crate::dom::attr::Attr;
29use crate::dom::bindings::cell::DomRefCell;
30use crate::dom::bindings::codegen::Bindings::HTMLVideoElementBinding::HTMLVideoElementMethods;
31use crate::dom::bindings::inheritance::Castable;
32use crate::dom::bindings::refcounted::Trusted;
33use crate::dom::bindings::reflector::DomGlobal;
34use crate::dom::bindings::root::{DomRoot, LayoutDom};
35use crate::dom::bindings::str::DOMString;
36use crate::dom::csp::{GlobalCspReporting, Violation};
37use crate::dom::document::Document;
38use crate::dom::element::{AttributeMutation, Element, LayoutElementHelpers};
39use crate::dom::globalscope::GlobalScope;
40use crate::dom::html::htmlmediaelement::{HTMLMediaElement, NetworkState, ReadyState};
41use crate::dom::node::{Node, NodeTraits};
42use crate::dom::performanceresourcetiming::InitiatorType;
43use crate::dom::virtualmethods::VirtualMethods;
44use crate::fetch::FetchCanceller;
45use crate::network_listener::{self, PreInvoke, ResourceTimingListener};
46use crate::script_runtime::CanGc;
47
48#[dom_struct]
49pub(crate) struct HTMLVideoElement {
50    htmlmediaelement: HTMLMediaElement,
51    /// <https://html.spec.whatwg.org/multipage/#dom-video-videowidth>
52    video_width: Cell<Option<u32>>,
53    /// <https://html.spec.whatwg.org/multipage/#dom-video-videoheight>
54    video_height: Cell<Option<u32>>,
55    /// Incremented whenever tasks associated with this element are cancelled.
56    generation_id: Cell<u32>,
57    /// Load event blocker. Will block the load event while the poster frame
58    /// is being fetched.
59    load_blocker: DomRefCell<Option<LoadBlocker>>,
60    /// A copy of the last frame
61    #[ignore_malloc_size_of = "VideoFrame"]
62    #[no_trace]
63    last_frame: DomRefCell<Option<VideoFrame>>,
64    /// Indicates if it has already sent a resize event for a given size
65    sent_resize: Cell<Option<(u32, u32)>>,
66}
67
68impl HTMLVideoElement {
69    fn new_inherited(
70        local_name: LocalName,
71        prefix: Option<Prefix>,
72        document: &Document,
73    ) -> HTMLVideoElement {
74        HTMLVideoElement {
75            htmlmediaelement: HTMLMediaElement::new_inherited(local_name, prefix, document),
76            video_width: Cell::new(None),
77            video_height: Cell::new(None),
78            generation_id: Cell::new(0),
79            load_blocker: Default::default(),
80            last_frame: Default::default(),
81            sent_resize: Cell::new(None),
82        }
83    }
84
85    #[cfg_attr(crown, allow(crown::unrooted_must_root))]
86    pub(crate) fn new(
87        local_name: LocalName,
88        prefix: Option<Prefix>,
89        document: &Document,
90        proto: Option<HandleObject>,
91        can_gc: CanGc,
92    ) -> DomRoot<HTMLVideoElement> {
93        Node::reflect_node_with_proto(
94            Box::new(HTMLVideoElement::new_inherited(
95                local_name, prefix, document,
96            )),
97            document,
98            proto,
99            can_gc,
100        )
101    }
102
103    pub(crate) fn get_video_width(&self) -> Option<u32> {
104        self.video_width.get()
105    }
106
107    pub(crate) fn get_video_height(&self) -> Option<u32> {
108        self.video_height.get()
109    }
110
111    /// <https://html.spec.whatwg.org/multipage#event-media-resize>
112    pub(crate) fn resize(&self, width: Option<u32>, height: Option<u32>) -> Option<(u32, u32)> {
113        self.video_width.set(width);
114        self.video_height.set(height);
115
116        let width = width?;
117        let height = height?;
118        if self.sent_resize.get() == Some((width, height)) {
119            return None;
120        }
121
122        let sent_resize = if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
123            None
124        } else {
125            self.owner_global()
126                .task_manager()
127                .media_element_task_source()
128                .queue_simple_event(self.upcast(), atom!("resize"));
129            Some((width, height))
130        };
131
132        self.sent_resize.set(sent_resize);
133        sent_resize
134    }
135
136    /// Gets the copy of the video frame at the current playback position,
137    /// if that is available, or else (e.g. when the video is seeking or buffering)
138    /// its previous appearance, if any.
139    pub(crate) fn get_current_frame_data(&self) -> Option<Snapshot> {
140        let frame = self.htmlmediaelement.get_current_frame();
141        if frame.is_some() {
142            *self.last_frame.borrow_mut() = frame;
143        }
144
145        match self.last_frame.borrow().as_ref() {
146            Some(frame) => {
147                let size = Size2D::new(frame.get_width() as u32, frame.get_height() as u32);
148                if !frame.is_gl_texture() {
149                    let alpha_mode = SnapshotAlphaMode::Transparent {
150                        premultiplied: false,
151                    };
152
153                    Some(Snapshot::from_vec(
154                        size.cast(),
155                        SnapshotPixelFormat::BGRA,
156                        alpha_mode,
157                        frame.get_data().to_vec(),
158                    ))
159                } else {
160                    // XXX(victor): here we only have the GL texture ID.
161                    Some(Snapshot::cleared(size.cast()))
162                }
163            },
164            None => None,
165        }
166    }
167
168    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
169    fn update_poster_frame(&self, poster_url: Option<&str>, can_gc: CanGc) {
170        // Step 1. If there is an existing instance of this algorithm running
171        // for this video element, abort that instance of this algorithm without
172        // changing the poster frame.
173        self.generation_id.set(self.generation_id.get() + 1);
174
175        // Step 2. If the poster attribute's value is the empty string or
176        // if the attribute is absent, then there is no poster frame; return.
177        let Some(poster_url) = poster_url.filter(|poster_url| !poster_url.is_empty()) else {
178            self.htmlmediaelement.set_poster_frame(None);
179            return;
180        };
181
182        // Step 3. Let url be the result of encoding-parsing a URL given
183        // the poster attribute's value, relative to the element's node
184        // document.
185        // Step 4. If url is failure, then return. There is no poster frame.
186        let poster_url = match self.owner_document().encoding_parse_a_url(poster_url) {
187            Ok(url) => url,
188            Err(_) => {
189                self.htmlmediaelement.set_poster_frame(None);
190                return;
191            },
192        };
193
194        // We use the image cache for poster frames so we save as much
195        // network activity as possible.
196        let window = self.owner_window();
197        let image_cache = window.image_cache();
198        let cache_result = image_cache.get_cached_image_status(
199            poster_url.clone(),
200            window.origin().immutable().clone(),
201            None,
202            UsePlaceholder::No,
203        );
204
205        let id = match cache_result {
206            ImageCacheResult::Available(ImageOrMetadataAvailable::ImageAvailable {
207                image,
208                url,
209                ..
210            }) => {
211                self.process_image_response(ImageResponse::Loaded(image, url), can_gc);
212                return;
213            },
214            ImageCacheResult::Available(ImageOrMetadataAvailable::MetadataAvailable(_, id)) => id,
215            ImageCacheResult::ReadyForRequest(id) => {
216                self.do_fetch_poster_frame(poster_url, id, can_gc);
217                id
218            },
219            ImageCacheResult::LoadError => {
220                self.process_image_response(ImageResponse::None, can_gc);
221                return;
222            },
223            ImageCacheResult::Pending(id) => id,
224        };
225
226        let trusted_node = Trusted::new(self);
227        let generation = self.generation_id();
228        let sender = window.register_image_cache_listener(id, move |response| {
229            let element = trusted_node.root();
230
231            // Ignore any image response for a previous request that has been discarded.
232            if generation != element.generation_id() {
233                return;
234            }
235            element.process_image_response(response.response, CanGc::note());
236        });
237
238        image_cache.add_listener(ImageLoadListener::new(sender, window.pipeline_id(), id));
239    }
240
241    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
242    fn do_fetch_poster_frame(&self, poster_url: ServoUrl, id: PendingImageId, can_gc: CanGc) {
243        // Step 5. Let request be a new request whose URL is url, client is the element's node
244        // document's relevant settings object, destination is "image", initiator type is "video",
245        // credentials mode is "include", and whose use-URL-credentials flag is set.
246        let document = self.owner_document();
247        let request = RequestBuilder::new(
248            Some(document.webview_id()),
249            poster_url.clone(),
250            document.global().get_referrer(),
251        )
252        .destination(Destination::Image)
253        .credentials_mode(CredentialsMode::Include)
254        .use_url_credentials(true)
255        .origin(document.origin().immutable().clone())
256        .pipeline_id(Some(document.global().pipeline_id()))
257        .insecure_requests_policy(document.insecure_requests_policy())
258        .has_trustworthy_ancestor_origin(document.has_trustworthy_ancestor_origin())
259        .policy_container(document.policy_container().to_owned());
260
261        // Step 6. Fetch request. This must delay the load event of the element's node document.
262        // This delay must be independent from the ones created by HTMLMediaElement during
263        // its media load algorithm, otherwise a code like
264        // <video poster="poster.png"></video>
265        // (which triggers no media load algorithm unless a explicit call to .load() is done)
266        // will block the document's load event forever.
267        let blocker = &self.load_blocker;
268        LoadBlocker::terminate(blocker, can_gc);
269        *blocker.borrow_mut() = Some(LoadBlocker::new(
270            &self.owner_document(),
271            LoadType::Image(poster_url.clone()),
272        ));
273
274        let context = PosterFrameFetchContext::new(
275            self,
276            poster_url,
277            id,
278            request.id,
279            self.global().core_resource_thread(),
280        );
281        self.owner_document().fetch_background(request, context);
282    }
283
284    fn generation_id(&self) -> u32 {
285        self.generation_id.get()
286    }
287
288    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
289    fn process_image_response(&self, response: ImageResponse, can_gc: CanGc) {
290        // Step 7. If an image is thus obtained, the poster frame is that image.
291        // Otherwise, there is no poster frame.
292        match response {
293            ImageResponse::Loaded(image, url) => {
294                debug!("Loaded poster image for video element: {:?}", url);
295                match image.as_raster_image() {
296                    Some(image) => self.htmlmediaelement.set_poster_frame(Some(image)),
297                    None => warn!("Vector images are not yet supported in video poster"),
298                }
299                LoadBlocker::terminate(&self.load_blocker, can_gc);
300            },
301            ImageResponse::MetadataLoaded(..) => {},
302            // The image cache may have loaded a placeholder for an invalid poster url
303            ImageResponse::PlaceholderLoaded(..) | ImageResponse::None => {
304                self.htmlmediaelement.set_poster_frame(None);
305                // A failed load should unblock the document load.
306                LoadBlocker::terminate(&self.load_blocker, can_gc);
307            },
308        }
309    }
310
311    /// <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
312    pub(crate) fn is_usable(&self) -> bool {
313        !matches!(
314            self.htmlmediaelement.get_ready_state(),
315            ReadyState::HaveNothing | ReadyState::HaveMetadata
316        )
317    }
318
319    pub(crate) fn origin_is_clean(&self) -> bool {
320        self.htmlmediaelement.origin_is_clean()
321    }
322
323    pub(crate) fn is_network_state_empty(&self) -> bool {
324        self.htmlmediaelement.network_state() == NetworkState::Empty
325    }
326}
327
328impl HTMLVideoElementMethods<crate::DomTypeHolder> for HTMLVideoElement {
329    // https://html.spec.whatwg.org/multipage/#dom-video-videowidth
330    fn VideoWidth(&self) -> u32 {
331        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
332            return 0;
333        }
334        self.video_width.get().unwrap_or(0)
335    }
336
337    // https://html.spec.whatwg.org/multipage/#dom-video-videoheight
338    fn VideoHeight(&self) -> u32 {
339        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
340            return 0;
341        }
342        self.video_height.get().unwrap_or(0)
343    }
344
345    // https://html.spec.whatwg.org/multipage/#dom-video-poster
346    make_getter!(Poster, "poster");
347
348    // https://html.spec.whatwg.org/multipage/#dom-video-poster
349    make_setter!(SetPoster, "poster");
350
351    // For testing purposes only. This is not an event from
352    // https://html.spec.whatwg.org/multipage/#dom-video-poster
353    event_handler!(postershown, GetOnpostershown, SetOnpostershown);
354}
355
356impl VirtualMethods for HTMLVideoElement {
357    fn super_type(&self) -> Option<&dyn VirtualMethods> {
358        Some(self.upcast::<HTMLMediaElement>() as &dyn VirtualMethods)
359    }
360
361    fn attribute_mutated(&self, attr: &Attr, mutation: AttributeMutation, can_gc: CanGc) {
362        self.super_type()
363            .unwrap()
364            .attribute_mutated(attr, mutation, can_gc);
365
366        if attr.local_name() == &local_name!("poster") {
367            if let Some(new_value) = mutation.new_value(attr) {
368                self.update_poster_frame(Some(&new_value), CanGc::note())
369            } else {
370                self.update_poster_frame(None, CanGc::note())
371            }
372        };
373    }
374
375    fn parse_plain_attribute(&self, name: &LocalName, value: DOMString) -> AttrValue {
376        match name {
377            &local_name!("width") | &local_name!("height") => {
378                AttrValue::from_dimension(value.into())
379            },
380            _ => self
381                .super_type()
382                .unwrap()
383                .parse_plain_attribute(name, value),
384        }
385    }
386}
387
388struct PosterFrameFetchContext {
389    /// Reference to the script thread image cache.
390    image_cache: Arc<dyn ImageCache>,
391    /// The element that initiated the request.
392    elem: Trusted<HTMLVideoElement>,
393    /// The cache ID for this request.
394    id: PendingImageId,
395    /// True if this response is invalid and should be ignored.
396    cancelled: bool,
397    /// Timing data for this resource
398    resource_timing: ResourceFetchTiming,
399    /// Url for the resource
400    url: ServoUrl,
401    /// A [`FetchCanceller`] for this request.
402    fetch_canceller: FetchCanceller,
403}
404
405impl FetchResponseListener for PosterFrameFetchContext {
406    fn process_request_body(&mut self, _: RequestId) {}
407
408    fn process_request_eof(&mut self, _: RequestId) {
409        self.fetch_canceller.ignore()
410    }
411
412    fn process_response(
413        &mut self,
414        request_id: RequestId,
415        metadata: Result<FetchMetadata, NetworkError>,
416    ) {
417        self.image_cache.notify_pending_response(
418            self.id,
419            FetchResponseMsg::ProcessResponse(request_id, metadata.clone()),
420        );
421
422        let metadata = metadata.ok().map(|meta| match meta {
423            FetchMetadata::Unfiltered(m) => m,
424            FetchMetadata::Filtered { unsafe_, .. } => unsafe_,
425        });
426
427        let status_is_ok = metadata
428            .as_ref()
429            .map_or(true, |m| m.status.in_range(200..300));
430
431        if !status_is_ok {
432            self.cancelled = true;
433            self.fetch_canceller.cancel();
434        }
435    }
436
437    fn process_response_chunk(&mut self, request_id: RequestId, payload: Vec<u8>) {
438        if self.cancelled {
439            // An error was received previously, skip processing the payload.
440            return;
441        }
442
443        self.image_cache.notify_pending_response(
444            self.id,
445            FetchResponseMsg::ProcessResponseChunk(request_id, payload),
446        );
447    }
448
449    fn process_response_eof(
450        &mut self,
451        request_id: RequestId,
452        response: Result<ResourceFetchTiming, NetworkError>,
453    ) {
454        self.image_cache.notify_pending_response(
455            self.id,
456            FetchResponseMsg::ProcessResponseEOF(request_id, response),
457        );
458    }
459
460    fn resource_timing_mut(&mut self) -> &mut ResourceFetchTiming {
461        &mut self.resource_timing
462    }
463
464    fn resource_timing(&self) -> &ResourceFetchTiming {
465        &self.resource_timing
466    }
467
468    fn submit_resource_timing(&mut self) {
469        network_listener::submit_timing(self, CanGc::note())
470    }
471
472    fn process_csp_violations(&mut self, _request_id: RequestId, violations: Vec<Violation>) {
473        let global = &self.resource_timing_global();
474        global.report_csp_violations(violations, None, None);
475    }
476}
477
478impl ResourceTimingListener for PosterFrameFetchContext {
479    fn resource_timing_information(&self) -> (InitiatorType, ServoUrl) {
480        let initiator_type = InitiatorType::LocalName(
481            self.elem
482                .root()
483                .upcast::<Element>()
484                .local_name()
485                .to_string(),
486        );
487        (initiator_type, self.url.clone())
488    }
489
490    fn resource_timing_global(&self) -> DomRoot<GlobalScope> {
491        self.elem.root().owner_document().global()
492    }
493}
494
495impl PreInvoke for PosterFrameFetchContext {
496    fn should_invoke(&self) -> bool {
497        true
498    }
499}
500
501impl PosterFrameFetchContext {
502    fn new(
503        elem: &HTMLVideoElement,
504        url: ServoUrl,
505        id: PendingImageId,
506        request_id: RequestId,
507        core_resource_thread: CoreResourceThread,
508    ) -> PosterFrameFetchContext {
509        let window = elem.owner_window();
510        PosterFrameFetchContext {
511            image_cache: window.image_cache(),
512            elem: Trusted::new(elem),
513            id,
514            cancelled: false,
515            resource_timing: ResourceFetchTiming::new(ResourceTimingType::Resource),
516            url,
517            fetch_canceller: FetchCanceller::new(request_id, core_resource_thread),
518        }
519    }
520}
521
522pub(crate) trait LayoutHTMLVideoElementHelpers {
523    fn data(self) -> HTMLMediaData;
524    fn get_width(self) -> LengthOrPercentageOrAuto;
525    fn get_height(self) -> LengthOrPercentageOrAuto;
526}
527
528impl LayoutDom<'_, HTMLVideoElement> {
529    fn width_attr(self) -> Option<LengthOrPercentageOrAuto> {
530        self.upcast::<Element>()
531            .get_attr_for_layout(&ns!(), &local_name!("width"))
532            .map(AttrValue::as_dimension)
533            .cloned()
534    }
535
536    fn height_attr(self) -> Option<LengthOrPercentageOrAuto> {
537        self.upcast::<Element>()
538            .get_attr_for_layout(&ns!(), &local_name!("height"))
539            .map(AttrValue::as_dimension)
540            .cloned()
541    }
542}
543
544impl LayoutHTMLVideoElementHelpers for LayoutDom<'_, HTMLVideoElement> {
545    fn data(self) -> HTMLMediaData {
546        let video = self.unsafe_get();
547
548        // Get the current frame being rendered.
549        let current_frame = video.htmlmediaelement.get_current_frame_to_present();
550
551        // This value represents the natural width and height of the video.
552        // It may exist even if there is no current frame (for example, after the
553        // metadata of the video is loaded).
554        let metadata = video
555            .get_video_width()
556            .zip(video.get_video_height())
557            .map(|(width, height)| MediaMetadata { width, height });
558
559        HTMLMediaData {
560            current_frame,
561            metadata,
562        }
563    }
564
565    fn get_width(self) -> LengthOrPercentageOrAuto {
566        self.width_attr().unwrap_or(LengthOrPercentageOrAuto::Auto)
567    }
568
569    fn get_height(self) -> LengthOrPercentageOrAuto {
570        self.height_attr().unwrap_or(LengthOrPercentageOrAuto::Auto)
571    }
572}