script/dom/html/embedded_content/
htmlvideoelement.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::cell::Cell;
6use std::rc::Rc;
7use std::sync::Arc;
8
9use atomic_refcell::AtomicRefCell;
10use bytes::Bytes;
11use dom_struct::dom_struct;
12use euclid::default::Size2D;
13use html5ever::{LocalName, Prefix, local_name, ns};
14use js::context::{JSContext, NoGC};
15use js::rust::HandleObject;
16use layout_api::{HTMLMediaData, MediaMetadata};
17use net_traits::blob_url_store::UrlWithBlobClaim;
18use net_traits::fetch::fetch_controller::FetchController;
19use net_traits::image_cache::{
20    ImageCache, ImageCacheResult, ImageLoadListener, ImageOrMetadataAvailable, ImageResponse,
21    PendingImageId,
22};
23use net_traits::request::{CredentialsMode, Destination, RequestBuilder, RequestId};
24use net_traits::{
25    CoreResourceThread, FetchMetadata, FetchResponseMsg, Metadata, NetworkError,
26    ResourceFetchTiming,
27};
28use pixels::{Snapshot, SnapshotAlphaMode, SnapshotPixelFormat};
29use script_bindings::cell::DomRefCell;
30use script_bindings::error::Fallible;
31use servo_media::player::video::VideoFrame;
32use servo_url::ServoUrl;
33use style::attr::{AttrValue, LengthOrPercentageOrAuto};
34
35use crate::dom::bindings::cell::AtomicSafeBorrowMut;
36use crate::dom::bindings::codegen::Bindings::HTMLVideoElementBinding::HTMLVideoElementMethods;
37use crate::dom::bindings::inheritance::Castable;
38use crate::dom::bindings::refcounted::Trusted;
39use crate::dom::bindings::reflector::DomGlobal;
40use crate::dom::bindings::root::{DomRoot, LayoutDom};
41use crate::dom::bindings::str::{DOMString, USVString};
42use crate::dom::csp::{GlobalCspReporting, Violation};
43use crate::dom::document::Document;
44use crate::dom::element::attributes::storage::AttrRef;
45use crate::dom::element::{AttributeMutation, Element};
46use crate::dom::globalscope::GlobalScope;
47use crate::dom::html::htmlmediaelement::{HTMLMediaElement, NetworkState, ReadyState};
48use crate::dom::node::virtualmethods::VirtualMethods;
49use crate::dom::node::{Node, NodeDamage, NodeTraits};
50use crate::dom::performance::performanceresourcetiming::InitiatorType;
51use crate::event_loop::document_loader::{LoadBlocker, LoadType};
52use crate::fetch::fetch::RequestWithGlobalScope;
53use crate::fetch::network_listener::{self, FetchResponseListener, ResourceTimingListener};
54use crate::url::ensure_blob_referenced_by_url_is_kept_alive;
55
56#[dom_struct]
57pub(crate) struct HTMLVideoElement {
58    htmlmediaelement: HTMLMediaElement,
59    /// <https://html.spec.whatwg.org/multipage/#dom-video-videowidth>
60    video_width: Cell<Option<u32>>,
61    /// <https://html.spec.whatwg.org/multipage/#dom-video-videoheight>
62    video_height: Cell<Option<u32>>,
63    /// <https://html.spec.whatwg.org/multipage/#dom-video-poster>
64    #[no_trace]
65    poster_url: AtomicRefCell<Option<UrlWithBlobClaim>>,
66    /// Incremented whenever tasks associated with this element are cancelled.
67    generation_id: Cell<u32>,
68    /// Load event blocker. Will block the load event while the poster frame
69    /// is being fetched.
70    load_blocker: DomRefCell<Option<LoadBlocker>>,
71    /// A copy of the last frame
72    #[no_trace]
73    last_frame: DomRefCell<Option<VideoFrame>>,
74    #[cfg(feature = "webgpu")]
75    #[conditional_malloc_size_of]
76    /// Planar texture for WebGPU
77    planar_texture: DomRefCell<Option<Rc<crate::dom::gpuexternaltexture::PlanarTexture>>>,
78}
79
80impl HTMLVideoElement {
81    fn new_inherited(
82        local_name: LocalName,
83        prefix: Option<Prefix>,
84        document: &Document,
85    ) -> HTMLVideoElement {
86        HTMLVideoElement {
87            htmlmediaelement: HTMLMediaElement::new_inherited(local_name, prefix, document),
88            video_width: Cell::new(None),
89            video_height: Cell::new(None),
90            poster_url: Default::default(),
91            generation_id: Cell::new(0),
92            load_blocker: Default::default(),
93            last_frame: Default::default(),
94            #[cfg(feature = "webgpu")]
95            planar_texture: DomRefCell::new(None),
96        }
97    }
98
99    pub(crate) fn new(
100        cx: &mut JSContext,
101        local_name: LocalName,
102        prefix: Option<Prefix>,
103        document: &Document,
104        proto: Option<HandleObject>,
105    ) -> DomRoot<HTMLVideoElement> {
106        Node::reflect_weak_referenceable_node_with_proto(
107            cx,
108            Rc::new(HTMLVideoElement::new_inherited(
109                local_name, prefix, document,
110            )),
111            document,
112            proto,
113        )
114    }
115
116    pub(crate) fn get_video_width(&self) -> Option<u32> {
117        self.video_width.get()
118    }
119
120    pub(crate) fn get_video_height(&self) -> Option<u32> {
121        self.video_height.get()
122    }
123
124    pub(crate) fn set_natural_dimensions(
125        &self,
126        no_gc: &NoGC,
127        width: Option<u32>,
128        height: Option<u32>,
129    ) -> bool {
130        if self.video_width.get() == width && self.video_height.get() == height {
131            return false;
132        }
133
134        self.video_width.set(width);
135        self.video_height.set(height);
136
137        self.upcast::<Node>().dirty(no_gc, NodeDamage::Other);
138        true
139    }
140
141    /// Gets the copy of the video frame at the current playback position,
142    /// if that is available, or else (e.g. when the video is seeking or buffering)
143    /// its previous appearance, if any.
144    pub(crate) fn get_current_frame_data(&self) -> Option<Snapshot> {
145        let frame = self.htmlmediaelement.get_current_frame();
146        if frame.is_some() {
147            *self.last_frame.borrow_mut() = frame;
148        }
149
150        match self.last_frame.borrow().as_ref() {
151            Some(frame) => {
152                let size = Size2D::new(frame.get_width() as u32, frame.get_height() as u32);
153                if !frame.is_gl_texture() {
154                    let alpha_mode = SnapshotAlphaMode::Transparent {
155                        premultiplied: false,
156                    };
157
158                    Some(Snapshot::from_vec(
159                        size.cast(),
160                        SnapshotPixelFormat::BGRA,
161                        alpha_mode,
162                        frame.get_data().to_vec(),
163                    ))
164                } else {
165                    // XXX(victor): here we only have the GL texture ID.
166                    Some(Snapshot::cleared(size.cast()))
167                }
168            },
169            None => None,
170        }
171    }
172
173    #[cfg(feature = "webgpu")]
174    pub(crate) fn planar_video_for_webgpu(
175        &self,
176        device: &crate::dom::types::GPUDevice,
177    ) -> Fallible<(
178        Size2D<u32>,
179        Option<Rc<crate::dom::gpuexternaltexture::PlanarTexture>>,
180    )> {
181        use crate::dom::gpuexternaltexture::PlanarTexture;
182        // 1. If source is not origin-clean, throw a SecurityError and return.
183        if !self.origin_is_clean() {
184            return Err(script_bindings::error::Error::Security(Some(
185                "Source is not origin-clean".to_string(),
186            )));
187        }
188        // 2. Let usability be ? check the usability of the image argument(source).
189        if !self.is_usable() {
190            // 3. If usability is not good:
191            // Generate a validation error.
192            // Return an invalidated GPUExternalTexture.
193            Ok((Size2D::zero(), None))
194        } else {
195            // 4. Let data be the result of converting the current image contents of source into the color space descriptor.colorSpace with unpremultiplied alpha.
196            let mut planar_texture = self.planar_texture.borrow_mut();
197            match planar_texture.as_ref() {
198                Some(planar_texture) => {
199                    if planar_texture.is_expired() &&
200                        let Some(snapshot) = self.get_current_frame_data()
201                    {
202                        planar_texture.update(snapshot);
203                    }
204                },
205                None => {
206                    *planar_texture = self.get_current_frame_data().map(|snapshot| {
207                        Rc::new(PlanarTexture::new(device.channel(), device, snapshot))
208                    });
209                },
210            };
211            Ok((
212                planar_texture
213                    .as_ref()
214                    .map(|pt| pt.size())
215                    .unwrap_or_default(),
216                planar_texture.as_ref().cloned(),
217            ))
218        }
219    }
220
221    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
222    fn update_poster_frame(&self, cx: &mut JSContext, poster_attribute: Option<&str>) {
223        // Step 1. If there is an existing instance of this algorithm running
224        // for this video element, abort that instance of this algorithm without
225        // changing the poster frame.
226        self.generation_id.set(self.generation_id.get() + 1);
227
228        // Step 2. If the poster attribute's value is the empty string or
229        // if the attribute is absent, then there is no poster frame; return.
230        *self.poster_url.safe_borrow_mut(cx.no_gc()) = None;
231        let Some(url) = poster_attribute.filter(|poster_attribute| !poster_attribute.is_empty())
232        else {
233            self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
234            return;
235        };
236
237        // Step 3. Let url be the result of encoding-parsing a URL given
238        // the poster attribute's value, relative to the element's node
239        // document.
240        // Step 4. If url is failure, then return. There is no poster frame.
241        let global = self.owner_global();
242        let Ok(poster_url) = self
243            .owner_document()
244            .encoding_parse_a_url(url)
245            .map(|url| ensure_blob_referenced_by_url_is_kept_alive(&global, url))
246        else {
247            self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
248            return;
249        };
250        *self.poster_url.safe_borrow_mut(cx.no_gc()) = Some(poster_url.clone());
251
252        // We use the image cache for poster frames so we save as much
253        // network activity as possible.
254        let window = self.owner_window();
255        let image_cache = window.image_cache();
256        let cache_result = image_cache.get_cached_image_status(
257            poster_url.url(),
258            window.origin().immutable(),
259            None,
260        );
261
262        let id = match cache_result {
263            ImageCacheResult::Available(ImageOrMetadataAvailable::ImageAvailable {
264                image,
265                url,
266                ..
267            }) => {
268                self.process_image_response(cx, ImageResponse::Loaded(image, url));
269                return;
270            },
271            ImageCacheResult::Available(ImageOrMetadataAvailable::MetadataAvailable(_, id)) => id,
272            ImageCacheResult::ReadyForRequest(id) => {
273                self.do_fetch_poster_frame(cx, poster_url, id);
274                id
275            },
276            ImageCacheResult::FailedToLoadOrDecode => {
277                self.process_image_response(cx, ImageResponse::FailedToLoadOrDecode);
278                return;
279            },
280            ImageCacheResult::Pending(id) => id,
281        };
282
283        let trusted_node = Trusted::new(self);
284        let generation = self.generation_id();
285        let callback = window.register_image_cache_listener(id, move |response, cx| {
286            let element = trusted_node.root();
287
288            // Ignore any image response for a previous request that has been discarded.
289            if generation != element.generation_id() {
290                return;
291            }
292            element.process_image_response(cx, response.response);
293        });
294
295        image_cache.add_listener(ImageLoadListener::new(callback, window.pipeline_id(), id));
296    }
297
298    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
299    fn do_fetch_poster_frame(
300        &self,
301        cx: &mut JSContext,
302        poster_url: UrlWithBlobClaim,
303        id: PendingImageId,
304    ) {
305        // Step 5. Let request be a new request whose URL is url, client is the element's node
306        // document's relevant settings object, destination is "image", initiator type is "video",
307        // credentials mode is "include", and whose use-URL-credentials flag is set.
308        let document = self.owner_document();
309        let global = self.owner_global();
310        let request = RequestBuilder::new(
311            Some(document.webview_id()),
312            poster_url.clone(),
313            global.get_referrer(),
314        )
315        .destination(Destination::Image)
316        .credentials_mode(CredentialsMode::Include)
317        .use_url_credentials(true)
318        .with_global_scope(&global);
319
320        // Step 6. Fetch request. This must delay the load event of the element's node document.
321        // This delay must be independent from the ones created by HTMLMediaElement during
322        // its media load algorithm, otherwise a code like
323        // <video poster="poster.png"></video>
324        // (which triggers no media load algorithm unless a explicit call to .load() is done)
325        // will block the document's load event forever.
326        let blocker = &self.load_blocker;
327        LoadBlocker::terminate(cx, blocker);
328        let document = self.owner_document();
329        *blocker.safe_borrow_mut(cx.no_gc()) = Some(LoadBlocker::new(
330            &document,
331            LoadType::Image(poster_url.url()),
332        ));
333
334        let context = PosterFrameFetchContext::new(
335            self,
336            poster_url.url(),
337            id,
338            request.id,
339            self.global().core_resource_thread(),
340        );
341        self.owner_document().fetch_background(request, context);
342    }
343
344    fn generation_id(&self) -> u32 {
345        self.generation_id.get()
346    }
347
348    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
349    fn process_image_response(&self, cx: &mut JSContext, response: ImageResponse) {
350        // Step 7. If an image is thus obtained, the poster frame is that image.
351        // Otherwise, there is no poster frame.
352        match response {
353            ImageResponse::Loaded(image, url) => {
354                debug!("Loaded poster image for video element: {:?}", url);
355                match image.as_raster_image() {
356                    Some(image) => self
357                        .htmlmediaelement
358                        .set_poster_frame(cx.no_gc(), Some(image)),
359                    None => warn!("Vector images are not yet supported in video poster"),
360                }
361                LoadBlocker::terminate(cx, &self.load_blocker);
362            },
363            ImageResponse::MetadataLoaded(..) => {},
364            // The image cache may have loaded a placeholder for an invalid poster url
365            ImageResponse::FailedToLoadOrDecode => {
366                self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
367                // A failed load should unblock the document load.
368                LoadBlocker::terminate(cx, &self.load_blocker);
369            },
370        }
371    }
372
373    /// <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
374    pub(crate) fn is_usable(&self) -> bool {
375        !matches!(
376            self.htmlmediaelement.get_ready_state(),
377            ReadyState::HaveNothing | ReadyState::HaveMetadata
378        )
379    }
380
381    pub(crate) fn origin_is_clean(&self) -> bool {
382        self.htmlmediaelement.origin_is_clean()
383    }
384
385    pub(crate) fn is_network_state_empty(&self) -> bool {
386        self.htmlmediaelement.network_state() == NetworkState::Empty
387    }
388}
389
390impl HTMLVideoElementMethods<crate::DomTypeHolder> for HTMLVideoElement {
391    // <https://html.spec.whatwg.org/multipage/#dom-video-width>
392    make_dimension_uint_getter!(Width, "width");
393
394    // <https://html.spec.whatwg.org/multipage/#dom-video-width>
395    make_dimension_uint_setter!(SetWidth, "width");
396
397    // <https://html.spec.whatwg.org/multipage/#dom-video-height>
398    make_dimension_uint_getter!(Height, "height");
399
400    // <https://html.spec.whatwg.org/multipage/#dom-video-height>
401    make_dimension_uint_setter!(SetHeight, "height");
402
403    /// <https://html.spec.whatwg.org/multipage/#dom-video-videowidth>
404    fn VideoWidth(&self) -> u32 {
405        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
406            return 0;
407        }
408        self.video_width.get().unwrap_or(0)
409    }
410
411    /// <https://html.spec.whatwg.org/multipage/#dom-video-videoheight>
412    fn VideoHeight(&self) -> u32 {
413        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
414            return 0;
415        }
416        self.video_height.get().unwrap_or(0)
417    }
418
419    // https://html.spec.whatwg.org/multipage/#dom-video-poster
420    make_url_getter!(Poster, "poster");
421
422    // https://html.spec.whatwg.org/multipage/#dom-video-poster
423    make_url_setter!(SetPoster, "poster");
424
425    // For testing purposes only. This is not an event from
426    // https://html.spec.whatwg.org/multipage/#dom-video-poster
427    event_handler!(postershown, GetOnpostershown, SetOnpostershown);
428}
429
430impl VirtualMethods for HTMLVideoElement {
431    fn super_type(&self) -> Option<&dyn VirtualMethods> {
432        Some(self.upcast::<HTMLMediaElement>() as &dyn VirtualMethods)
433    }
434
435    fn attribute_mutated(
436        &self,
437        cx: &mut JSContext,
438        attr: AttrRef<'_>,
439        mutation: AttributeMutation,
440    ) {
441        self.super_type()
442            .unwrap()
443            .attribute_mutated(cx, attr, mutation);
444
445        if attr.local_name() == &local_name!("poster") {
446            if let Some(new_value) = mutation.new_value(attr) {
447                self.update_poster_frame(cx, Some(&new_value))
448            } else {
449                self.update_poster_frame(cx, None)
450            }
451        };
452    }
453
454    fn attribute_affects_presentational_hints(&self, attr: AttrRef<'_>) -> bool {
455        match attr.local_name() {
456            &local_name!("width") | &local_name!("height") => true,
457            _ => self
458                .super_type()
459                .unwrap()
460                .attribute_affects_presentational_hints(attr),
461        }
462    }
463
464    fn parse_plain_attribute(&self, name: &LocalName, value: DOMString) -> AttrValue {
465        match name {
466            &local_name!("width") | &local_name!("height") => {
467                AttrValue::from_dimension(value.into())
468            },
469            _ => self
470                .super_type()
471                .unwrap()
472                .parse_plain_attribute(name, value),
473        }
474    }
475}
476
477struct PosterFrameFetchContext {
478    /// Reference to the script thread image cache.
479    image_cache: Arc<dyn ImageCache>,
480    /// The element that initiated the request.
481    elem: Trusted<HTMLVideoElement>,
482    /// The cache ID for this request.
483    id: PendingImageId,
484    /// True if this response is invalid and should be ignored.
485    cancelled: bool,
486    /// Url for the resource
487    url: ServoUrl,
488    /// A [`FetchController`] for this request.
489    fetch_controller: FetchController,
490}
491
492impl FetchResponseListener for PosterFrameFetchContext {
493    fn process_request_body(&mut self, _: &GlobalScope, _: RequestId) {
494        self.fetch_controller.ignore()
495    }
496
497    fn process_response(
498        &mut self,
499        _: &mut JSContext,
500        _: &GlobalScope,
501        request_id: RequestId,
502        metadata: Result<FetchMetadata, NetworkError>,
503    ) {
504        self.image_cache.notify_pending_response(
505            self.id,
506            FetchResponseMsg::ProcessResponse(request_id, metadata.clone()),
507        );
508
509        let metadata: Option<Metadata> = metadata.ok().map(Into::into);
510
511        let status_is_ok = metadata
512            .as_ref()
513            .is_none_or(|m| m.status.in_range(200..300));
514
515        if !status_is_ok {
516            self.cancelled = true;
517            self.fetch_controller.abort();
518        }
519    }
520
521    fn process_response_chunk(
522        &mut self,
523        _: &mut JSContext,
524        _: &GlobalScope,
525        request_id: RequestId,
526        payload: Bytes,
527    ) {
528        if self.cancelled {
529            // An error was received previously, skip processing the payload.
530            return;
531        }
532
533        self.image_cache.notify_pending_response(
534            self.id,
535            FetchResponseMsg::ProcessResponseChunk(request_id, payload),
536        );
537    }
538
539    fn process_response_eof(
540        self,
541        cx: &mut JSContext,
542        global: &GlobalScope,
543        request_id: RequestId,
544        response: Result<(), NetworkError>,
545        timing: ResourceFetchTiming,
546    ) {
547        self.image_cache.notify_pending_response(
548            self.id,
549            FetchResponseMsg::ProcessResponseEOF(request_id, response.clone(), timing.clone()),
550        );
551        network_listener::submit_timing(cx, global, &self, &response, &timing);
552    }
553
554    fn process_csp_violations(
555        &mut self,
556        cx: &mut JSContext,
557        global: &GlobalScope,
558        _request_id: RequestId,
559        violations: Vec<Violation>,
560    ) {
561        global.report_csp_violations(cx, violations, None, None);
562    }
563
564    fn process_content_length(&mut self, request_id: RequestId, size: usize) {
565        self.image_cache.notify_pending_response(
566            self.id,
567            FetchResponseMsg::ProcessContentLength(request_id, size),
568        );
569    }
570}
571
572impl ResourceTimingListener for PosterFrameFetchContext {
573    fn resource_timing_information(&self) -> (InitiatorType, ServoUrl) {
574        let initiator_type =
575            InitiatorType::LocalName(self.elem.root().upcast::<Element>().local_name().clone());
576        (initiator_type, self.url.clone())
577    }
578}
579
580impl PosterFrameFetchContext {
581    fn new(
582        elem: &HTMLVideoElement,
583        url: ServoUrl,
584        id: PendingImageId,
585        request_id: RequestId,
586        core_resource_thread: CoreResourceThread,
587    ) -> PosterFrameFetchContext {
588        let window = elem.owner_window();
589        PosterFrameFetchContext {
590            image_cache: window.image_cache(),
591            elem: Trusted::new(elem),
592            id,
593            cancelled: false,
594            url,
595            fetch_controller: FetchController::new(request_id, false, core_resource_thread),
596        }
597    }
598}
599
600impl LayoutDom<'_, HTMLVideoElement> {
601    pub(crate) fn data(self) -> HTMLMediaData {
602        let video = self.unsafe_get();
603
604        // Get the current frame being rendered.
605        let current_frame = video.htmlmediaelement.get_current_frame_to_present();
606
607        // This value represents the natural width and height of the video.
608        // It may exist even if there is no current frame (for example, after the
609        // metadata of the video is loaded).
610        let metadata = video
611            .get_video_width()
612            .zip(video.get_video_height())
613            .map(|(width, height)| MediaMetadata { width, height });
614
615        HTMLMediaData {
616            current_frame,
617            metadata,
618            poster_or_src_url: if video.htmlmediaelement.show_poster() {
619                video.poster_url.borrow().clone().map(|url| url.url())
620            } else {
621                video.htmlmediaelement.current_src_url()
622            },
623        }
624    }
625
626    pub(crate) fn width(self) -> LengthOrPercentageOrAuto {
627        self.upcast::<Element>()
628            .get_attr_for_layout(&ns!(), &local_name!("width"))
629            .map(AttrValue::as_dimension)
630            .cloned()
631            .unwrap_or(LengthOrPercentageOrAuto::Auto)
632    }
633
634    pub(crate) fn height(self) -> LengthOrPercentageOrAuto {
635        self.upcast::<Element>()
636            .get_attr_for_layout(&ns!(), &local_name!("height"))
637            .map(AttrValue::as_dimension)
638            .cloned()
639            .unwrap_or(LengthOrPercentageOrAuto::Auto)
640    }
641}
script/dom/html/embedded_content/htmlvideoelement.rs

script/dom/html/embedded_content/
htmlvideoelement.rs
