script/dom/html/embedded_content/
htmlvideoelement.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::cell::Cell;
6use std::rc::Rc;
7use std::sync::Arc;
8
9use dom_struct::dom_struct;
10use euclid::default::Size2D;
11use html5ever::{LocalName, Prefix, local_name, ns};
12use js::context::{JSContext, NoGC};
13use js::rust::HandleObject;
14use layout_api::{HTMLMediaData, MediaMetadata};
15use net_traits::blob_url_store::UrlWithBlobClaim;
16use net_traits::image_cache::{
17    ImageCache, ImageCacheResult, ImageLoadListener, ImageOrMetadataAvailable, ImageResponse,
18    PendingImageId,
19};
20use net_traits::request::{CredentialsMode, Destination, RequestBuilder, RequestId};
21use net_traits::{
22    CoreResourceThread, FetchMetadata, FetchResponseMsg, NetworkError, ResourceFetchTiming,
23};
24use pixels::{Snapshot, SnapshotAlphaMode, SnapshotPixelFormat};
25use script_bindings::cell::DomRefCell;
26use script_bindings::error::Fallible;
27use servo_media::player::video::VideoFrame;
28use servo_url::ServoUrl;
29use style::attr::{AttrValue, LengthOrPercentageOrAuto};
30
31use crate::document_loader::{LoadBlocker, LoadType};
32use crate::dom::bindings::codegen::Bindings::HTMLVideoElementBinding::HTMLVideoElementMethods;
33use crate::dom::bindings::inheritance::Castable;
34use crate::dom::bindings::refcounted::Trusted;
35use crate::dom::bindings::reflector::DomGlobal;
36use crate::dom::bindings::root::{DomRoot, LayoutDom};
37use crate::dom::bindings::str::{DOMString, USVString};
38use crate::dom::csp::{GlobalCspReporting, Violation};
39use crate::dom::document::Document;
40use crate::dom::element::attributes::storage::AttrRef;
41use crate::dom::element::{AttributeMutation, Element};
42use crate::dom::globalscope::GlobalScope;
43use crate::dom::html::htmlmediaelement::{HTMLMediaElement, NetworkState, ReadyState};
44use crate::dom::node::virtualmethods::VirtualMethods;
45use crate::dom::node::{Node, NodeDamage, NodeTraits};
46use crate::dom::performance::performanceresourcetiming::InitiatorType;
47use crate::fetch::{FetchCanceller, RequestWithGlobalScope};
48use crate::network_listener::{self, FetchResponseListener, ResourceTimingListener};
49use crate::url::ensure_blob_referenced_by_url_is_kept_alive;
50
51#[dom_struct]
52pub(crate) struct HTMLVideoElement {
53    htmlmediaelement: HTMLMediaElement,
54    /// <https://html.spec.whatwg.org/multipage/#dom-video-videowidth>
55    video_width: Cell<Option<u32>>,
56    /// <https://html.spec.whatwg.org/multipage/#dom-video-videoheight>
57    video_height: Cell<Option<u32>>,
58    /// <https://html.spec.whatwg.org/multipage/#dom-video-poster>
59    #[no_trace]
60    poster_url: DomRefCell<Option<UrlWithBlobClaim>>,
61    /// Incremented whenever tasks associated with this element are cancelled.
62    generation_id: Cell<u32>,
63    /// Load event blocker. Will block the load event while the poster frame
64    /// is being fetched.
65    load_blocker: DomRefCell<Option<LoadBlocker>>,
66    /// A copy of the last frame
67    #[ignore_malloc_size_of = "VideoFrame"]
68    #[no_trace]
69    last_frame: DomRefCell<Option<VideoFrame>>,
70    #[cfg(feature = "webgpu")]
71    #[conditional_malloc_size_of]
72    /// Planar texture for WebGPU
73    planar_texture: DomRefCell<Option<Rc<crate::dom::gpuexternaltexture::PlanarTexture>>>,
74}
75
76impl HTMLVideoElement {
77    fn new_inherited(
78        local_name: LocalName,
79        prefix: Option<Prefix>,
80        document: &Document,
81    ) -> HTMLVideoElement {
82        HTMLVideoElement {
83            htmlmediaelement: HTMLMediaElement::new_inherited(local_name, prefix, document),
84            video_width: Cell::new(None),
85            video_height: Cell::new(None),
86            poster_url: Default::default(),
87            generation_id: Cell::new(0),
88            load_blocker: Default::default(),
89            last_frame: Default::default(),
90            #[cfg(feature = "webgpu")]
91            planar_texture: DomRefCell::new(None),
92        }
93    }
94
95    pub(crate) fn new(
96        cx: &mut JSContext,
97        local_name: LocalName,
98        prefix: Option<Prefix>,
99        document: &Document,
100        proto: Option<HandleObject>,
101    ) -> DomRoot<HTMLVideoElement> {
102        Node::reflect_weak_referenceable_node_with_proto(
103            cx,
104            Rc::new(HTMLVideoElement::new_inherited(
105                local_name, prefix, document,
106            )),
107            document,
108            proto,
109        )
110    }
111
112    pub(crate) fn get_video_width(&self) -> Option<u32> {
113        self.video_width.get()
114    }
115
116    pub(crate) fn get_video_height(&self) -> Option<u32> {
117        self.video_height.get()
118    }
119
120    pub(crate) fn set_natural_dimensions(
121        &self,
122        no_gc: &NoGC,
123        width: Option<u32>,
124        height: Option<u32>,
125    ) -> bool {
126        if self.video_width.get() == width && self.video_height.get() == height {
127            return false;
128        }
129
130        self.video_width.set(width);
131        self.video_height.set(height);
132
133        self.upcast::<Node>().dirty(no_gc, NodeDamage::Other);
134        true
135    }
136
137    /// Gets the copy of the video frame at the current playback position,
138    /// if that is available, or else (e.g. when the video is seeking or buffering)
139    /// its previous appearance, if any.
140    pub(crate) fn get_current_frame_data(&self) -> Option<Snapshot> {
141        let frame = self.htmlmediaelement.get_current_frame();
142        if frame.is_some() {
143            *self.last_frame.borrow_mut() = frame;
144        }
145
146        match self.last_frame.borrow().as_ref() {
147            Some(frame) => {
148                let size = Size2D::new(frame.get_width() as u32, frame.get_height() as u32);
149                if !frame.is_gl_texture() {
150                    let alpha_mode = SnapshotAlphaMode::Transparent {
151                        premultiplied: false,
152                    };
153
154                    Some(Snapshot::from_vec(
155                        size.cast(),
156                        SnapshotPixelFormat::BGRA,
157                        alpha_mode,
158                        frame.get_data().to_vec(),
159                    ))
160                } else {
161                    // XXX(victor): here we only have the GL texture ID.
162                    Some(Snapshot::cleared(size.cast()))
163                }
164            },
165            None => None,
166        }
167    }
168
169    #[cfg(feature = "webgpu")]
170    pub(crate) fn planar_video_for_webgpu(
171        &self,
172        device: &crate::dom::types::GPUDevice,
173    ) -> Fallible<(
174        Size2D<u32>,
175        Option<Rc<crate::dom::gpuexternaltexture::PlanarTexture>>,
176    )> {
177        use crate::dom::gpuexternaltexture::PlanarTexture;
178        // 1. If source is not origin-clean, throw a SecurityError and return.
179        if !self.origin_is_clean() {
180            return Err(script_bindings::error::Error::Security(Some(
181                "Source is not origin-clean".to_string(),
182            )));
183        }
184        // 2. Let usability be ? check the usability of the image argument(source).
185        if !self.is_usable() {
186            // 3. If usability is not good:
187            // Generate a validation error.
188            // Return an invalidated GPUExternalTexture.
189            Ok((Size2D::zero(), None))
190        } else {
191            // 4. Let data be the result of converting the current image contents of source into the color space descriptor.colorSpace with unpremultiplied alpha.
192            let mut planar_texture = self.planar_texture.borrow_mut();
193            match planar_texture.as_ref() {
194                Some(planar_texture) => {
195                    if planar_texture.is_expired() &&
196                        let Some(snapshot) = self.get_current_frame_data()
197                    {
198                        planar_texture.update(snapshot);
199                    }
200                },
201                None => {
202                    *planar_texture = self.get_current_frame_data().map(|snapshot| {
203                        Rc::new(PlanarTexture::new(device.channel(), device, snapshot))
204                    });
205                },
206            };
207            Ok((
208                planar_texture
209                    .as_ref()
210                    .map(|pt| pt.size())
211                    .unwrap_or_default(),
212                planar_texture.as_ref().cloned(),
213            ))
214        }
215    }
216
217    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
218    fn update_poster_frame(&self, poster_attribute: Option<&str>, cx: &mut JSContext) {
219        // Step 1. If there is an existing instance of this algorithm running
220        // for this video element, abort that instance of this algorithm without
221        // changing the poster frame.
222        self.generation_id.set(self.generation_id.get() + 1);
223
224        // Step 2. If the poster attribute's value is the empty string or
225        // if the attribute is absent, then there is no poster frame; return.
226        *self.poster_url.safe_borrow_mut(cx.no_gc()) = None;
227        let Some(url) = poster_attribute.filter(|poster_attribute| !poster_attribute.is_empty())
228        else {
229            self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
230            return;
231        };
232
233        // Step 3. Let url be the result of encoding-parsing a URL given
234        // the poster attribute's value, relative to the element's node
235        // document.
236        // Step 4. If url is failure, then return. There is no poster frame.
237        let global = self.owner_global();
238        let Ok(poster_url) = self
239            .owner_document()
240            .encoding_parse_a_url(url)
241            .map(|url| ensure_blob_referenced_by_url_is_kept_alive(&global, url))
242        else {
243            self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
244            return;
245        };
246        *self.poster_url.safe_borrow_mut(cx.no_gc()) = Some(poster_url.clone());
247
248        // We use the image cache for poster frames so we save as much
249        // network activity as possible.
250        let window = self.owner_window();
251        let image_cache = window.image_cache();
252        let cache_result = image_cache.get_cached_image_status(
253            poster_url.url(),
254            window.origin().immutable().clone(),
255            None,
256        );
257
258        let id = match cache_result {
259            ImageCacheResult::Available(ImageOrMetadataAvailable::ImageAvailable {
260                image,
261                url,
262                ..
263            }) => {
264                self.process_image_response(ImageResponse::Loaded(image, url), cx);
265                return;
266            },
267            ImageCacheResult::Available(ImageOrMetadataAvailable::MetadataAvailable(_, id)) => id,
268            ImageCacheResult::ReadyForRequest(id) => {
269                self.do_fetch_poster_frame(poster_url, id, cx);
270                id
271            },
272            ImageCacheResult::FailedToLoadOrDecode => {
273                self.process_image_response(ImageResponse::FailedToLoadOrDecode, cx);
274                return;
275            },
276            ImageCacheResult::Pending(id) => id,
277        };
278
279        let trusted_node = Trusted::new(self);
280        let generation = self.generation_id();
281        let callback = window.register_image_cache_listener(id, move |response, cx| {
282            let element = trusted_node.root();
283
284            // Ignore any image response for a previous request that has been discarded.
285            if generation != element.generation_id() {
286                return;
287            }
288            element.process_image_response(response.response, cx);
289        });
290
291        image_cache.add_listener(ImageLoadListener::new(callback, window.pipeline_id(), id));
292    }
293
294    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
295    fn do_fetch_poster_frame(
296        &self,
297        poster_url: UrlWithBlobClaim,
298        id: PendingImageId,
299        cx: &mut JSContext,
300    ) {
301        // Step 5. Let request be a new request whose URL is url, client is the element's node
302        // document's relevant settings object, destination is "image", initiator type is "video",
303        // credentials mode is "include", and whose use-URL-credentials flag is set.
304        let document = self.owner_document();
305        let global = self.owner_global();
306        let request = RequestBuilder::new(
307            Some(document.webview_id()),
308            poster_url.clone(),
309            global.get_referrer(),
310        )
311        .destination(Destination::Image)
312        .credentials_mode(CredentialsMode::Include)
313        .use_url_credentials(true)
314        .with_global_scope(&global);
315
316        // Step 6. Fetch request. This must delay the load event of the element's node document.
317        // This delay must be independent from the ones created by HTMLMediaElement during
318        // its media load algorithm, otherwise a code like
319        // <video poster="poster.png"></video>
320        // (which triggers no media load algorithm unless a explicit call to .load() is done)
321        // will block the document's load event forever.
322        let blocker = &self.load_blocker;
323        LoadBlocker::terminate(blocker, cx);
324        let document = self.owner_document();
325        *blocker.borrow_mut() = Some(LoadBlocker::new(
326            &document,
327            LoadType::Image(poster_url.url()),
328        ));
329
330        let context = PosterFrameFetchContext::new(
331            self,
332            poster_url.url(),
333            id,
334            request.id,
335            self.global().core_resource_thread(),
336        );
337        self.owner_document().fetch_background(request, context);
338    }
339
340    fn generation_id(&self) -> u32 {
341        self.generation_id.get()
342    }
343
344    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
345    fn process_image_response(&self, response: ImageResponse, cx: &mut JSContext) {
346        // Step 7. If an image is thus obtained, the poster frame is that image.
347        // Otherwise, there is no poster frame.
348        match response {
349            ImageResponse::Loaded(image, url) => {
350                debug!("Loaded poster image for video element: {:?}", url);
351                match image.as_raster_image() {
352                    Some(image) => self
353                        .htmlmediaelement
354                        .set_poster_frame(cx.no_gc(), Some(image)),
355                    None => warn!("Vector images are not yet supported in video poster"),
356                }
357                LoadBlocker::terminate(&self.load_blocker, cx);
358            },
359            ImageResponse::MetadataLoaded(..) => {},
360            // The image cache may have loaded a placeholder for an invalid poster url
361            ImageResponse::FailedToLoadOrDecode => {
362                self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
363                // A failed load should unblock the document load.
364                LoadBlocker::terminate(&self.load_blocker, cx);
365            },
366        }
367    }
368
369    /// <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
370    pub(crate) fn is_usable(&self) -> bool {
371        !matches!(
372            self.htmlmediaelement.get_ready_state(),
373            ReadyState::HaveNothing | ReadyState::HaveMetadata
374        )
375    }
376
377    pub(crate) fn origin_is_clean(&self) -> bool {
378        self.htmlmediaelement.origin_is_clean()
379    }
380
381    pub(crate) fn is_network_state_empty(&self) -> bool {
382        self.htmlmediaelement.network_state() == NetworkState::Empty
383    }
384}
385
386impl HTMLVideoElementMethods<crate::DomTypeHolder> for HTMLVideoElement {
387    // <https://html.spec.whatwg.org/multipage/#dom-video-width>
388    make_dimension_uint_getter!(Width, "width");
389
390    // <https://html.spec.whatwg.org/multipage/#dom-video-width>
391    make_dimension_uint_setter!(SetWidth, "width");
392
393    // <https://html.spec.whatwg.org/multipage/#dom-video-height>
394    make_dimension_uint_getter!(Height, "height");
395
396    // <https://html.spec.whatwg.org/multipage/#dom-video-height>
397    make_dimension_uint_setter!(SetHeight, "height");
398
399    /// <https://html.spec.whatwg.org/multipage/#dom-video-videowidth>
400    fn VideoWidth(&self) -> u32 {
401        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
402            return 0;
403        }
404        self.video_width.get().unwrap_or(0)
405    }
406
407    /// <https://html.spec.whatwg.org/multipage/#dom-video-videoheight>
408    fn VideoHeight(&self) -> u32 {
409        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
410            return 0;
411        }
412        self.video_height.get().unwrap_or(0)
413    }
414
415    // https://html.spec.whatwg.org/multipage/#dom-video-poster
416    make_url_getter!(Poster, "poster");
417
418    // https://html.spec.whatwg.org/multipage/#dom-video-poster
419    make_url_setter!(SetPoster, "poster");
420
421    // For testing purposes only. This is not an event from
422    // https://html.spec.whatwg.org/multipage/#dom-video-poster
423    event_handler!(postershown, GetOnpostershown, SetOnpostershown);
424}
425
426impl VirtualMethods for HTMLVideoElement {
427    fn super_type(&self) -> Option<&dyn VirtualMethods> {
428        Some(self.upcast::<HTMLMediaElement>() as &dyn VirtualMethods)
429    }
430
431    fn attribute_mutated(
432        &self,
433        cx: &mut JSContext,
434        attr: AttrRef<'_>,
435        mutation: AttributeMutation,
436    ) {
437        self.super_type()
438            .unwrap()
439            .attribute_mutated(cx, attr, mutation);
440
441        if attr.local_name() == &local_name!("poster") {
442            if let Some(new_value) = mutation.new_value(attr) {
443                self.update_poster_frame(Some(&new_value), cx)
444            } else {
445                self.update_poster_frame(None, cx)
446            }
447        };
448    }
449
450    fn attribute_affects_presentational_hints(&self, attr: AttrRef<'_>) -> bool {
451        match attr.local_name() {
452            &local_name!("width") | &local_name!("height") => true,
453            _ => self
454                .super_type()
455                .unwrap()
456                .attribute_affects_presentational_hints(attr),
457        }
458    }
459
460    fn parse_plain_attribute(&self, name: &LocalName, value: DOMString) -> AttrValue {
461        match name {
462            &local_name!("width") | &local_name!("height") => {
463                AttrValue::from_dimension(value.into())
464            },
465            _ => self
466                .super_type()
467                .unwrap()
468                .parse_plain_attribute(name, value),
469        }
470    }
471}
472
473struct PosterFrameFetchContext {
474    /// Reference to the script thread image cache.
475    image_cache: Arc<dyn ImageCache>,
476    /// The element that initiated the request.
477    elem: Trusted<HTMLVideoElement>,
478    /// The cache ID for this request.
479    id: PendingImageId,
480    /// True if this response is invalid and should be ignored.
481    cancelled: bool,
482    /// Url for the resource
483    url: ServoUrl,
484    /// A [`FetchCanceller`] for this request.
485    fetch_canceller: FetchCanceller,
486}
487
488impl FetchResponseListener for PosterFrameFetchContext {
489    fn process_request_body(&mut self, _: RequestId) {
490        self.fetch_canceller.ignore()
491    }
492
493    fn process_response(
494        &mut self,
495        _: &mut JSContext,
496        request_id: RequestId,
497        metadata: Result<FetchMetadata, NetworkError>,
498    ) {
499        self.image_cache.notify_pending_response(
500            self.id,
501            FetchResponseMsg::ProcessResponse(request_id, metadata.clone()),
502        );
503
504        let metadata = metadata.ok().map(|meta| match meta {
505            FetchMetadata::Unfiltered(m) => m,
506            FetchMetadata::Filtered { unsafe_, .. } => unsafe_,
507        });
508
509        let status_is_ok = metadata
510            .as_ref()
511            .is_none_or(|m| m.status.in_range(200..300));
512
513        if !status_is_ok {
514            self.cancelled = true;
515            self.fetch_canceller.abort();
516        }
517    }
518
519    fn process_response_chunk(
520        &mut self,
521        _: &mut JSContext,
522        request_id: RequestId,
523        payload: Vec<u8>,
524    ) {
525        if self.cancelled {
526            // An error was received previously, skip processing the payload.
527            return;
528        }
529
530        self.image_cache.notify_pending_response(
531            self.id,
532            FetchResponseMsg::ProcessResponseChunk(request_id, payload.into()),
533        );
534    }
535
536    fn process_response_eof(
537        self,
538        cx: &mut JSContext,
539        request_id: RequestId,
540        response: Result<(), NetworkError>,
541        timing: ResourceFetchTiming,
542    ) {
543        self.image_cache.notify_pending_response(
544            self.id,
545            FetchResponseMsg::ProcessResponseEOF(request_id, response.clone(), timing.clone()),
546        );
547        network_listener::submit_timing(cx, &self, &response, &timing);
548    }
549
550    fn process_csp_violations(
551        &mut self,
552        cx: &mut js::context::JSContext,
553        _request_id: RequestId,
554        violations: Vec<Violation>,
555    ) {
556        let global = &self.resource_timing_global();
557        global.report_csp_violations(cx, violations, None, None);
558    }
559
560    fn process_content_length(&mut self, request_id: RequestId, size: usize) {
561        self.image_cache.notify_pending_response(
562            self.id,
563            FetchResponseMsg::ProcessContentLength(request_id, size),
564        );
565    }
566}
567
568impl ResourceTimingListener for PosterFrameFetchContext {
569    fn resource_timing_information(&self) -> (InitiatorType, ServoUrl) {
570        let initiator_type = InitiatorType::LocalName(
571            self.elem
572                .root()
573                .upcast::<Element>()
574                .local_name()
575                .to_string(),
576        );
577        (initiator_type, self.url.clone())
578    }
579
580    fn resource_timing_global(&self) -> DomRoot<GlobalScope> {
581        self.elem.root().owner_document().global()
582    }
583}
584
585impl PosterFrameFetchContext {
586    fn new(
587        elem: &HTMLVideoElement,
588        url: ServoUrl,
589        id: PendingImageId,
590        request_id: RequestId,
591        core_resource_thread: CoreResourceThread,
592    ) -> PosterFrameFetchContext {
593        let window = elem.owner_window();
594        PosterFrameFetchContext {
595            image_cache: window.image_cache(),
596            elem: Trusted::new(elem),
597            id,
598            cancelled: false,
599            url,
600            fetch_canceller: FetchCanceller::new(request_id, false, core_resource_thread),
601        }
602    }
603}
604
605impl LayoutDom<'_, HTMLVideoElement> {
606    #[expect(unsafe_code)]
607    pub(crate) fn data(self) -> HTMLMediaData {
608        let video = self.unsafe_get();
609
610        // Get the current frame being rendered.
611        let current_frame = video.htmlmediaelement.get_current_frame_to_present();
612
613        // This value represents the natural width and height of the video.
614        // It may exist even if there is no current frame (for example, after the
615        // metadata of the video is loaded).
616        let metadata = video
617            .get_video_width()
618            .zip(video.get_video_height())
619            .map(|(width, height)| MediaMetadata { width, height });
620
621        HTMLMediaData {
622            current_frame,
623            metadata,
624            poster_url: unsafe {
625                video
626                    .poster_url
627                    .borrow_for_layout()
628                    .clone()
629                    .map(|url| url.url())
630            },
631        }
632    }
633
634    pub(crate) fn get_width(self) -> LengthOrPercentageOrAuto {
635        self.upcast::<Element>()
636            .get_attr_for_layout(&ns!(), &local_name!("width"))
637            .map(AttrValue::as_dimension)
638            .cloned()
639            .unwrap_or(LengthOrPercentageOrAuto::Auto)
640    }
641
642    pub(crate) fn get_height(self) -> LengthOrPercentageOrAuto {
643        self.upcast::<Element>()
644            .get_attr_for_layout(&ns!(), &local_name!("height"))
645            .map(AttrValue::as_dimension)
646            .cloned()
647            .unwrap_or(LengthOrPercentageOrAuto::Auto)
648    }
649}
script/dom/html/embedded_content/htmlvideoelement.rs

script/dom/html/embedded_content/
htmlvideoelement.rs
