script/dom/html/embedded_content/
htmlvideoelement.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5#![cfg_attr(crown, allow(crown::jscontext_first_arg))]
6
7use std::cell::Cell;
8use std::rc::Rc;
9use std::sync::Arc;
10
11use bytes::Bytes;
12use dom_struct::dom_struct;
13use euclid::default::Size2D;
14use html5ever::{LocalName, Prefix, local_name, ns};
15use js::context::{JSContext, NoGC};
16use js::rust::HandleObject;
17use layout_api::{HTMLMediaData, MediaMetadata};
18use net_traits::blob_url_store::UrlWithBlobClaim;
19use net_traits::image_cache::{
20    ImageCache, ImageCacheResult, ImageLoadListener, ImageOrMetadataAvailable, ImageResponse,
21    PendingImageId,
22};
23use net_traits::request::{CredentialsMode, Destination, RequestBuilder, RequestId};
24use net_traits::{
25    CoreResourceThread, FetchMetadata, FetchResponseMsg, NetworkError, ResourceFetchTiming,
26};
27use pixels::{Snapshot, SnapshotAlphaMode, SnapshotPixelFormat};
28use script_bindings::cell::DomRefCell;
29use script_bindings::error::Fallible;
30use servo_media::player::video::VideoFrame;
31use servo_url::ServoUrl;
32use style::attr::{AttrValue, LengthOrPercentageOrAuto};
33
34use crate::dom::bindings::codegen::Bindings::HTMLVideoElementBinding::HTMLVideoElementMethods;
35use crate::dom::bindings::inheritance::Castable;
36use crate::dom::bindings::refcounted::Trusted;
37use crate::dom::bindings::reflector::DomGlobal;
38use crate::dom::bindings::root::{DomRoot, LayoutDom};
39use crate::dom::bindings::str::{DOMString, USVString};
40use crate::dom::csp::{GlobalCspReporting, Violation};
41use crate::dom::document::Document;
42use crate::dom::element::attributes::storage::AttrRef;
43use crate::dom::element::{AttributeMutation, Element};
44use crate::dom::globalscope::GlobalScope;
45use crate::dom::html::htmlmediaelement::{HTMLMediaElement, NetworkState, ReadyState};
46use crate::dom::node::virtualmethods::VirtualMethods;
47use crate::dom::node::{Node, NodeDamage, NodeTraits};
48use crate::dom::performance::performanceresourcetiming::InitiatorType;
49use crate::event_loop::document_loader::{LoadBlocker, LoadType};
50use crate::fetch::fetch::{FetchCanceller, RequestWithGlobalScope};
51use crate::fetch::network_listener::{self, FetchResponseListener, ResourceTimingListener};
52use crate::url::ensure_blob_referenced_by_url_is_kept_alive;
53
54#[dom_struct]
55pub(crate) struct HTMLVideoElement {
56    htmlmediaelement: HTMLMediaElement,
57    /// <https://html.spec.whatwg.org/multipage/#dom-video-videowidth>
58    video_width: Cell<Option<u32>>,
59    /// <https://html.spec.whatwg.org/multipage/#dom-video-videoheight>
60    video_height: Cell<Option<u32>>,
61    /// <https://html.spec.whatwg.org/multipage/#dom-video-poster>
62    #[no_trace]
63    poster_url: DomRefCell<Option<UrlWithBlobClaim>>,
64    /// Incremented whenever tasks associated with this element are cancelled.
65    generation_id: Cell<u32>,
66    /// Load event blocker. Will block the load event while the poster frame
67    /// is being fetched.
68    load_blocker: DomRefCell<Option<LoadBlocker>>,
69    /// A copy of the last frame
70    #[ignore_malloc_size_of = "VideoFrame"]
71    #[no_trace]
72    last_frame: DomRefCell<Option<VideoFrame>>,
73    #[cfg(feature = "webgpu")]
74    #[conditional_malloc_size_of]
75    /// Planar texture for WebGPU
76    planar_texture: DomRefCell<Option<Rc<crate::dom::gpuexternaltexture::PlanarTexture>>>,
77}
78
79impl HTMLVideoElement {
80    fn new_inherited(
81        local_name: LocalName,
82        prefix: Option<Prefix>,
83        document: &Document,
84    ) -> HTMLVideoElement {
85        HTMLVideoElement {
86            htmlmediaelement: HTMLMediaElement::new_inherited(local_name, prefix, document),
87            video_width: Cell::new(None),
88            video_height: Cell::new(None),
89            poster_url: Default::default(),
90            generation_id: Cell::new(0),
91            load_blocker: Default::default(),
92            last_frame: Default::default(),
93            #[cfg(feature = "webgpu")]
94            planar_texture: DomRefCell::new(None),
95        }
96    }
97
98    pub(crate) fn new(
99        cx: &mut JSContext,
100        local_name: LocalName,
101        prefix: Option<Prefix>,
102        document: &Document,
103        proto: Option<HandleObject>,
104    ) -> DomRoot<HTMLVideoElement> {
105        Node::reflect_weak_referenceable_node_with_proto(
106            cx,
107            Rc::new(HTMLVideoElement::new_inherited(
108                local_name, prefix, document,
109            )),
110            document,
111            proto,
112        )
113    }
114
115    pub(crate) fn get_video_width(&self) -> Option<u32> {
116        self.video_width.get()
117    }
118
119    pub(crate) fn get_video_height(&self) -> Option<u32> {
120        self.video_height.get()
121    }
122
123    pub(crate) fn set_natural_dimensions(
124        &self,
125        no_gc: &NoGC,
126        width: Option<u32>,
127        height: Option<u32>,
128    ) -> bool {
129        if self.video_width.get() == width && self.video_height.get() == height {
130            return false;
131        }
132
133        self.video_width.set(width);
134        self.video_height.set(height);
135
136        self.upcast::<Node>().dirty(no_gc, NodeDamage::Other);
137        true
138    }
139
140    /// Gets the copy of the video frame at the current playback position,
141    /// if that is available, or else (e.g. when the video is seeking or buffering)
142    /// its previous appearance, if any.
143    pub(crate) fn get_current_frame_data(&self) -> Option<Snapshot> {
144        let frame = self.htmlmediaelement.get_current_frame();
145        if frame.is_some() {
146            *self.last_frame.borrow_mut() = frame;
147        }
148
149        match self.last_frame.borrow().as_ref() {
150            Some(frame) => {
151                let size = Size2D::new(frame.get_width() as u32, frame.get_height() as u32);
152                if !frame.is_gl_texture() {
153                    let alpha_mode = SnapshotAlphaMode::Transparent {
154                        premultiplied: false,
155                    };
156
157                    Some(Snapshot::from_vec(
158                        size.cast(),
159                        SnapshotPixelFormat::BGRA,
160                        alpha_mode,
161                        frame.get_data().to_vec(),
162                    ))
163                } else {
164                    // XXX(victor): here we only have the GL texture ID.
165                    Some(Snapshot::cleared(size.cast()))
166                }
167            },
168            None => None,
169        }
170    }
171
172    #[cfg(feature = "webgpu")]
173    pub(crate) fn planar_video_for_webgpu(
174        &self,
175        device: &crate::dom::types::GPUDevice,
176    ) -> Fallible<(
177        Size2D<u32>,
178        Option<Rc<crate::dom::gpuexternaltexture::PlanarTexture>>,
179    )> {
180        use crate::dom::gpuexternaltexture::PlanarTexture;
181        // 1. If source is not origin-clean, throw a SecurityError and return.
182        if !self.origin_is_clean() {
183            return Err(script_bindings::error::Error::Security(Some(
184                "Source is not origin-clean".to_string(),
185            )));
186        }
187        // 2. Let usability be ? check the usability of the image argument(source).
188        if !self.is_usable() {
189            // 3. If usability is not good:
190            // Generate a validation error.
191            // Return an invalidated GPUExternalTexture.
192            Ok((Size2D::zero(), None))
193        } else {
194            // 4. Let data be the result of converting the current image contents of source into the color space descriptor.colorSpace with unpremultiplied alpha.
195            let mut planar_texture = self.planar_texture.borrow_mut();
196            match planar_texture.as_ref() {
197                Some(planar_texture) => {
198                    if planar_texture.is_expired() &&
199                        let Some(snapshot) = self.get_current_frame_data()
200                    {
201                        planar_texture.update(snapshot);
202                    }
203                },
204                None => {
205                    *planar_texture = self.get_current_frame_data().map(|snapshot| {
206                        Rc::new(PlanarTexture::new(device.channel(), device, snapshot))
207                    });
208                },
209            };
210            Ok((
211                planar_texture
212                    .as_ref()
213                    .map(|pt| pt.size())
214                    .unwrap_or_default(),
215                planar_texture.as_ref().cloned(),
216            ))
217        }
218    }
219
220    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
221    fn update_poster_frame(&self, poster_attribute: Option<&str>, cx: &mut JSContext) {
222        // Step 1. If there is an existing instance of this algorithm running
223        // for this video element, abort that instance of this algorithm without
224        // changing the poster frame.
225        self.generation_id.set(self.generation_id.get() + 1);
226
227        // Step 2. If the poster attribute's value is the empty string or
228        // if the attribute is absent, then there is no poster frame; return.
229        *self.poster_url.safe_borrow_mut(cx.no_gc()) = None;
230        let Some(url) = poster_attribute.filter(|poster_attribute| !poster_attribute.is_empty())
231        else {
232            self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
233            return;
234        };
235
236        // Step 3. Let url be the result of encoding-parsing a URL given
237        // the poster attribute's value, relative to the element's node
238        // document.
239        // Step 4. If url is failure, then return. There is no poster frame.
240        let global = self.owner_global();
241        let Ok(poster_url) = self
242            .owner_document()
243            .encoding_parse_a_url(url)
244            .map(|url| ensure_blob_referenced_by_url_is_kept_alive(&global, url))
245        else {
246            self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
247            return;
248        };
249        *self.poster_url.safe_borrow_mut(cx.no_gc()) = Some(poster_url.clone());
250
251        // We use the image cache for poster frames so we save as much
252        // network activity as possible.
253        let window = self.owner_window();
254        let image_cache = window.image_cache();
255        let cache_result = image_cache.get_cached_image_status(
256            poster_url.url(),
257            window.origin().immutable().clone(),
258            None,
259        );
260
261        let id = match cache_result {
262            ImageCacheResult::Available(ImageOrMetadataAvailable::ImageAvailable {
263                image,
264                url,
265                ..
266            }) => {
267                self.process_image_response(ImageResponse::Loaded(image, url), cx);
268                return;
269            },
270            ImageCacheResult::Available(ImageOrMetadataAvailable::MetadataAvailable(_, id)) => id,
271            ImageCacheResult::ReadyForRequest(id) => {
272                self.do_fetch_poster_frame(poster_url, id, cx);
273                id
274            },
275            ImageCacheResult::FailedToLoadOrDecode => {
276                self.process_image_response(ImageResponse::FailedToLoadOrDecode, cx);
277                return;
278            },
279            ImageCacheResult::Pending(id) => id,
280        };
281
282        let trusted_node = Trusted::new(self);
283        let generation = self.generation_id();
284        let callback = window.register_image_cache_listener(id, move |response, cx| {
285            let element = trusted_node.root();
286
287            // Ignore any image response for a previous request that has been discarded.
288            if generation != element.generation_id() {
289                return;
290            }
291            element.process_image_response(response.response, cx);
292        });
293
294        image_cache.add_listener(ImageLoadListener::new(callback, window.pipeline_id(), id));
295    }
296
297    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
298    fn do_fetch_poster_frame(
299        &self,
300        poster_url: UrlWithBlobClaim,
301        id: PendingImageId,
302        cx: &mut JSContext,
303    ) {
304        // Step 5. Let request be a new request whose URL is url, client is the element's node
305        // document's relevant settings object, destination is "image", initiator type is "video",
306        // credentials mode is "include", and whose use-URL-credentials flag is set.
307        let document = self.owner_document();
308        let global = self.owner_global();
309        let request = RequestBuilder::new(
310            Some(document.webview_id()),
311            poster_url.clone(),
312            global.get_referrer(),
313        )
314        .destination(Destination::Image)
315        .credentials_mode(CredentialsMode::Include)
316        .use_url_credentials(true)
317        .with_global_scope(&global);
318
319        // Step 6. Fetch request. This must delay the load event of the element's node document.
320        // This delay must be independent from the ones created by HTMLMediaElement during
321        // its media load algorithm, otherwise a code like
322        // <video poster="poster.png"></video>
323        // (which triggers no media load algorithm unless a explicit call to .load() is done)
324        // will block the document's load event forever.
325        let blocker = &self.load_blocker;
326        LoadBlocker::terminate(blocker, cx);
327        let document = self.owner_document();
328        *blocker.borrow_mut() = Some(LoadBlocker::new(
329            &document,
330            LoadType::Image(poster_url.url()),
331        ));
332
333        let context = PosterFrameFetchContext::new(
334            self,
335            poster_url.url(),
336            id,
337            request.id,
338            self.global().core_resource_thread(),
339        );
340        self.owner_document().fetch_background(request, context);
341    }
342
343    fn generation_id(&self) -> u32 {
344        self.generation_id.get()
345    }
346
347    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
348    fn process_image_response(&self, response: ImageResponse, cx: &mut JSContext) {
349        // Step 7. If an image is thus obtained, the poster frame is that image.
350        // Otherwise, there is no poster frame.
351        match response {
352            ImageResponse::Loaded(image, url) => {
353                debug!("Loaded poster image for video element: {:?}", url);
354                match image.as_raster_image() {
355                    Some(image) => self
356                        .htmlmediaelement
357                        .set_poster_frame(cx.no_gc(), Some(image)),
358                    None => warn!("Vector images are not yet supported in video poster"),
359                }
360                LoadBlocker::terminate(&self.load_blocker, cx);
361            },
362            ImageResponse::MetadataLoaded(..) => {},
363            // The image cache may have loaded a placeholder for an invalid poster url
364            ImageResponse::FailedToLoadOrDecode => {
365                self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
366                // A failed load should unblock the document load.
367                LoadBlocker::terminate(&self.load_blocker, cx);
368            },
369        }
370    }
371
372    /// <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
373    pub(crate) fn is_usable(&self) -> bool {
374        !matches!(
375            self.htmlmediaelement.get_ready_state(),
376            ReadyState::HaveNothing | ReadyState::HaveMetadata
377        )
378    }
379
380    pub(crate) fn origin_is_clean(&self) -> bool {
381        self.htmlmediaelement.origin_is_clean()
382    }
383
384    pub(crate) fn is_network_state_empty(&self) -> bool {
385        self.htmlmediaelement.network_state() == NetworkState::Empty
386    }
387}
388
389impl HTMLVideoElementMethods<crate::DomTypeHolder> for HTMLVideoElement {
390    // <https://html.spec.whatwg.org/multipage/#dom-video-width>
391    make_dimension_uint_getter!(Width, "width");
392
393    // <https://html.spec.whatwg.org/multipage/#dom-video-width>
394    make_dimension_uint_setter!(SetWidth, "width");
395
396    // <https://html.spec.whatwg.org/multipage/#dom-video-height>
397    make_dimension_uint_getter!(Height, "height");
398
399    // <https://html.spec.whatwg.org/multipage/#dom-video-height>
400    make_dimension_uint_setter!(SetHeight, "height");
401
402    /// <https://html.spec.whatwg.org/multipage/#dom-video-videowidth>
403    fn VideoWidth(&self) -> u32 {
404        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
405            return 0;
406        }
407        self.video_width.get().unwrap_or(0)
408    }
409
410    /// <https://html.spec.whatwg.org/multipage/#dom-video-videoheight>
411    fn VideoHeight(&self) -> u32 {
412        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
413            return 0;
414        }
415        self.video_height.get().unwrap_or(0)
416    }
417
418    // https://html.spec.whatwg.org/multipage/#dom-video-poster
419    make_url_getter!(Poster, "poster");
420
421    // https://html.spec.whatwg.org/multipage/#dom-video-poster
422    make_url_setter!(SetPoster, "poster");
423
424    // For testing purposes only. This is not an event from
425    // https://html.spec.whatwg.org/multipage/#dom-video-poster
426    event_handler!(postershown, GetOnpostershown, SetOnpostershown);
427}
428
429impl VirtualMethods for HTMLVideoElement {
430    fn super_type(&self) -> Option<&dyn VirtualMethods> {
431        Some(self.upcast::<HTMLMediaElement>() as &dyn VirtualMethods)
432    }
433
434    fn attribute_mutated(
435        &self,
436        cx: &mut JSContext,
437        attr: AttrRef<'_>,
438        mutation: AttributeMutation,
439    ) {
440        self.super_type()
441            .unwrap()
442            .attribute_mutated(cx, attr, mutation);
443
444        if attr.local_name() == &local_name!("poster") {
445            if let Some(new_value) = mutation.new_value(attr) {
446                self.update_poster_frame(Some(&new_value), cx)
447            } else {
448                self.update_poster_frame(None, cx)
449            }
450        };
451    }
452
453    fn attribute_affects_presentational_hints(&self, attr: AttrRef<'_>) -> bool {
454        match attr.local_name() {
455            &local_name!("width") | &local_name!("height") => true,
456            _ => self
457                .super_type()
458                .unwrap()
459                .attribute_affects_presentational_hints(attr),
460        }
461    }
462
463    fn parse_plain_attribute(&self, name: &LocalName, value: DOMString) -> AttrValue {
464        match name {
465            &local_name!("width") | &local_name!("height") => {
466                AttrValue::from_dimension(value.into())
467            },
468            _ => self
469                .super_type()
470                .unwrap()
471                .parse_plain_attribute(name, value),
472        }
473    }
474}
475
476struct PosterFrameFetchContext {
477    /// Reference to the script thread image cache.
478    image_cache: Arc<dyn ImageCache>,
479    /// The element that initiated the request.
480    elem: Trusted<HTMLVideoElement>,
481    /// The cache ID for this request.
482    id: PendingImageId,
483    /// True if this response is invalid and should be ignored.
484    cancelled: bool,
485    /// Url for the resource
486    url: ServoUrl,
487    /// A [`FetchCanceller`] for this request.
488    fetch_canceller: FetchCanceller,
489}
490
491impl FetchResponseListener for PosterFrameFetchContext {
492    fn process_request_body(&mut self, _: RequestId) {
493        self.fetch_canceller.ignore()
494    }
495
496    fn process_response(
497        &mut self,
498        _: &mut JSContext,
499        request_id: RequestId,
500        metadata: Result<FetchMetadata, NetworkError>,
501    ) {
502        self.image_cache.notify_pending_response(
503            self.id,
504            FetchResponseMsg::ProcessResponse(request_id, metadata.clone()),
505        );
506
507        let metadata = metadata.ok().map(|meta| match meta {
508            FetchMetadata::Unfiltered(m) => m,
509            FetchMetadata::Filtered { unsafe_, .. } => unsafe_,
510        });
511
512        let status_is_ok = metadata
513            .as_ref()
514            .is_none_or(|m| m.status.in_range(200..300));
515
516        if !status_is_ok {
517            self.cancelled = true;
518            self.fetch_canceller.abort();
519        }
520    }
521
522    fn process_response_chunk(&mut self, _: &mut JSContext, request_id: RequestId, payload: Bytes) {
523        if self.cancelled {
524            // An error was received previously, skip processing the payload.
525            return;
526        }
527
528        self.image_cache.notify_pending_response(
529            self.id,
530            FetchResponseMsg::ProcessResponseChunk(request_id, payload),
531        );
532    }
533
534    fn process_response_eof(
535        self,
536        cx: &mut JSContext,
537        request_id: RequestId,
538        response: Result<(), NetworkError>,
539        timing: ResourceFetchTiming,
540    ) {
541        self.image_cache.notify_pending_response(
542            self.id,
543            FetchResponseMsg::ProcessResponseEOF(request_id, response.clone(), timing.clone()),
544        );
545        network_listener::submit_timing(cx, &self, &response, &timing);
546    }
547
548    fn process_csp_violations(
549        &mut self,
550        cx: &mut js::context::JSContext,
551        _request_id: RequestId,
552        violations: Vec<Violation>,
553    ) {
554        let global = &self.resource_timing_global();
555        global.report_csp_violations(cx, violations, None, None);
556    }
557
558    fn process_content_length(&mut self, request_id: RequestId, size: usize) {
559        self.image_cache.notify_pending_response(
560            self.id,
561            FetchResponseMsg::ProcessContentLength(request_id, size),
562        );
563    }
564}
565
566impl ResourceTimingListener for PosterFrameFetchContext {
567    fn resource_timing_information(&self) -> (InitiatorType, ServoUrl) {
568        let initiator_type = InitiatorType::LocalName(
569            self.elem
570                .root()
571                .upcast::<Element>()
572                .local_name()
573                .to_string(),
574        );
575        (initiator_type, self.url.clone())
576    }
577
578    fn resource_timing_global(&self) -> DomRoot<GlobalScope> {
579        self.elem.root().owner_document().global()
580    }
581}
582
583impl PosterFrameFetchContext {
584    fn new(
585        elem: &HTMLVideoElement,
586        url: ServoUrl,
587        id: PendingImageId,
588        request_id: RequestId,
589        core_resource_thread: CoreResourceThread,
590    ) -> PosterFrameFetchContext {
591        let window = elem.owner_window();
592        PosterFrameFetchContext {
593            image_cache: window.image_cache(),
594            elem: Trusted::new(elem),
595            id,
596            cancelled: false,
597            url,
598            fetch_canceller: FetchCanceller::new(request_id, false, core_resource_thread),
599        }
600    }
601}
602
603impl LayoutDom<'_, HTMLVideoElement> {
604    #[expect(unsafe_code)]
605    pub(crate) fn data(self) -> HTMLMediaData {
606        let video = self.unsafe_get();
607
608        // Get the current frame being rendered.
609        let current_frame = video.htmlmediaelement.get_current_frame_to_present();
610
611        // This value represents the natural width and height of the video.
612        // It may exist even if there is no current frame (for example, after the
613        // metadata of the video is loaded).
614        let metadata = video
615            .get_video_width()
616            .zip(video.get_video_height())
617            .map(|(width, height)| MediaMetadata { width, height });
618
619        HTMLMediaData {
620            current_frame,
621            metadata,
622            poster_url: unsafe {
623                video
624                    .poster_url
625                    .borrow_for_layout()
626                    .clone()
627                    .map(|url| url.url())
628            },
629        }
630    }
631
632    pub(crate) fn get_width(self) -> LengthOrPercentageOrAuto {
633        self.upcast::<Element>()
634            .get_attr_for_layout(&ns!(), &local_name!("width"))
635            .map(AttrValue::as_dimension)
636            .cloned()
637            .unwrap_or(LengthOrPercentageOrAuto::Auto)
638    }
639
640    pub(crate) fn get_height(self) -> LengthOrPercentageOrAuto {
641        self.upcast::<Element>()
642            .get_attr_for_layout(&ns!(), &local_name!("height"))
643            .map(AttrValue::as_dimension)
644            .cloned()
645            .unwrap_or(LengthOrPercentageOrAuto::Auto)
646    }
647}
script/dom/html/embedded_content/htmlvideoelement.rs

script/dom/html/embedded_content/
htmlvideoelement.rs
