script/dom/html/embedded_content/
htmlvideoelement.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::cell::Cell;
6use std::rc::Rc;
7use std::sync::Arc;
8
9use bytes::Bytes;
10use dom_struct::dom_struct;
11use euclid::default::Size2D;
12use html5ever::{LocalName, Prefix, local_name, ns};
13use js::context::{JSContext, NoGC};
14use js::rust::HandleObject;
15use layout_api::{HTMLMediaData, MediaMetadata};
16use net_traits::blob_url_store::UrlWithBlobClaim;
17use net_traits::image_cache::{
18    ImageCache, ImageCacheResult, ImageLoadListener, ImageOrMetadataAvailable, ImageResponse,
19    PendingImageId,
20};
21use net_traits::request::{CredentialsMode, Destination, RequestBuilder, RequestId};
22use net_traits::{
23    CoreResourceThread, FetchMetadata, FetchResponseMsg, Metadata, NetworkError,
24    ResourceFetchTiming,
25};
26use pixels::{Snapshot, SnapshotAlphaMode, SnapshotPixelFormat};
27use script_bindings::cell::DomRefCell;
28use script_bindings::error::Fallible;
29use servo_media::player::video::VideoFrame;
30use servo_url::ServoUrl;
31use style::attr::{AttrValue, LengthOrPercentageOrAuto};
32
33use crate::dom::bindings::codegen::Bindings::HTMLVideoElementBinding::HTMLVideoElementMethods;
34use crate::dom::bindings::inheritance::Castable;
35use crate::dom::bindings::refcounted::Trusted;
36use crate::dom::bindings::reflector::DomGlobal;
37use crate::dom::bindings::root::{DomRoot, LayoutDom};
38use crate::dom::bindings::str::{DOMString, USVString};
39use crate::dom::csp::{GlobalCspReporting, Violation};
40use crate::dom::document::Document;
41use crate::dom::element::attributes::storage::AttrRef;
42use crate::dom::element::{AttributeMutation, Element};
43use crate::dom::globalscope::GlobalScope;
44use crate::dom::html::htmlmediaelement::{HTMLMediaElement, NetworkState, ReadyState};
45use crate::dom::node::virtualmethods::VirtualMethods;
46use crate::dom::node::{Node, NodeDamage, NodeTraits};
47use crate::dom::performance::performanceresourcetiming::InitiatorType;
48use crate::event_loop::document_loader::{LoadBlocker, LoadType};
49use crate::fetch::fetch::{FetchCanceller, RequestWithGlobalScope};
50use crate::fetch::network_listener::{self, FetchResponseListener, ResourceTimingListener};
51use crate::url::ensure_blob_referenced_by_url_is_kept_alive;
52
53#[dom_struct]
54pub(crate) struct HTMLVideoElement {
55    htmlmediaelement: HTMLMediaElement,
56    /// <https://html.spec.whatwg.org/multipage/#dom-video-videowidth>
57    video_width: Cell<Option<u32>>,
58    /// <https://html.spec.whatwg.org/multipage/#dom-video-videoheight>
59    video_height: Cell<Option<u32>>,
60    /// <https://html.spec.whatwg.org/multipage/#dom-video-poster>
61    #[no_trace]
62    poster_url: DomRefCell<Option<UrlWithBlobClaim>>,
63    /// Incremented whenever tasks associated with this element are cancelled.
64    generation_id: Cell<u32>,
65    /// Load event blocker. Will block the load event while the poster frame
66    /// is being fetched.
67    load_blocker: DomRefCell<Option<LoadBlocker>>,
68    /// A copy of the last frame
69    #[ignore_malloc_size_of = "VideoFrame"]
70    #[no_trace]
71    last_frame: DomRefCell<Option<VideoFrame>>,
72    #[cfg(feature = "webgpu")]
73    #[conditional_malloc_size_of]
74    /// Planar texture for WebGPU
75    planar_texture: DomRefCell<Option<Rc<crate::dom::gpuexternaltexture::PlanarTexture>>>,
76}
77
78impl HTMLVideoElement {
79    fn new_inherited(
80        local_name: LocalName,
81        prefix: Option<Prefix>,
82        document: &Document,
83    ) -> HTMLVideoElement {
84        HTMLVideoElement {
85            htmlmediaelement: HTMLMediaElement::new_inherited(local_name, prefix, document),
86            video_width: Cell::new(None),
87            video_height: Cell::new(None),
88            poster_url: Default::default(),
89            generation_id: Cell::new(0),
90            load_blocker: Default::default(),
91            last_frame: Default::default(),
92            #[cfg(feature = "webgpu")]
93            planar_texture: DomRefCell::new(None),
94        }
95    }
96
97    pub(crate) fn new(
98        cx: &mut JSContext,
99        local_name: LocalName,
100        prefix: Option<Prefix>,
101        document: &Document,
102        proto: Option<HandleObject>,
103    ) -> DomRoot<HTMLVideoElement> {
104        Node::reflect_weak_referenceable_node_with_proto(
105            cx,
106            Rc::new(HTMLVideoElement::new_inherited(
107                local_name, prefix, document,
108            )),
109            document,
110            proto,
111        )
112    }
113
114    pub(crate) fn get_video_width(&self) -> Option<u32> {
115        self.video_width.get()
116    }
117
118    pub(crate) fn get_video_height(&self) -> Option<u32> {
119        self.video_height.get()
120    }
121
122    pub(crate) fn set_natural_dimensions(
123        &self,
124        no_gc: &NoGC,
125        width: Option<u32>,
126        height: Option<u32>,
127    ) -> bool {
128        if self.video_width.get() == width && self.video_height.get() == height {
129            return false;
130        }
131
132        self.video_width.set(width);
133        self.video_height.set(height);
134
135        self.upcast::<Node>().dirty(no_gc, NodeDamage::Other);
136        true
137    }
138
139    /// Gets the copy of the video frame at the current playback position,
140    /// if that is available, or else (e.g. when the video is seeking or buffering)
141    /// its previous appearance, if any.
142    pub(crate) fn get_current_frame_data(&self) -> Option<Snapshot> {
143        let frame = self.htmlmediaelement.get_current_frame();
144        if frame.is_some() {
145            *self.last_frame.borrow_mut() = frame;
146        }
147
148        match self.last_frame.borrow().as_ref() {
149            Some(frame) => {
150                let size = Size2D::new(frame.get_width() as u32, frame.get_height() as u32);
151                if !frame.is_gl_texture() {
152                    let alpha_mode = SnapshotAlphaMode::Transparent {
153                        premultiplied: false,
154                    };
155
156                    Some(Snapshot::from_vec(
157                        size.cast(),
158                        SnapshotPixelFormat::BGRA,
159                        alpha_mode,
160                        frame.get_data().to_vec(),
161                    ))
162                } else {
163                    // XXX(victor): here we only have the GL texture ID.
164                    Some(Snapshot::cleared(size.cast()))
165                }
166            },
167            None => None,
168        }
169    }
170
171    #[cfg(feature = "webgpu")]
172    pub(crate) fn planar_video_for_webgpu(
173        &self,
174        device: &crate::dom::types::GPUDevice,
175    ) -> Fallible<(
176        Size2D<u32>,
177        Option<Rc<crate::dom::gpuexternaltexture::PlanarTexture>>,
178    )> {
179        use crate::dom::gpuexternaltexture::PlanarTexture;
180        // 1. If source is not origin-clean, throw a SecurityError and return.
181        if !self.origin_is_clean() {
182            return Err(script_bindings::error::Error::Security(Some(
183                "Source is not origin-clean".to_string(),
184            )));
185        }
186        // 2. Let usability be ? check the usability of the image argument(source).
187        if !self.is_usable() {
188            // 3. If usability is not good:
189            // Generate a validation error.
190            // Return an invalidated GPUExternalTexture.
191            Ok((Size2D::zero(), None))
192        } else {
193            // 4. Let data be the result of converting the current image contents of source into the color space descriptor.colorSpace with unpremultiplied alpha.
194            let mut planar_texture = self.planar_texture.borrow_mut();
195            match planar_texture.as_ref() {
196                Some(planar_texture) => {
197                    if planar_texture.is_expired() &&
198                        let Some(snapshot) = self.get_current_frame_data()
199                    {
200                        planar_texture.update(snapshot);
201                    }
202                },
203                None => {
204                    *planar_texture = self.get_current_frame_data().map(|snapshot| {
205                        Rc::new(PlanarTexture::new(device.channel(), device, snapshot))
206                    });
207                },
208            };
209            Ok((
210                planar_texture
211                    .as_ref()
212                    .map(|pt| pt.size())
213                    .unwrap_or_default(),
214                planar_texture.as_ref().cloned(),
215            ))
216        }
217    }
218
219    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
220    fn update_poster_frame(&self, cx: &mut JSContext, poster_attribute: Option<&str>) {
221        // Step 1. If there is an existing instance of this algorithm running
222        // for this video element, abort that instance of this algorithm without
223        // changing the poster frame.
224        self.generation_id.set(self.generation_id.get() + 1);
225
226        // Step 2. If the poster attribute's value is the empty string or
227        // if the attribute is absent, then there is no poster frame; return.
228        *self.poster_url.safe_borrow_mut(cx.no_gc()) = None;
229        let Some(url) = poster_attribute.filter(|poster_attribute| !poster_attribute.is_empty())
230        else {
231            self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
232            return;
233        };
234
235        // Step 3. Let url be the result of encoding-parsing a URL given
236        // the poster attribute's value, relative to the element's node
237        // document.
238        // Step 4. If url is failure, then return. There is no poster frame.
239        let global = self.owner_global();
240        let Ok(poster_url) = self
241            .owner_document()
242            .encoding_parse_a_url(url)
243            .map(|url| ensure_blob_referenced_by_url_is_kept_alive(&global, url))
244        else {
245            self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
246            return;
247        };
248        *self.poster_url.safe_borrow_mut(cx.no_gc()) = Some(poster_url.clone());
249
250        // We use the image cache for poster frames so we save as much
251        // network activity as possible.
252        let window = self.owner_window();
253        let image_cache = window.image_cache();
254        let cache_result = image_cache.get_cached_image_status(
255            poster_url.url(),
256            window.origin().immutable().clone(),
257            None,
258        );
259
260        let id = match cache_result {
261            ImageCacheResult::Available(ImageOrMetadataAvailable::ImageAvailable {
262                image,
263                url,
264                ..
265            }) => {
266                self.process_image_response(cx, ImageResponse::Loaded(image, url));
267                return;
268            },
269            ImageCacheResult::Available(ImageOrMetadataAvailable::MetadataAvailable(_, id)) => id,
270            ImageCacheResult::ReadyForRequest(id) => {
271                self.do_fetch_poster_frame(cx, poster_url, id);
272                id
273            },
274            ImageCacheResult::FailedToLoadOrDecode => {
275                self.process_image_response(cx, ImageResponse::FailedToLoadOrDecode);
276                return;
277            },
278            ImageCacheResult::Pending(id) => id,
279        };
280
281        let trusted_node = Trusted::new(self);
282        let generation = self.generation_id();
283        let callback = window.register_image_cache_listener(id, move |response, cx| {
284            let element = trusted_node.root();
285
286            // Ignore any image response for a previous request that has been discarded.
287            if generation != element.generation_id() {
288                return;
289            }
290            element.process_image_response(cx, response.response);
291        });
292
293        image_cache.add_listener(ImageLoadListener::new(callback, window.pipeline_id(), id));
294    }
295
296    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
297    fn do_fetch_poster_frame(
298        &self,
299        cx: &mut JSContext,
300        poster_url: UrlWithBlobClaim,
301        id: PendingImageId,
302    ) {
303        // Step 5. Let request be a new request whose URL is url, client is the element's node
304        // document's relevant settings object, destination is "image", initiator type is "video",
305        // credentials mode is "include", and whose use-URL-credentials flag is set.
306        let document = self.owner_document();
307        let global = self.owner_global();
308        let request = RequestBuilder::new(
309            Some(document.webview_id()),
310            poster_url.clone(),
311            global.get_referrer(),
312        )
313        .destination(Destination::Image)
314        .credentials_mode(CredentialsMode::Include)
315        .use_url_credentials(true)
316        .with_global_scope(&global);
317
318        // Step 6. Fetch request. This must delay the load event of the element's node document.
319        // This delay must be independent from the ones created by HTMLMediaElement during
320        // its media load algorithm, otherwise a code like
321        // <video poster="poster.png"></video>
322        // (which triggers no media load algorithm unless a explicit call to .load() is done)
323        // will block the document's load event forever.
324        let blocker = &self.load_blocker;
325        LoadBlocker::terminate(blocker, cx);
326        let document = self.owner_document();
327        *blocker.borrow_mut() = Some(LoadBlocker::new(
328            &document,
329            LoadType::Image(poster_url.url()),
330        ));
331
332        let context = PosterFrameFetchContext::new(
333            self,
334            poster_url.url(),
335            id,
336            request.id,
337            self.global().core_resource_thread(),
338        );
339        self.owner_document().fetch_background(request, context);
340    }
341
342    fn generation_id(&self) -> u32 {
343        self.generation_id.get()
344    }
345
346    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
347    fn process_image_response(&self, cx: &mut JSContext, response: ImageResponse) {
348        // Step 7. If an image is thus obtained, the poster frame is that image.
349        // Otherwise, there is no poster frame.
350        match response {
351            ImageResponse::Loaded(image, url) => {
352                debug!("Loaded poster image for video element: {:?}", url);
353                match image.as_raster_image() {
354                    Some(image) => self
355                        .htmlmediaelement
356                        .set_poster_frame(cx.no_gc(), Some(image)),
357                    None => warn!("Vector images are not yet supported in video poster"),
358                }
359                LoadBlocker::terminate(&self.load_blocker, cx);
360            },
361            ImageResponse::MetadataLoaded(..) => {},
362            // The image cache may have loaded a placeholder for an invalid poster url
363            ImageResponse::FailedToLoadOrDecode => {
364                self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
365                // A failed load should unblock the document load.
366                LoadBlocker::terminate(&self.load_blocker, cx);
367            },
368        }
369    }
370
371    /// <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
372    pub(crate) fn is_usable(&self) -> bool {
373        !matches!(
374            self.htmlmediaelement.get_ready_state(),
375            ReadyState::HaveNothing | ReadyState::HaveMetadata
376        )
377    }
378
379    pub(crate) fn origin_is_clean(&self) -> bool {
380        self.htmlmediaelement.origin_is_clean()
381    }
382
383    pub(crate) fn is_network_state_empty(&self) -> bool {
384        self.htmlmediaelement.network_state() == NetworkState::Empty
385    }
386}
387
388impl HTMLVideoElementMethods<crate::DomTypeHolder> for HTMLVideoElement {
389    // <https://html.spec.whatwg.org/multipage/#dom-video-width>
390    make_dimension_uint_getter!(Width, "width");
391
392    // <https://html.spec.whatwg.org/multipage/#dom-video-width>
393    make_dimension_uint_setter!(SetWidth, "width");
394
395    // <https://html.spec.whatwg.org/multipage/#dom-video-height>
396    make_dimension_uint_getter!(Height, "height");
397
398    // <https://html.spec.whatwg.org/multipage/#dom-video-height>
399    make_dimension_uint_setter!(SetHeight, "height");
400
401    /// <https://html.spec.whatwg.org/multipage/#dom-video-videowidth>
402    fn VideoWidth(&self) -> u32 {
403        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
404            return 0;
405        }
406        self.video_width.get().unwrap_or(0)
407    }
408
409    /// <https://html.spec.whatwg.org/multipage/#dom-video-videoheight>
410    fn VideoHeight(&self) -> u32 {
411        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
412            return 0;
413        }
414        self.video_height.get().unwrap_or(0)
415    }
416
417    // https://html.spec.whatwg.org/multipage/#dom-video-poster
418    make_url_getter!(Poster, "poster");
419
420    // https://html.spec.whatwg.org/multipage/#dom-video-poster
421    make_url_setter!(SetPoster, "poster");
422
423    // For testing purposes only. This is not an event from
424    // https://html.spec.whatwg.org/multipage/#dom-video-poster
425    event_handler!(postershown, GetOnpostershown, SetOnpostershown);
426}
427
428impl VirtualMethods for HTMLVideoElement {
429    fn super_type(&self) -> Option<&dyn VirtualMethods> {
430        Some(self.upcast::<HTMLMediaElement>() as &dyn VirtualMethods)
431    }
432
433    fn attribute_mutated(
434        &self,
435        cx: &mut JSContext,
436        attr: AttrRef<'_>,
437        mutation: AttributeMutation,
438    ) {
439        self.super_type()
440            .unwrap()
441            .attribute_mutated(cx, attr, mutation);
442
443        if attr.local_name() == &local_name!("poster") {
444            if let Some(new_value) = mutation.new_value(attr) {
445                self.update_poster_frame(cx, Some(&new_value))
446            } else {
447                self.update_poster_frame(cx, None)
448            }
449        };
450    }
451
452    fn attribute_affects_presentational_hints(&self, attr: AttrRef<'_>) -> bool {
453        match attr.local_name() {
454            &local_name!("width") | &local_name!("height") => true,
455            _ => self
456                .super_type()
457                .unwrap()
458                .attribute_affects_presentational_hints(attr),
459        }
460    }
461
462    fn parse_plain_attribute(&self, name: &LocalName, value: DOMString) -> AttrValue {
463        match name {
464            &local_name!("width") | &local_name!("height") => {
465                AttrValue::from_dimension(value.into())
466            },
467            _ => self
468                .super_type()
469                .unwrap()
470                .parse_plain_attribute(name, value),
471        }
472    }
473}
474
475struct PosterFrameFetchContext {
476    /// Reference to the script thread image cache.
477    image_cache: Arc<dyn ImageCache>,
478    /// The element that initiated the request.
479    elem: Trusted<HTMLVideoElement>,
480    /// The cache ID for this request.
481    id: PendingImageId,
482    /// True if this response is invalid and should be ignored.
483    cancelled: bool,
484    /// Url for the resource
485    url: ServoUrl,
486    /// A [`FetchCanceller`] for this request.
487    fetch_canceller: FetchCanceller,
488}
489
490impl FetchResponseListener for PosterFrameFetchContext {
491    fn process_request_body(&mut self, _: RequestId) {
492        self.fetch_canceller.ignore()
493    }
494
495    fn process_response(
496        &mut self,
497        _: &mut JSContext,
498        request_id: RequestId,
499        metadata: Result<FetchMetadata, NetworkError>,
500    ) {
501        self.image_cache.notify_pending_response(
502            self.id,
503            FetchResponseMsg::ProcessResponse(request_id, metadata.clone()),
504        );
505
506        let metadata: Option<Metadata> = metadata.ok().map(Into::into);
507
508        let status_is_ok = metadata
509            .as_ref()
510            .is_none_or(|m| m.status.in_range(200..300));
511
512        if !status_is_ok {
513            self.cancelled = true;
514            self.fetch_canceller.abort();
515        }
516    }
517
518    fn process_response_chunk(&mut self, _: &mut JSContext, request_id: RequestId, payload: Bytes) {
519        if self.cancelled {
520            // An error was received previously, skip processing the payload.
521            return;
522        }
523
524        self.image_cache.notify_pending_response(
525            self.id,
526            FetchResponseMsg::ProcessResponseChunk(request_id, payload),
527        );
528    }
529
530    fn process_response_eof(
531        self,
532        cx: &mut JSContext,
533        request_id: RequestId,
534        response: Result<(), NetworkError>,
535        timing: ResourceFetchTiming,
536    ) {
537        self.image_cache.notify_pending_response(
538            self.id,
539            FetchResponseMsg::ProcessResponseEOF(request_id, response.clone(), timing.clone()),
540        );
541        network_listener::submit_timing(cx, &self, &response, &timing);
542    }
543
544    fn process_csp_violations(
545        &mut self,
546        cx: &mut JSContext,
547        _request_id: RequestId,
548        violations: Vec<Violation>,
549    ) {
550        let global = &self.resource_timing_global();
551        global.report_csp_violations(cx, violations, None, None);
552    }
553
554    fn process_content_length(&mut self, request_id: RequestId, size: usize) {
555        self.image_cache.notify_pending_response(
556            self.id,
557            FetchResponseMsg::ProcessContentLength(request_id, size),
558        );
559    }
560}
561
562impl ResourceTimingListener for PosterFrameFetchContext {
563    fn resource_timing_information(&self) -> (InitiatorType, ServoUrl) {
564        let initiator_type = InitiatorType::LocalName(
565            self.elem
566                .root()
567                .upcast::<Element>()
568                .local_name()
569                .to_string(),
570        );
571        (initiator_type, self.url.clone())
572    }
573
574    fn resource_timing_global(&self) -> DomRoot<GlobalScope> {
575        self.elem.root().owner_document().global()
576    }
577}
578
579impl PosterFrameFetchContext {
580    fn new(
581        elem: &HTMLVideoElement,
582        url: ServoUrl,
583        id: PendingImageId,
584        request_id: RequestId,
585        core_resource_thread: CoreResourceThread,
586    ) -> PosterFrameFetchContext {
587        let window = elem.owner_window();
588        PosterFrameFetchContext {
589            image_cache: window.image_cache(),
590            elem: Trusted::new(elem),
591            id,
592            cancelled: false,
593            url,
594            fetch_canceller: FetchCanceller::new(request_id, false, core_resource_thread),
595        }
596    }
597}
598
599impl LayoutDom<'_, HTMLVideoElement> {
600    #[expect(unsafe_code)]
601    pub(crate) fn data(self) -> HTMLMediaData {
602        let video = self.unsafe_get();
603
604        // Get the current frame being rendered.
605        let current_frame = video.htmlmediaelement.get_current_frame_to_present();
606
607        // This value represents the natural width and height of the video.
608        // It may exist even if there is no current frame (for example, after the
609        // metadata of the video is loaded).
610        let metadata = video
611            .get_video_width()
612            .zip(video.get_video_height())
613            .map(|(width, height)| MediaMetadata { width, height });
614
615        HTMLMediaData {
616            current_frame,
617            metadata,
618            poster_url: unsafe {
619                video
620                    .poster_url
621                    .borrow_for_layout()
622                    .clone()
623                    .map(|url| url.url())
624            },
625        }
626    }
627
628    pub(crate) fn width(self) -> LengthOrPercentageOrAuto {
629        self.upcast::<Element>()
630            .get_attr_for_layout(&ns!(), &local_name!("width"))
631            .map(AttrValue::as_dimension)
632            .cloned()
633            .unwrap_or(LengthOrPercentageOrAuto::Auto)
634    }
635
636    pub(crate) fn height(self) -> LengthOrPercentageOrAuto {
637        self.upcast::<Element>()
638            .get_attr_for_layout(&ns!(), &local_name!("height"))
639            .map(AttrValue::as_dimension)
640            .cloned()
641            .unwrap_or(LengthOrPercentageOrAuto::Auto)
642    }
643}
script/dom/html/embedded_content/htmlvideoelement.rs

script/dom/html/embedded_content/
htmlvideoelement.rs
