script/dom/html/embedded_content/
htmlvideoelement.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::cell::Cell;
6use std::rc::Rc;
7use std::sync::Arc;
8
9use bytes::Bytes;
10use dom_struct::dom_struct;
11use euclid::default::Size2D;
12use html5ever::{LocalName, Prefix, local_name, ns};
13use js::context::{JSContext, NoGC};
14use js::rust::HandleObject;
15use layout_api::{HTMLMediaData, MediaMetadata};
16use net_traits::blob_url_store::UrlWithBlobClaim;
17use net_traits::image_cache::{
18    ImageCache, ImageCacheResult, ImageLoadListener, ImageOrMetadataAvailable, ImageResponse,
19    PendingImageId,
20};
21use net_traits::request::{CredentialsMode, Destination, RequestBuilder, RequestId};
22use net_traits::{
23    CoreResourceThread, FetchMetadata, FetchResponseMsg, NetworkError, ResourceFetchTiming,
24};
25use pixels::{Snapshot, SnapshotAlphaMode, SnapshotPixelFormat};
26use script_bindings::cell::DomRefCell;
27use script_bindings::error::Fallible;
28use servo_media::player::video::VideoFrame;
29use servo_url::ServoUrl;
30use style::attr::{AttrValue, LengthOrPercentageOrAuto};
31
32use crate::dom::bindings::codegen::Bindings::HTMLVideoElementBinding::HTMLVideoElementMethods;
33use crate::dom::bindings::inheritance::Castable;
34use crate::dom::bindings::refcounted::Trusted;
35use crate::dom::bindings::reflector::DomGlobal;
36use crate::dom::bindings::root::{DomRoot, LayoutDom};
37use crate::dom::bindings::str::{DOMString, USVString};
38use crate::dom::csp::{GlobalCspReporting, Violation};
39use crate::dom::document::Document;
40use crate::dom::element::attributes::storage::AttrRef;
41use crate::dom::element::{AttributeMutation, Element};
42use crate::dom::globalscope::GlobalScope;
43use crate::dom::html::htmlmediaelement::{HTMLMediaElement, NetworkState, ReadyState};
44use crate::dom::node::virtualmethods::VirtualMethods;
45use crate::dom::node::{Node, NodeDamage, NodeTraits};
46use crate::dom::performance::performanceresourcetiming::InitiatorType;
47use crate::event_loop::document_loader::{LoadBlocker, LoadType};
48use crate::fetch::fetch::{FetchCanceller, RequestWithGlobalScope};
49use crate::fetch::network_listener::{self, FetchResponseListener, ResourceTimingListener};
50use crate::url::ensure_blob_referenced_by_url_is_kept_alive;
51
52#[dom_struct]
53pub(crate) struct HTMLVideoElement {
54    htmlmediaelement: HTMLMediaElement,
55    /// <https://html.spec.whatwg.org/multipage/#dom-video-videowidth>
56    video_width: Cell<Option<u32>>,
57    /// <https://html.spec.whatwg.org/multipage/#dom-video-videoheight>
58    video_height: Cell<Option<u32>>,
59    /// <https://html.spec.whatwg.org/multipage/#dom-video-poster>
60    #[no_trace]
61    poster_url: DomRefCell<Option<UrlWithBlobClaim>>,
62    /// Incremented whenever tasks associated with this element are cancelled.
63    generation_id: Cell<u32>,
64    /// Load event blocker. Will block the load event while the poster frame
65    /// is being fetched.
66    load_blocker: DomRefCell<Option<LoadBlocker>>,
67    /// A copy of the last frame
68    #[ignore_malloc_size_of = "VideoFrame"]
69    #[no_trace]
70    last_frame: DomRefCell<Option<VideoFrame>>,
71    #[cfg(feature = "webgpu")]
72    #[conditional_malloc_size_of]
73    /// Planar texture for WebGPU
74    planar_texture: DomRefCell<Option<Rc<crate::dom::gpuexternaltexture::PlanarTexture>>>,
75}
76
77impl HTMLVideoElement {
78    fn new_inherited(
79        local_name: LocalName,
80        prefix: Option<Prefix>,
81        document: &Document,
82    ) -> HTMLVideoElement {
83        HTMLVideoElement {
84            htmlmediaelement: HTMLMediaElement::new_inherited(local_name, prefix, document),
85            video_width: Cell::new(None),
86            video_height: Cell::new(None),
87            poster_url: Default::default(),
88            generation_id: Cell::new(0),
89            load_blocker: Default::default(),
90            last_frame: Default::default(),
91            #[cfg(feature = "webgpu")]
92            planar_texture: DomRefCell::new(None),
93        }
94    }
95
96    pub(crate) fn new(
97        cx: &mut JSContext,
98        local_name: LocalName,
99        prefix: Option<Prefix>,
100        document: &Document,
101        proto: Option<HandleObject>,
102    ) -> DomRoot<HTMLVideoElement> {
103        Node::reflect_weak_referenceable_node_with_proto(
104            cx,
105            Rc::new(HTMLVideoElement::new_inherited(
106                local_name, prefix, document,
107            )),
108            document,
109            proto,
110        )
111    }
112
113    pub(crate) fn get_video_width(&self) -> Option<u32> {
114        self.video_width.get()
115    }
116
117    pub(crate) fn get_video_height(&self) -> Option<u32> {
118        self.video_height.get()
119    }
120
121    pub(crate) fn set_natural_dimensions(
122        &self,
123        no_gc: &NoGC,
124        width: Option<u32>,
125        height: Option<u32>,
126    ) -> bool {
127        if self.video_width.get() == width && self.video_height.get() == height {
128            return false;
129        }
130
131        self.video_width.set(width);
132        self.video_height.set(height);
133
134        self.upcast::<Node>().dirty(no_gc, NodeDamage::Other);
135        true
136    }
137
138    /// Gets the copy of the video frame at the current playback position,
139    /// if that is available, or else (e.g. when the video is seeking or buffering)
140    /// its previous appearance, if any.
141    pub(crate) fn get_current_frame_data(&self) -> Option<Snapshot> {
142        let frame = self.htmlmediaelement.get_current_frame();
143        if frame.is_some() {
144            *self.last_frame.borrow_mut() = frame;
145        }
146
147        match self.last_frame.borrow().as_ref() {
148            Some(frame) => {
149                let size = Size2D::new(frame.get_width() as u32, frame.get_height() as u32);
150                if !frame.is_gl_texture() {
151                    let alpha_mode = SnapshotAlphaMode::Transparent {
152                        premultiplied: false,
153                    };
154
155                    Some(Snapshot::from_vec(
156                        size.cast(),
157                        SnapshotPixelFormat::BGRA,
158                        alpha_mode,
159                        frame.get_data().to_vec(),
160                    ))
161                } else {
162                    // XXX(victor): here we only have the GL texture ID.
163                    Some(Snapshot::cleared(size.cast()))
164                }
165            },
166            None => None,
167        }
168    }
169
170    #[cfg(feature = "webgpu")]
171    pub(crate) fn planar_video_for_webgpu(
172        &self,
173        device: &crate::dom::types::GPUDevice,
174    ) -> Fallible<(
175        Size2D<u32>,
176        Option<Rc<crate::dom::gpuexternaltexture::PlanarTexture>>,
177    )> {
178        use crate::dom::gpuexternaltexture::PlanarTexture;
179        // 1. If source is not origin-clean, throw a SecurityError and return.
180        if !self.origin_is_clean() {
181            return Err(script_bindings::error::Error::Security(Some(
182                "Source is not origin-clean".to_string(),
183            )));
184        }
185        // 2. Let usability be ? check the usability of the image argument(source).
186        if !self.is_usable() {
187            // 3. If usability is not good:
188            // Generate a validation error.
189            // Return an invalidated GPUExternalTexture.
190            Ok((Size2D::zero(), None))
191        } else {
192            // 4. Let data be the result of converting the current image contents of source into the color space descriptor.colorSpace with unpremultiplied alpha.
193            let mut planar_texture = self.planar_texture.borrow_mut();
194            match planar_texture.as_ref() {
195                Some(planar_texture) => {
196                    if planar_texture.is_expired() &&
197                        let Some(snapshot) = self.get_current_frame_data()
198                    {
199                        planar_texture.update(snapshot);
200                    }
201                },
202                None => {
203                    *planar_texture = self.get_current_frame_data().map(|snapshot| {
204                        Rc::new(PlanarTexture::new(device.channel(), device, snapshot))
205                    });
206                },
207            };
208            Ok((
209                planar_texture
210                    .as_ref()
211                    .map(|pt| pt.size())
212                    .unwrap_or_default(),
213                planar_texture.as_ref().cloned(),
214            ))
215        }
216    }
217
218    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
219    fn update_poster_frame(&self, poster_attribute: Option<&str>, cx: &mut JSContext) {
220        // Step 1. If there is an existing instance of this algorithm running
221        // for this video element, abort that instance of this algorithm without
222        // changing the poster frame.
223        self.generation_id.set(self.generation_id.get() + 1);
224
225        // Step 2. If the poster attribute's value is the empty string or
226        // if the attribute is absent, then there is no poster frame; return.
227        *self.poster_url.safe_borrow_mut(cx.no_gc()) = None;
228        let Some(url) = poster_attribute.filter(|poster_attribute| !poster_attribute.is_empty())
229        else {
230            self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
231            return;
232        };
233
234        // Step 3. Let url be the result of encoding-parsing a URL given
235        // the poster attribute's value, relative to the element's node
236        // document.
237        // Step 4. If url is failure, then return. There is no poster frame.
238        let global = self.owner_global();
239        let Ok(poster_url) = self
240            .owner_document()
241            .encoding_parse_a_url(url)
242            .map(|url| ensure_blob_referenced_by_url_is_kept_alive(&global, url))
243        else {
244            self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
245            return;
246        };
247        *self.poster_url.safe_borrow_mut(cx.no_gc()) = Some(poster_url.clone());
248
249        // We use the image cache for poster frames so we save as much
250        // network activity as possible.
251        let window = self.owner_window();
252        let image_cache = window.image_cache();
253        let cache_result = image_cache.get_cached_image_status(
254            poster_url.url(),
255            window.origin().immutable().clone(),
256            None,
257        );
258
259        let id = match cache_result {
260            ImageCacheResult::Available(ImageOrMetadataAvailable::ImageAvailable {
261                image,
262                url,
263                ..
264            }) => {
265                self.process_image_response(ImageResponse::Loaded(image, url), cx);
266                return;
267            },
268            ImageCacheResult::Available(ImageOrMetadataAvailable::MetadataAvailable(_, id)) => id,
269            ImageCacheResult::ReadyForRequest(id) => {
270                self.do_fetch_poster_frame(poster_url, id, cx);
271                id
272            },
273            ImageCacheResult::FailedToLoadOrDecode => {
274                self.process_image_response(ImageResponse::FailedToLoadOrDecode, cx);
275                return;
276            },
277            ImageCacheResult::Pending(id) => id,
278        };
279
280        let trusted_node = Trusted::new(self);
281        let generation = self.generation_id();
282        let callback = window.register_image_cache_listener(id, move |response, cx| {
283            let element = trusted_node.root();
284
285            // Ignore any image response for a previous request that has been discarded.
286            if generation != element.generation_id() {
287                return;
288            }
289            element.process_image_response(response.response, cx);
290        });
291
292        image_cache.add_listener(ImageLoadListener::new(callback, window.pipeline_id(), id));
293    }
294
295    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
296    fn do_fetch_poster_frame(
297        &self,
298        poster_url: UrlWithBlobClaim,
299        id: PendingImageId,
300        cx: &mut JSContext,
301    ) {
302        // Step 5. Let request be a new request whose URL is url, client is the element's node
303        // document's relevant settings object, destination is "image", initiator type is "video",
304        // credentials mode is "include", and whose use-URL-credentials flag is set.
305        let document = self.owner_document();
306        let global = self.owner_global();
307        let request = RequestBuilder::new(
308            Some(document.webview_id()),
309            poster_url.clone(),
310            global.get_referrer(),
311        )
312        .destination(Destination::Image)
313        .credentials_mode(CredentialsMode::Include)
314        .use_url_credentials(true)
315        .with_global_scope(&global);
316
317        // Step 6. Fetch request. This must delay the load event of the element's node document.
318        // This delay must be independent from the ones created by HTMLMediaElement during
319        // its media load algorithm, otherwise a code like
320        // <video poster="poster.png"></video>
321        // (which triggers no media load algorithm unless a explicit call to .load() is done)
322        // will block the document's load event forever.
323        let blocker = &self.load_blocker;
324        LoadBlocker::terminate(blocker, cx);
325        let document = self.owner_document();
326        *blocker.borrow_mut() = Some(LoadBlocker::new(
327            &document,
328            LoadType::Image(poster_url.url()),
329        ));
330
331        let context = PosterFrameFetchContext::new(
332            self,
333            poster_url.url(),
334            id,
335            request.id,
336            self.global().core_resource_thread(),
337        );
338        self.owner_document().fetch_background(request, context);
339    }
340
341    fn generation_id(&self) -> u32 {
342        self.generation_id.get()
343    }
344
345    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
346    fn process_image_response(&self, response: ImageResponse, cx: &mut JSContext) {
347        // Step 7. If an image is thus obtained, the poster frame is that image.
348        // Otherwise, there is no poster frame.
349        match response {
350            ImageResponse::Loaded(image, url) => {
351                debug!("Loaded poster image for video element: {:?}", url);
352                match image.as_raster_image() {
353                    Some(image) => self
354                        .htmlmediaelement
355                        .set_poster_frame(cx.no_gc(), Some(image)),
356                    None => warn!("Vector images are not yet supported in video poster"),
357                }
358                LoadBlocker::terminate(&self.load_blocker, cx);
359            },
360            ImageResponse::MetadataLoaded(..) => {},
361            // The image cache may have loaded a placeholder for an invalid poster url
362            ImageResponse::FailedToLoadOrDecode => {
363                self.htmlmediaelement.set_poster_frame(cx.no_gc(), None);
364                // A failed load should unblock the document load.
365                LoadBlocker::terminate(&self.load_blocker, cx);
366            },
367        }
368    }
369
370    /// <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
371    pub(crate) fn is_usable(&self) -> bool {
372        !matches!(
373            self.htmlmediaelement.get_ready_state(),
374            ReadyState::HaveNothing | ReadyState::HaveMetadata
375        )
376    }
377
378    pub(crate) fn origin_is_clean(&self) -> bool {
379        self.htmlmediaelement.origin_is_clean()
380    }
381
382    pub(crate) fn is_network_state_empty(&self) -> bool {
383        self.htmlmediaelement.network_state() == NetworkState::Empty
384    }
385}
386
387impl HTMLVideoElementMethods<crate::DomTypeHolder> for HTMLVideoElement {
388    // <https://html.spec.whatwg.org/multipage/#dom-video-width>
389    make_dimension_uint_getter!(Width, "width");
390
391    // <https://html.spec.whatwg.org/multipage/#dom-video-width>
392    make_dimension_uint_setter!(SetWidth, "width");
393
394    // <https://html.spec.whatwg.org/multipage/#dom-video-height>
395    make_dimension_uint_getter!(Height, "height");
396
397    // <https://html.spec.whatwg.org/multipage/#dom-video-height>
398    make_dimension_uint_setter!(SetHeight, "height");
399
400    /// <https://html.spec.whatwg.org/multipage/#dom-video-videowidth>
401    fn VideoWidth(&self) -> u32 {
402        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
403            return 0;
404        }
405        self.video_width.get().unwrap_or(0)
406    }
407
408    /// <https://html.spec.whatwg.org/multipage/#dom-video-videoheight>
409    fn VideoHeight(&self) -> u32 {
410        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
411            return 0;
412        }
413        self.video_height.get().unwrap_or(0)
414    }
415
416    // https://html.spec.whatwg.org/multipage/#dom-video-poster
417    make_url_getter!(Poster, "poster");
418
419    // https://html.spec.whatwg.org/multipage/#dom-video-poster
420    make_url_setter!(SetPoster, "poster");
421
422    // For testing purposes only. This is not an event from
423    // https://html.spec.whatwg.org/multipage/#dom-video-poster
424    event_handler!(postershown, GetOnpostershown, SetOnpostershown);
425}
426
427impl VirtualMethods for HTMLVideoElement {
428    fn super_type(&self) -> Option<&dyn VirtualMethods> {
429        Some(self.upcast::<HTMLMediaElement>() as &dyn VirtualMethods)
430    }
431
432    fn attribute_mutated(
433        &self,
434        cx: &mut JSContext,
435        attr: AttrRef<'_>,
436        mutation: AttributeMutation,
437    ) {
438        self.super_type()
439            .unwrap()
440            .attribute_mutated(cx, attr, mutation);
441
442        if attr.local_name() == &local_name!("poster") {
443            if let Some(new_value) = mutation.new_value(attr) {
444                self.update_poster_frame(Some(&new_value), cx)
445            } else {
446                self.update_poster_frame(None, cx)
447            }
448        };
449    }
450
451    fn attribute_affects_presentational_hints(&self, attr: AttrRef<'_>) -> bool {
452        match attr.local_name() {
453            &local_name!("width") | &local_name!("height") => true,
454            _ => self
455                .super_type()
456                .unwrap()
457                .attribute_affects_presentational_hints(attr),
458        }
459    }
460
461    fn parse_plain_attribute(&self, name: &LocalName, value: DOMString) -> AttrValue {
462        match name {
463            &local_name!("width") | &local_name!("height") => {
464                AttrValue::from_dimension(value.into())
465            },
466            _ => self
467                .super_type()
468                .unwrap()
469                .parse_plain_attribute(name, value),
470        }
471    }
472}
473
474struct PosterFrameFetchContext {
475    /// Reference to the script thread image cache.
476    image_cache: Arc<dyn ImageCache>,
477    /// The element that initiated the request.
478    elem: Trusted<HTMLVideoElement>,
479    /// The cache ID for this request.
480    id: PendingImageId,
481    /// True if this response is invalid and should be ignored.
482    cancelled: bool,
483    /// Url for the resource
484    url: ServoUrl,
485    /// A [`FetchCanceller`] for this request.
486    fetch_canceller: FetchCanceller,
487}
488
489impl FetchResponseListener for PosterFrameFetchContext {
490    fn process_request_body(&mut self, _: RequestId) {
491        self.fetch_canceller.ignore()
492    }
493
494    fn process_response(
495        &mut self,
496        _: &mut JSContext,
497        request_id: RequestId,
498        metadata: Result<FetchMetadata, NetworkError>,
499    ) {
500        self.image_cache.notify_pending_response(
501            self.id,
502            FetchResponseMsg::ProcessResponse(request_id, metadata.clone()),
503        );
504
505        let metadata = metadata.ok().map(|meta| match meta {
506            FetchMetadata::Unfiltered(m) => m,
507            FetchMetadata::Filtered { unsafe_, .. } => unsafe_,
508        });
509
510        let status_is_ok = metadata
511            .as_ref()
512            .is_none_or(|m| m.status.in_range(200..300));
513
514        if !status_is_ok {
515            self.cancelled = true;
516            self.fetch_canceller.abort();
517        }
518    }
519
520    fn process_response_chunk(&mut self, _: &mut JSContext, request_id: RequestId, payload: Bytes) {
521        if self.cancelled {
522            // An error was received previously, skip processing the payload.
523            return;
524        }
525
526        self.image_cache.notify_pending_response(
527            self.id,
528            FetchResponseMsg::ProcessResponseChunk(request_id, payload),
529        );
530    }
531
532    fn process_response_eof(
533        self,
534        cx: &mut JSContext,
535        request_id: RequestId,
536        response: Result<(), NetworkError>,
537        timing: ResourceFetchTiming,
538    ) {
539        self.image_cache.notify_pending_response(
540            self.id,
541            FetchResponseMsg::ProcessResponseEOF(request_id, response.clone(), timing.clone()),
542        );
543        network_listener::submit_timing(cx, &self, &response, &timing);
544    }
545
546    fn process_csp_violations(
547        &mut self,
548        cx: &mut js::context::JSContext,
549        _request_id: RequestId,
550        violations: Vec<Violation>,
551    ) {
552        let global = &self.resource_timing_global();
553        global.report_csp_violations(cx, violations, None, None);
554    }
555
556    fn process_content_length(&mut self, request_id: RequestId, size: usize) {
557        self.image_cache.notify_pending_response(
558            self.id,
559            FetchResponseMsg::ProcessContentLength(request_id, size),
560        );
561    }
562}
563
564impl ResourceTimingListener for PosterFrameFetchContext {
565    fn resource_timing_information(&self) -> (InitiatorType, ServoUrl) {
566        let initiator_type = InitiatorType::LocalName(
567            self.elem
568                .root()
569                .upcast::<Element>()
570                .local_name()
571                .to_string(),
572        );
573        (initiator_type, self.url.clone())
574    }
575
576    fn resource_timing_global(&self) -> DomRoot<GlobalScope> {
577        self.elem.root().owner_document().global()
578    }
579}
580
581impl PosterFrameFetchContext {
582    fn new(
583        elem: &HTMLVideoElement,
584        url: ServoUrl,
585        id: PendingImageId,
586        request_id: RequestId,
587        core_resource_thread: CoreResourceThread,
588    ) -> PosterFrameFetchContext {
589        let window = elem.owner_window();
590        PosterFrameFetchContext {
591            image_cache: window.image_cache(),
592            elem: Trusted::new(elem),
593            id,
594            cancelled: false,
595            url,
596            fetch_canceller: FetchCanceller::new(request_id, false, core_resource_thread),
597        }
598    }
599}
600
601impl LayoutDom<'_, HTMLVideoElement> {
602    #[expect(unsafe_code)]
603    pub(crate) fn data(self) -> HTMLMediaData {
604        let video = self.unsafe_get();
605
606        // Get the current frame being rendered.
607        let current_frame = video.htmlmediaelement.get_current_frame_to_present();
608
609        // This value represents the natural width and height of the video.
610        // It may exist even if there is no current frame (for example, after the
611        // metadata of the video is loaded).
612        let metadata = video
613            .get_video_width()
614            .zip(video.get_video_height())
615            .map(|(width, height)| MediaMetadata { width, height });
616
617        HTMLMediaData {
618            current_frame,
619            metadata,
620            poster_url: unsafe {
621                video
622                    .poster_url
623                    .borrow_for_layout()
624                    .clone()
625                    .map(|url| url.url())
626            },
627        }
628    }
629
630    pub(crate) fn get_width(self) -> LengthOrPercentageOrAuto {
631        self.upcast::<Element>()
632            .get_attr_for_layout(&ns!(), &local_name!("width"))
633            .map(AttrValue::as_dimension)
634            .cloned()
635            .unwrap_or(LengthOrPercentageOrAuto::Auto)
636    }
637
638    pub(crate) fn get_height(self) -> LengthOrPercentageOrAuto {
639        self.upcast::<Element>()
640            .get_attr_for_layout(&ns!(), &local_name!("height"))
641            .map(AttrValue::as_dimension)
642            .cloned()
643            .unwrap_or(LengthOrPercentageOrAuto::Auto)
644    }
645}
script/dom/html/embedded_content/htmlvideoelement.rs

script/dom/html/embedded_content/
htmlvideoelement.rs
