script/dom/html/
htmlvideoelement.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::cell::Cell;
6use std::rc::Rc;
7use std::sync::Arc;
8
9use dom_struct::dom_struct;
10use euclid::default::Size2D;
11use html5ever::{LocalName, Prefix, local_name, ns};
12use js::context::JSContext;
13use js::rust::HandleObject;
14use layout_api::{HTMLMediaData, MediaMetadata};
15use net_traits::blob_url_store::UrlWithBlobClaim;
16use net_traits::image_cache::{
17    ImageCache, ImageCacheResult, ImageLoadListener, ImageOrMetadataAvailable, ImageResponse,
18    PendingImageId,
19};
20use net_traits::request::{CredentialsMode, Destination, RequestBuilder, RequestId};
21use net_traits::{
22    CoreResourceThread, FetchMetadata, FetchResponseMsg, NetworkError, ResourceFetchTiming,
23};
24use pixels::{Snapshot, SnapshotAlphaMode, SnapshotPixelFormat};
25use script_bindings::cell::DomRefCell;
26use script_bindings::error::Fallible;
27use servo_media::player::video::VideoFrame;
28use servo_url::ServoUrl;
29use style::attr::{AttrValue, LengthOrPercentageOrAuto};
30
31use crate::document_loader::{LoadBlocker, LoadType};
32use crate::dom::bindings::codegen::Bindings::HTMLVideoElementBinding::HTMLVideoElementMethods;
33use crate::dom::bindings::inheritance::Castable;
34use crate::dom::bindings::refcounted::Trusted;
35use crate::dom::bindings::reflector::DomGlobal;
36use crate::dom::bindings::root::{DomRoot, LayoutDom};
37use crate::dom::bindings::str::{DOMString, USVString};
38use crate::dom::csp::{GlobalCspReporting, Violation};
39use crate::dom::document::Document;
40use crate::dom::element::attributes::storage::AttrRef;
41use crate::dom::element::{AttributeMutation, Element};
42use crate::dom::globalscope::GlobalScope;
43use crate::dom::html::htmlmediaelement::{HTMLMediaElement, NetworkState, ReadyState};
44use crate::dom::node::virtualmethods::VirtualMethods;
45use crate::dom::node::{Node, NodeDamage, NodeTraits};
46use crate::dom::performance::performanceresourcetiming::InitiatorType;
47use crate::fetch::{FetchCanceller, RequestWithGlobalScope};
48use crate::network_listener::{self, FetchResponseListener, ResourceTimingListener};
49use crate::url::ensure_blob_referenced_by_url_is_kept_alive;
50
51#[dom_struct]
52pub(crate) struct HTMLVideoElement {
53    htmlmediaelement: HTMLMediaElement,
54    /// <https://html.spec.whatwg.org/multipage/#dom-video-videowidth>
55    video_width: Cell<Option<u32>>,
56    /// <https://html.spec.whatwg.org/multipage/#dom-video-videoheight>
57    video_height: Cell<Option<u32>>,
58    /// Incremented whenever tasks associated with this element are cancelled.
59    generation_id: Cell<u32>,
60    /// Load event blocker. Will block the load event while the poster frame
61    /// is being fetched.
62    load_blocker: DomRefCell<Option<LoadBlocker>>,
63    /// A copy of the last frame
64    #[ignore_malloc_size_of = "VideoFrame"]
65    #[no_trace]
66    last_frame: DomRefCell<Option<VideoFrame>>,
67    #[cfg(feature = "webgpu")]
68    #[conditional_malloc_size_of]
69    /// Planar texture for WebGPU
70    planar_texture: DomRefCell<Option<Rc<crate::dom::gpuexternaltexture::PlanarTexture>>>,
71}
72
73impl HTMLVideoElement {
74    fn new_inherited(
75        local_name: LocalName,
76        prefix: Option<Prefix>,
77        document: &Document,
78    ) -> HTMLVideoElement {
79        HTMLVideoElement {
80            htmlmediaelement: HTMLMediaElement::new_inherited(local_name, prefix, document),
81            video_width: Cell::new(None),
82            video_height: Cell::new(None),
83            generation_id: Cell::new(0),
84            load_blocker: Default::default(),
85            last_frame: Default::default(),
86            #[cfg(feature = "webgpu")]
87            planar_texture: DomRefCell::new(None),
88        }
89    }
90
91    pub(crate) fn new(
92        cx: &mut JSContext,
93        local_name: LocalName,
94        prefix: Option<Prefix>,
95        document: &Document,
96        proto: Option<HandleObject>,
97    ) -> DomRoot<HTMLVideoElement> {
98        Node::reflect_node_with_proto(
99            cx,
100            Box::new(HTMLVideoElement::new_inherited(
101                local_name, prefix, document,
102            )),
103            document,
104            proto,
105        )
106    }
107
108    pub(crate) fn get_video_width(&self) -> Option<u32> {
109        self.video_width.get()
110    }
111
112    pub(crate) fn get_video_height(&self) -> Option<u32> {
113        self.video_height.get()
114    }
115
116    pub(crate) fn set_natural_dimensions(&self, width: Option<u32>, height: Option<u32>) -> bool {
117        if self.video_width.get() == width && self.video_height.get() == height {
118            return false;
119        }
120
121        self.video_width.set(width);
122        self.video_height.set(height);
123
124        self.upcast::<Node>().dirty(NodeDamage::Other);
125        true
126    }
127
128    /// Gets the copy of the video frame at the current playback position,
129    /// if that is available, or else (e.g. when the video is seeking or buffering)
130    /// its previous appearance, if any.
131    pub(crate) fn get_current_frame_data(&self) -> Option<Snapshot> {
132        let frame = self.htmlmediaelement.get_current_frame();
133        if frame.is_some() {
134            *self.last_frame.borrow_mut() = frame;
135        }
136
137        match self.last_frame.borrow().as_ref() {
138            Some(frame) => {
139                let size = Size2D::new(frame.get_width() as u32, frame.get_height() as u32);
140                if !frame.is_gl_texture() {
141                    let alpha_mode = SnapshotAlphaMode::Transparent {
142                        premultiplied: false,
143                    };
144
145                    Some(Snapshot::from_vec(
146                        size.cast(),
147                        SnapshotPixelFormat::BGRA,
148                        alpha_mode,
149                        frame.get_data().to_vec(),
150                    ))
151                } else {
152                    // XXX(victor): here we only have the GL texture ID.
153                    Some(Snapshot::cleared(size.cast()))
154                }
155            },
156            None => None,
157        }
158    }
159
160    #[cfg(feature = "webgpu")]
161    pub(crate) fn planar_video_for_webgpu(
162        &self,
163        device: &crate::dom::types::GPUDevice,
164    ) -> Fallible<(
165        Size2D<u32>,
166        Option<Rc<crate::dom::gpuexternaltexture::PlanarTexture>>,
167    )> {
168        use crate::dom::gpuexternaltexture::PlanarTexture;
169        // 1. If source is not origin-clean, throw a SecurityError and return.
170        if !self.origin_is_clean() {
171            return Err(script_bindings::error::Error::Security(Some(
172                "Source is not origin-clean".to_string(),
173            )));
174        }
175        // 2. Let usability be ? check the usability of the image argument(source).
176        if !self.is_usable() {
177            // 3. If usability is not good:
178            // Generate a validation error.
179            // Return an invalidated GPUExternalTexture.
180            Ok((Size2D::zero(), None))
181        } else {
182            // 4. Let data be the result of converting the current image contents of source into the color space descriptor.colorSpace with unpremultiplied alpha.
183            let mut planar_texture = self.planar_texture.borrow_mut();
184            match planar_texture.as_ref() {
185                Some(planar_texture) => {
186                    if planar_texture.is_expired() &&
187                        let Some(snapshot) = self.get_current_frame_data()
188                    {
189                        planar_texture.update(snapshot);
190                    }
191                },
192                None => {
193                    *planar_texture = self.get_current_frame_data().map(|snapshot| {
194                        Rc::new(PlanarTexture::new(device.channel(), device, snapshot))
195                    });
196                },
197            };
198            Ok((
199                planar_texture
200                    .as_ref()
201                    .map(|pt| pt.size())
202                    .unwrap_or_default(),
203                planar_texture.as_ref().cloned(),
204            ))
205        }
206    }
207
208    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
209    fn update_poster_frame(&self, poster_url: Option<&str>, cx: &mut JSContext) {
210        // Step 1. If there is an existing instance of this algorithm running
211        // for this video element, abort that instance of this algorithm without
212        // changing the poster frame.
213        self.generation_id.set(self.generation_id.get() + 1);
214
215        // Step 2. If the poster attribute's value is the empty string or
216        // if the attribute is absent, then there is no poster frame; return.
217        let Some(poster_url) = poster_url.filter(|poster_url| !poster_url.is_empty()) else {
218            self.htmlmediaelement.set_poster_frame(None);
219            return;
220        };
221
222        // Step 3. Let url be the result of encoding-parsing a URL given
223        // the poster attribute's value, relative to the element's node
224        // document.
225        // Step 4. If url is failure, then return. There is no poster frame.
226        let global = self.owner_global();
227        let poster_url = match self
228            .owner_document()
229            .encoding_parse_a_url(poster_url)
230            .map(|url| ensure_blob_referenced_by_url_is_kept_alive(&global, url))
231        {
232            Ok(url) => url,
233            Err(_) => {
234                self.htmlmediaelement.set_poster_frame(None);
235                return;
236            },
237        };
238
239        // We use the image cache for poster frames so we save as much
240        // network activity as possible.
241        let window = self.owner_window();
242        let image_cache = window.image_cache();
243        let cache_result = image_cache.get_cached_image_status(
244            poster_url.url(),
245            window.origin().immutable().clone(),
246            None,
247        );
248
249        let id = match cache_result {
250            ImageCacheResult::Available(ImageOrMetadataAvailable::ImageAvailable {
251                image,
252                url,
253                ..
254            }) => {
255                self.process_image_response(ImageResponse::Loaded(image, url), cx);
256                return;
257            },
258            ImageCacheResult::Available(ImageOrMetadataAvailable::MetadataAvailable(_, id)) => id,
259            ImageCacheResult::ReadyForRequest(id) => {
260                self.do_fetch_poster_frame(poster_url, id, cx);
261                id
262            },
263            ImageCacheResult::FailedToLoadOrDecode => {
264                self.process_image_response(ImageResponse::FailedToLoadOrDecode, cx);
265                return;
266            },
267            ImageCacheResult::Pending(id) => id,
268        };
269
270        let trusted_node = Trusted::new(self);
271        let generation = self.generation_id();
272        let callback = window.register_image_cache_listener(id, move |response, cx| {
273            let element = trusted_node.root();
274
275            // Ignore any image response for a previous request that has been discarded.
276            if generation != element.generation_id() {
277                return;
278            }
279            element.process_image_response(response.response, cx);
280        });
281
282        image_cache.add_listener(ImageLoadListener::new(callback, window.pipeline_id(), id));
283    }
284
285    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
286    fn do_fetch_poster_frame(
287        &self,
288        poster_url: UrlWithBlobClaim,
289        id: PendingImageId,
290        cx: &mut JSContext,
291    ) {
292        // Step 5. Let request be a new request whose URL is url, client is the element's node
293        // document's relevant settings object, destination is "image", initiator type is "video",
294        // credentials mode is "include", and whose use-URL-credentials flag is set.
295        let document = self.owner_document();
296        let global = self.owner_global();
297        let request = RequestBuilder::new(
298            Some(document.webview_id()),
299            poster_url.clone(),
300            global.get_referrer(),
301        )
302        .destination(Destination::Image)
303        .credentials_mode(CredentialsMode::Include)
304        .use_url_credentials(true)
305        .with_global_scope(&global);
306
307        // Step 6. Fetch request. This must delay the load event of the element's node document.
308        // This delay must be independent from the ones created by HTMLMediaElement during
309        // its media load algorithm, otherwise a code like
310        // <video poster="poster.png"></video>
311        // (which triggers no media load algorithm unless a explicit call to .load() is done)
312        // will block the document's load event forever.
313        let blocker = &self.load_blocker;
314        LoadBlocker::terminate(blocker, cx);
315        let document = self.owner_document();
316        *blocker.borrow_mut() = Some(LoadBlocker::new(
317            &document,
318            LoadType::Image(poster_url.url()),
319        ));
320
321        let context = PosterFrameFetchContext::new(
322            self,
323            poster_url.url(),
324            id,
325            request.id,
326            self.global().core_resource_thread(),
327        );
328        self.owner_document().fetch_background(request, context);
329    }
330
331    fn generation_id(&self) -> u32 {
332        self.generation_id.get()
333    }
334
335    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
336    fn process_image_response(&self, response: ImageResponse, cx: &mut JSContext) {
337        // Step 7. If an image is thus obtained, the poster frame is that image.
338        // Otherwise, there is no poster frame.
339        match response {
340            ImageResponse::Loaded(image, url) => {
341                debug!("Loaded poster image for video element: {:?}", url);
342                match image.as_raster_image() {
343                    Some(image) => self.htmlmediaelement.set_poster_frame(Some(image)),
344                    None => warn!("Vector images are not yet supported in video poster"),
345                }
346                LoadBlocker::terminate(&self.load_blocker, cx);
347            },
348            ImageResponse::MetadataLoaded(..) => {},
349            // The image cache may have loaded a placeholder for an invalid poster url
350            ImageResponse::FailedToLoadOrDecode => {
351                self.htmlmediaelement.set_poster_frame(None);
352                // A failed load should unblock the document load.
353                LoadBlocker::terminate(&self.load_blocker, cx);
354            },
355        }
356    }
357
358    /// <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
359    pub(crate) fn is_usable(&self) -> bool {
360        !matches!(
361            self.htmlmediaelement.get_ready_state(),
362            ReadyState::HaveNothing | ReadyState::HaveMetadata
363        )
364    }
365
366    pub(crate) fn origin_is_clean(&self) -> bool {
367        self.htmlmediaelement.origin_is_clean()
368    }
369
370    pub(crate) fn is_network_state_empty(&self) -> bool {
371        self.htmlmediaelement.network_state() == NetworkState::Empty
372    }
373}
374
375impl HTMLVideoElementMethods<crate::DomTypeHolder> for HTMLVideoElement {
376    // <https://html.spec.whatwg.org/multipage/#dom-video-width>
377    make_dimension_uint_getter!(Width, "width");
378
379    // <https://html.spec.whatwg.org/multipage/#dom-video-width>
380    make_dimension_uint_setter!(SetWidth, "width");
381
382    // <https://html.spec.whatwg.org/multipage/#dom-video-height>
383    make_dimension_uint_getter!(Height, "height");
384
385    // <https://html.spec.whatwg.org/multipage/#dom-video-height>
386    make_dimension_uint_setter!(SetHeight, "height");
387
388    /// <https://html.spec.whatwg.org/multipage/#dom-video-videowidth>
389    fn VideoWidth(&self) -> u32 {
390        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
391            return 0;
392        }
393        self.video_width.get().unwrap_or(0)
394    }
395
396    /// <https://html.spec.whatwg.org/multipage/#dom-video-videoheight>
397    fn VideoHeight(&self) -> u32 {
398        if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing {
399            return 0;
400        }
401        self.video_height.get().unwrap_or(0)
402    }
403
404    // https://html.spec.whatwg.org/multipage/#dom-video-poster
405    make_url_getter!(Poster, "poster");
406
407    // https://html.spec.whatwg.org/multipage/#dom-video-poster
408    make_url_setter!(SetPoster, "poster");
409
410    // For testing purposes only. This is not an event from
411    // https://html.spec.whatwg.org/multipage/#dom-video-poster
412    event_handler!(postershown, GetOnpostershown, SetOnpostershown);
413}
414
415impl VirtualMethods for HTMLVideoElement {
416    fn super_type(&self) -> Option<&dyn VirtualMethods> {
417        Some(self.upcast::<HTMLMediaElement>() as &dyn VirtualMethods)
418    }
419
420    fn attribute_mutated(
421        &self,
422        cx: &mut JSContext,
423        attr: AttrRef<'_>,
424        mutation: AttributeMutation,
425    ) {
426        self.super_type()
427            .unwrap()
428            .attribute_mutated(cx, attr, mutation);
429
430        if attr.local_name() == &local_name!("poster") {
431            if let Some(new_value) = mutation.new_value(attr) {
432                self.update_poster_frame(Some(&new_value), cx)
433            } else {
434                self.update_poster_frame(None, cx)
435            }
436        };
437    }
438
439    fn attribute_affects_presentational_hints(&self, attr: AttrRef<'_>) -> bool {
440        match attr.local_name() {
441            &local_name!("width") | &local_name!("height") => true,
442            _ => self
443                .super_type()
444                .unwrap()
445                .attribute_affects_presentational_hints(attr),
446        }
447    }
448
449    fn parse_plain_attribute(&self, name: &LocalName, value: DOMString) -> AttrValue {
450        match name {
451            &local_name!("width") | &local_name!("height") => {
452                AttrValue::from_dimension(value.into())
453            },
454            _ => self
455                .super_type()
456                .unwrap()
457                .parse_plain_attribute(name, value),
458        }
459    }
460}
461
462struct PosterFrameFetchContext {
463    /// Reference to the script thread image cache.
464    image_cache: Arc<dyn ImageCache>,
465    /// The element that initiated the request.
466    elem: Trusted<HTMLVideoElement>,
467    /// The cache ID for this request.
468    id: PendingImageId,
469    /// True if this response is invalid and should be ignored.
470    cancelled: bool,
471    /// Url for the resource
472    url: ServoUrl,
473    /// A [`FetchCanceller`] for this request.
474    fetch_canceller: FetchCanceller,
475}
476
477impl FetchResponseListener for PosterFrameFetchContext {
478    fn process_request_body(&mut self, _: RequestId) {
479        self.fetch_canceller.ignore()
480    }
481
482    fn process_response(
483        &mut self,
484        _: &mut JSContext,
485        request_id: RequestId,
486        metadata: Result<FetchMetadata, NetworkError>,
487    ) {
488        self.image_cache.notify_pending_response(
489            self.id,
490            FetchResponseMsg::ProcessResponse(request_id, metadata.clone()),
491        );
492
493        let metadata = metadata.ok().map(|meta| match meta {
494            FetchMetadata::Unfiltered(m) => m,
495            FetchMetadata::Filtered { unsafe_, .. } => unsafe_,
496        });
497
498        let status_is_ok = metadata
499            .as_ref()
500            .is_none_or(|m| m.status.in_range(200..300));
501
502        if !status_is_ok {
503            self.cancelled = true;
504            self.fetch_canceller.abort();
505        }
506    }
507
508    fn process_response_chunk(
509        &mut self,
510        _: &mut JSContext,
511        request_id: RequestId,
512        payload: Vec<u8>,
513    ) {
514        if self.cancelled {
515            // An error was received previously, skip processing the payload.
516            return;
517        }
518
519        self.image_cache.notify_pending_response(
520            self.id,
521            FetchResponseMsg::ProcessResponseChunk(request_id, payload.into()),
522        );
523    }
524
525    fn process_response_eof(
526        self,
527        cx: &mut JSContext,
528        request_id: RequestId,
529        response: Result<(), NetworkError>,
530        timing: ResourceFetchTiming,
531    ) {
532        self.image_cache.notify_pending_response(
533            self.id,
534            FetchResponseMsg::ProcessResponseEOF(request_id, response.clone(), timing.clone()),
535        );
536        network_listener::submit_timing(cx, &self, &response, &timing);
537    }
538
539    fn process_csp_violations(
540        &mut self,
541        cx: &mut js::context::JSContext,
542        _request_id: RequestId,
543        violations: Vec<Violation>,
544    ) {
545        let global = &self.resource_timing_global();
546        global.report_csp_violations(cx, violations, None, None);
547    }
548}
549
550impl ResourceTimingListener for PosterFrameFetchContext {
551    fn resource_timing_information(&self) -> (InitiatorType, ServoUrl) {
552        let initiator_type = InitiatorType::LocalName(
553            self.elem
554                .root()
555                .upcast::<Element>()
556                .local_name()
557                .to_string(),
558        );
559        (initiator_type, self.url.clone())
560    }
561
562    fn resource_timing_global(&self) -> DomRoot<GlobalScope> {
563        self.elem.root().owner_document().global()
564    }
565}
566
567impl PosterFrameFetchContext {
568    fn new(
569        elem: &HTMLVideoElement,
570        url: ServoUrl,
571        id: PendingImageId,
572        request_id: RequestId,
573        core_resource_thread: CoreResourceThread,
574    ) -> PosterFrameFetchContext {
575        let window = elem.owner_window();
576        PosterFrameFetchContext {
577            image_cache: window.image_cache(),
578            elem: Trusted::new(elem),
579            id,
580            cancelled: false,
581            url,
582            fetch_canceller: FetchCanceller::new(request_id, false, core_resource_thread),
583        }
584    }
585}
586
587impl LayoutDom<'_, HTMLVideoElement> {
588    pub(crate) fn data(self) -> HTMLMediaData {
589        let video = self.unsafe_get();
590
591        // Get the current frame being rendered.
592        let current_frame = video.htmlmediaelement.get_current_frame_to_present();
593
594        // This value represents the natural width and height of the video.
595        // It may exist even if there is no current frame (for example, after the
596        // metadata of the video is loaded).
597        let metadata = video
598            .get_video_width()
599            .zip(video.get_video_height())
600            .map(|(width, height)| MediaMetadata { width, height });
601
602        HTMLMediaData {
603            current_frame,
604            metadata,
605        }
606    }
607
608    pub(crate) fn get_width(self) -> LengthOrPercentageOrAuto {
609        self.upcast::<Element>()
610            .get_attr_for_layout(&ns!(), &local_name!("width"))
611            .map(AttrValue::as_dimension)
612            .cloned()
613            .unwrap_or(LengthOrPercentageOrAuto::Auto)
614    }
615
616    pub(crate) fn get_height(self) -> LengthOrPercentageOrAuto {
617        self.upcast::<Element>()
618            .get_attr_for_layout(&ns!(), &local_name!("height"))
619            .map(AttrValue::as_dimension)
620            .cloned()
621            .unwrap_or(LengthOrPercentageOrAuto::Auto)
622    }
623}
script/dom/html/htmlvideoelement.rs

script/dom/html/
htmlvideoelement.rs
